Skip to content

Commit

Permalink
feat: added modules
Browse files Browse the repository at this point in the history
  • Loading branch information
tikazyq committed Jun 14, 2024
1 parent 0b67fd9 commit dc21bce
Show file tree
Hide file tree
Showing 138 changed files with 3,231 additions and 120 deletions.
2 changes: 1 addition & 1 deletion core/controllers/base_v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ package controllers

import (
"errors"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/base_v2_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"net/http/httptest"
"testing"

"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
)
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/data_collection.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package controllers

import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson/primitive"
mongo2 "go.mongodb.org/mongo-driver/mongo"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/data_source.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package controllers

import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/ds"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
Expand All @@ -10,6 +9,7 @@ import (
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/go-trace"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson/primitive"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/delegate_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ package controllers

import (
"github.com/apex/log"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/errors"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/delegate"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/go-trace"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/filter.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package controllers

import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
mongo2 "go.mongodb.org/mongo-driver/mongo"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/filter_v2.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package controllers

import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
mongo2 "go.mongodb.org/mongo-driver/mongo"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/result.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package controllers

import (
"github.com/crawlab-team/crawlab-db/generic"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/result"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/result_v2.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package controllers

import (
"github.com/crawlab-team/crawlab-db/generic"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/result"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/spider.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package controllers
import (
"bytes"
"fmt"
"github.com/crawlab-team/crawlab-db/mongo"
vcs "github.com/crawlab-team/crawlab-vcs"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/container"
Expand All @@ -15,6 +14,7 @@ import (
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/go-trace"
"github.com/gin-gonic/gin"
"github.com/go-git/go-git/v5"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/spider_v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"errors"
"fmt"
log2 "github.com/apex/log"
"github.com/crawlab-team/crawlab-db/mongo"
vcs "github.com/crawlab-team/crawlab-vcs"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
Expand All @@ -14,6 +13,7 @@ import (
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/spider/admin"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/go-trace"
"github.com/gin-gonic/gin"
"github.com/go-git/go-git/v5"
Expand Down
4 changes: 2 additions & 2 deletions core/controllers/task.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package controllers

import (
"github.com/crawlab-team/crawlab-db/generic"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/errors"
Expand All @@ -13,6 +11,8 @@ import (
"github.com/crawlab-team/crawlab/core/result"
"github.com/crawlab-team/crawlab/core/task/log"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
Expand Down
4 changes: 2 additions & 2 deletions core/controllers/task_v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ package controllers
import (
"errors"
log2 "github.com/apex/log"
"github.com/crawlab-team/crawlab-db/generic"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
Expand All @@ -14,6 +12,8 @@ import (
"github.com/crawlab-team/crawlab/core/task/log"
"github.com/crawlab-team/crawlab/core/task/scheduler"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/gin-gonic/gin"
"github.com/spf13/viper"
"go.mongodb.org/mongo-driver/bson"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/test/export_test.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package test

import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
"go.mongodb.org/mongo-driver/bson"
Expand Down
2 changes: 1 addition & 1 deletion core/controllers/test/filter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package test
import (
"encoding/json"
"fmt"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
"go.mongodb.org/mongo-driver/bson"
Expand Down
2 changes: 1 addition & 1 deletion core/ds/es.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/crawlab-team/crawlab-db/generic"
"github.com/crawlab-team/crawlab/core/constants"
constants2 "github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
Expand All @@ -14,6 +13,7 @@ import (
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/go-trace"
"github.com/elastic/go-elasticsearch/v8"
"github.com/elastic/go-elasticsearch/v8/esapi"
Expand Down
2 changes: 1 addition & 1 deletion core/ds/kafka.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ package ds

import (
"github.com/cenkalti/backoff/v4"
"github.com/crawlab-team/crawlab-db/generic"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/go-trace"
"github.com/segmentio/kafka-go"
"go.mongodb.org/mongo-driver/bson/primitive"
Expand Down
4 changes: 2 additions & 2 deletions core/ds/mongo.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
package ds

import (
"github.com/crawlab-team/crawlab-db/generic"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/utils"
utils2 "github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson/primitive"
mongo2 "go.mongodb.org/mongo-driver/mongo"
"time"
Expand Down
2 changes: 1 addition & 1 deletion core/ds/sql.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package ds

import (
"github.com/crawlab-team/crawlab-db/generic"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
utils2 "github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/generic"
"github.com/crawlab-team/go-trace"
"github.com/upper/db/v4"
"time"
Expand Down
2 changes: 1 addition & 1 deletion core/entity/grpc_base_service_params.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package entity

import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
Expand Down
2 changes: 1 addition & 1 deletion core/export/csv_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ import (
"fmt"
"github.com/ReneKroon/ttlcache"
"github.com/apex/log"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/go-trace"
"github.com/hashicorp/go-uuid"
"go.mongodb.org/mongo-driver/bson"
Expand Down
2 changes: 1 addition & 1 deletion core/export/csv_service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ package export
import (
"encoding/csv"
"fmt"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/stretchr/testify/require"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
Expand Down
2 changes: 1 addition & 1 deletion core/export/json_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ import (
"errors"
"github.com/ReneKroon/ttlcache"
"github.com/apex/log"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
"github.com/crawlab-team/go-trace"
"github.com/hashicorp/go-uuid"
mongo2 "go.mongodb.org/mongo-driver/mongo"
Expand Down
7 changes: 4 additions & 3 deletions core/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,17 @@ module github.com/crawlab-team/crawlab/core
go 1.22

replace (
github.com/crawlab-team/crawlab/grpc latest => ../grpc latest
github.com/crawlab-team/crawlab/grpc => ../grpc
github.com/crawlab-team/crawlab/db => ../db
)

require (
github.com/ReneKroon/ttlcache v1.7.0
github.com/apex/log v1.9.0
github.com/cenkalti/backoff/v4 v4.1.0
github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484
github.com/crawlab-team/crawlab/db v0.0.0
github.com/crawlab-team/crawlab-fs v0.6.3-2
github.com/crawlab-team/crawlab/grpc latest
github.com/crawlab-team/crawlab/grpc v0.0.0
github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185
github.com/crawlab-team/go-trace v0.1.1
github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e
Expand Down
10 changes: 3 additions & 7 deletions core/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -151,15 +151,11 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/crawlab-team/crawlab-db v0.6.0-1/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU=
github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 h1:1CXWC3lYcVWcgPRc3PNKzZ3fcfX5WZ/V8xwzHEMUFHQ=
github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU=
github.com/crawlab-team/crawlab/db v0.6.0-1/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU=
github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 h1:1CXWC3lYcVWcgPRc3PNKzZ3fcfX5WZ/V8xwzHEMUFHQ=
github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU=
github.com/crawlab-team/crawlab-fs v0.6.3-2 h1:GAovTF1R1PLoQgj+0F1GpePrlp1k7RtW/jK8p0L9HhA=
github.com/crawlab-team/crawlab-fs v0.6.3-2/go.mod h1:Nob0uFr82IPbkk6LEYG0BAB2NgJ3PKoNVhtcbf5fLf0=
github.com/crawlab-team/crawlab/grpc v0.6.4-0.20240614065204-c753fc33f7ca h1:7LOFAWCIK+153f5J437xXhlI1miUQ3mrm3OXGq2/m3U=
github.com/crawlab-team/crawlab/grpc v0.6.4-0.20240614065204-c753fc33f7ca/go.mod h1:j5FaFuWfIxHbmFXedFaUZHO2DpLqS3QZ0x7W3APN0fQ=
github.com/crawlab-team/crawlab/grpc v0.6.4-0.20240614072247-7558f2150cee h1:jlT64UPWbf5Xny3jOzKE35od1F+OuGK+y8PTNomyxiM=
github.com/crawlab-team/crawlab/grpc v0.6.4-0.20240614072247-7558f2150cee/go.mod h1:j5FaFuWfIxHbmFXedFaUZHO2DpLqS3QZ0x7W3APN0fQ=
github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 h1:A/XSUuGgGMn+z+lFd2ye2ClgIKhDZYUerhOL5jePQhU=
github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185/go.mod h1:YHMYUEoSqfXUZHsWW/M/DaLh/zOpRtiElaRWcrGyv/I=
github.com/crawlab-team/go-trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8=
Expand Down
2 changes: 1 addition & 1 deletion core/grpc/payload/model_service_v2_payload.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package payload

import (
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
Expand Down
2 changes: 1 addition & 1 deletion core/grpc/server/model_base_service_v2_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package server
import (
"context"
"encoding/json"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
grpc "github.com/crawlab-team/crawlab/grpc"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
Expand Down
2 changes: 1 addition & 1 deletion core/grpc/server/task_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"github.com/apex/log"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/container"
"github.com/crawlab-team/crawlab/core/entity"
Expand All @@ -15,6 +14,7 @@ import (
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/notification"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/go-trace"
"go.mongodb.org/mongo-driver/bson"
Expand Down
2 changes: 1 addition & 1 deletion core/grpc/server/task_server_v2.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"github.com/apex/log"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"github.com/crawlab-team/crawlab/core/interfaces"
Expand All @@ -15,6 +14,7 @@ import (
"github.com/crawlab-team/crawlab/core/notification"
"github.com/crawlab-team/crawlab/core/task/stats"
"github.com/crawlab-team/crawlab/core/utils"
"github.com/crawlab-team/crawlab/db/mongo"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/go-trace"
"go.mongodb.org/mongo-driver/bson"
Expand Down
Loading

0 comments on commit dc21bce

Please sign in to comment.