diff --git a/constants/data_source.go b/constants/data_source.go deleted file mode 100644 index be39670..0000000 --- a/constants/data_source.go +++ /dev/null @@ -1,6 +0,0 @@ -package constants - -const ( - DataSourceStatusOnline = "on" - DataSourceStatusOffline = "off" -) diff --git a/constants/ds.go b/constants/ds.go new file mode 100644 index 0000000..1a92a8a --- /dev/null +++ b/constants/ds.go @@ -0,0 +1,31 @@ +package constants + +const ( + DataSourceTypeMongo = "mongo" + DataSourceTypeMysql = "mysql" + DataSourceTypePostgresql = "postgresql" + DataSourceTypeMssql = "mssql" + DataSourceTypeSqlite = "sqlite" + DataSourceTypeCockroachdb = "cockroachdb" + DataSourceTypeElasticSearch = "elasticsearch" + DataSourceTypeKafka = "kafka" +) + +const ( + DefaultHost = "localhost" +) + +const ( + DefaultMongoPort = "27017" + DefaultMysqlPort = "3306" + DefaultPostgresqlPort = "5432" + DefaultMssqlPort = "1433" + DefaultCockroachdbPort = "26257" + DefaultElasticsearchPort = "9200" + DefaultKafkaPort = "9092" +) + +const ( + DataSourceStatusOnline = "on" + DataSourceStatusOffline = "off" +) diff --git a/controllers/data_source.go b/controllers/data_source.go new file mode 100644 index 0000000..5bcc4f9 --- /dev/null +++ b/controllers/data_source.go @@ -0,0 +1,131 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab-core/ds" + "github.com/crawlab-team/crawlab-core/errors" + "github.com/crawlab-team/crawlab-core/interfaces" + interfaces2 "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/delegate" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + "github.com/crawlab-team/crawlab-core/utils" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "net/http" +) + +var DataSourceController *dataSourceController + +func getDataSourceActions() []Action { + ctx := newDataSourceContext() + return []Action{ + { + Path: "/:id/change-password", + Method: http.MethodPost, + HandlerFunc: ctx.changePassword, + }, + } +} + +type dataSourceController struct { + ListActionControllerDelegate + d ListActionControllerDelegate + ctx *dataSourceContext +} + +func (ctr *dataSourceController) Post(c *gin.Context) { + // data source + var _ds models.DataSource + if err := c.ShouldBindJSON(&_ds); err != nil { + HandleErrorBadRequest(c, err) + return + } + + // add data source to db + if err := mongo.RunTransaction(func(ctx mongo2.SessionContext) error { + if err := delegate.NewModelDelegate(&_ds).Add(); err != nil { + return trace.TraceError(err) + } + pwd, err := utils.EncryptAES(_ds.Password) + if err != nil { + return trace.TraceError(err) + } + p := models.Password{Id: _ds.Id, Password: pwd} + if err := delegate.NewModelDelegate(&p).Add(); err != nil { + return trace.TraceError(err) + } + return nil + }); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // check data source status + go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }() + + HandleSuccess(c) +} + +type dataSourceContext struct { + dsSvc interfaces.DataSourceService +} + +var _dataSourceCtx *dataSourceContext + +func newDataSourceContext() *dataSourceContext { + if _dataSourceCtx != nil { + return _dataSourceCtx + } + dsSvc, err := ds.GetDataSourceService() + if err != nil { + panic(err) + } + _dataSourceCtx = &dataSourceContext{ + dsSvc: dsSvc, + } + return _dataSourceCtx +} + +func (ctx *dataSourceContext) changePassword(c *gin.Context) { + id, err := primitive.ObjectIDFromHex(c.Param("id")) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + var payload map[string]string + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + password, ok := payload["password"] + if !ok { + HandleErrorBadRequest(c, errors.ErrorDataSourceMissingRequiredFields) + return + } + if err := ctx.dsSvc.ChangePassword(id, password); err != nil { + HandleErrorInternalServerError(c, err) + return + } + HandleSuccess(c) +} + +func newDataSourceController() *dataSourceController { + actions := getDataSourceActions() + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + + ctr := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions) + d := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions) + ctx := newDataSourceContext() + + return &dataSourceController{ + ListActionControllerDelegate: *ctr, + d: *d, + ctx: ctx, + } +} diff --git a/controllers/init.go b/controllers/init.go index e55385a..8b5c513 100644 --- a/controllers/init.go +++ b/controllers/init.go @@ -39,6 +39,7 @@ func InitControllers() (err error) { EnvDepsController = NewActionControllerDelegate(ControllerIdEnvDeps, getEnvDepsActions()) NotificationController = NewActionControllerDelegate(ControllerIdNotification, getNotificationActions()) FilterController = NewActionControllerDelegate(ControllerIdFilter, getFilterActions()) + DataSourceController = newDataSourceController() return nil } diff --git a/ds/cockroachdb.go b/ds/cockroachdb.go new file mode 100644 index 0000000..18ee499 --- /dev/null +++ b/ds/cockroachdb.go @@ -0,0 +1,70 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + "github.com/crawlab-team/crawlab-core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type CockroachdbService struct { + SqlService +} + +func NewDataSourceCockroachdbService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &CockroachdbService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, trace.TraceError(err) + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, trace.TraceError(err) + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultCockroachdbPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, trace.TraceError(err) + } + + // session + svc.s, err = utils.GetCockroachdbSession(svc.ds) + if err != nil { + return nil, trace.TraceError(err) + } + + // collection + svc.col = svc.s.Collection(svc.dc.Name) + + return svc, nil +} diff --git a/ds/default.go b/ds/default.go new file mode 100644 index 0000000..e5143a2 --- /dev/null +++ b/ds/default.go @@ -0,0 +1 @@ +package ds diff --git a/ds/es.go b/ds/es.go new file mode 100644 index 0000000..b02bc5c --- /dev/null +++ b/ds/es.go @@ -0,0 +1,220 @@ +package ds + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/crawlab-team/crawlab-core/constants" + constants2 "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/entity" + entity2 "github.com/crawlab-team/crawlab-core/entity" + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + "github.com/crawlab-team/crawlab-core/utils" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/go-trace" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "github.com/google/uuid" + "go.mongodb.org/mongo-driver/bson/primitive" + "strings" + "sync" + "time" +) + +type ElasticsearchService struct { + // dependencies + modelSvc service.ModelService + + // internals + dc *models.DataCollection // models.DataCollection + ds *models.DataSource // models.DataSource + c *elasticsearch.Client // elasticsearch.Client + t time.Time +} + +func (svc *ElasticsearchService) Insert(records ...interface{}) (err error) { + // wait group + var wg sync.WaitGroup + wg.Add(len(records)) + + // iterate records + for _, r := range records { + // async operation + go func(r interface{}) { + switch r.(type) { + case entity.Result: + // convert type to entity.Result + d := r.(entity.Result) + + // get document id + id := d.GetValue("id") + var docId string + switch id.(type) { + case string: + docId = id.(string) + } + if docId == "" { + docId = uuid.New().String() // generate new uuid if id is empty + } + + // collection + d[constants2.DataCollectionKey] = svc.dc.Name + + // index request + req := esapi.IndexRequest{ + Index: svc.getIndexName(), + DocumentID: docId, + Body: strings.NewReader(d.String()), + } + + // perform request + res, err := req.Do(context.Background(), svc.c) + if err != nil { + trace.PrintError(err) + wg.Done() + return + } + defer res.Body.Close() + if res.IsError() { + trace.PrintError(errors.New(fmt.Sprintf("[ElasticsearchService] [%s] error inserting record: %v", res.Status(), r))) + } + + // release + wg.Done() + default: + wg.Done() + return + } + }(r) + } + + // wait + wg.Wait() + + return nil +} + +func (svc *ElasticsearchService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { + data, err := svc.getListResponse(query, opts, false) + if err != nil { + return nil, err + } + for _, hit := range data.Hits.Hits { + results = append(results, hit.Source) + } + return results, nil +} + +func (svc *ElasticsearchService) Count(query generic.ListQuery) (n int, err error) { + data, err := svc.getListResponse(query, nil, true) + if err != nil { + return n, err + } + return int(data.Hits.Total.Value), nil +} + +func (svc *ElasticsearchService) getListResponse(query generic.ListQuery, opts *generic.ListOptions, trackTotalHits bool) (data *entity2.ElasticsearchResponseData, err error) { + if opts == nil { + opts = &generic.ListOptions{} + } + query = append(query, generic.ListQueryCondition{ + Key: constants2.DataCollectionKey, + Op: constants2.FilterOpEqual, + Value: svc.dc.Name, + }) + res, err := svc.c.Search( + svc.c.Search.WithContext(context.Background()), + svc.c.Search.WithIndex(svc.getIndexName()), + svc.c.Search.WithBody(utils.GetElasticsearchQueryWithOptions(query, opts)), + svc.c.Search.WithTrackTotalHits(trackTotalHits), + ) + if err != nil { + return nil, trace.TraceError(err) + } + defer res.Body.Close() + if res.IsError() { + err = errors.New(fmt.Sprintf("[ElasticsearchService] [%s] error listing records: response=%s, query=%v opts=%v", res.Status(), res.String(), query, opts)) + trace.PrintError(err) + return nil, err + } + data = &entity2.ElasticsearchResponseData{} + if err := json.NewDecoder(res.Body).Decode(data); err != nil { + return nil, trace.TraceError(err) + } + return data, nil +} + +func (svc *ElasticsearchService) getIndexName() (index string) { + if svc.ds.Database == "" { + return svc.dc.Name + } else { + return svc.ds.Name + } +} + +func NewDataSourceElasticsearchService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &ElasticsearchService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, err + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultElasticsearchPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, err + } + + // client + svc.c, err = utils.GetElasticsearchClient(svc.ds) + if err != nil { + return nil, err + } + + return svc, nil +} + +func (svc *ElasticsearchService) Index(fields []string) { + // TODO: implement me +} + +func (svc *ElasticsearchService) SetTime(t time.Time) { + svc.t = t +} + +func (svc *ElasticsearchService) GetTime() (t time.Time) { + return svc.t +} diff --git a/ds/kafka.go b/ds/kafka.go new file mode 100644 index 0000000..0f11cbe --- /dev/null +++ b/ds/kafka.go @@ -0,0 +1,116 @@ +package ds + +import ( + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/entity" + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + "github.com/crawlab-team/crawlab-core/utils" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/go-trace" + "github.com/segmentio/kafka-go" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type KafkaService struct { + // dependencies + modelSvc service.ModelService + + // internals + dc *models.DataCollection // models.DataCollection + ds *models.DataSource // models.DataSource + c *kafka.Conn // kafka.Conn + rb backoff.BackOff + t time.Time +} + +func (svc *KafkaService) Insert(records ...interface{}) (err error) { + var messages []kafka.Message + for _, r := range records { + switch r.(type) { + case entity.Result: + d := r.(entity.Result) + messages = append(messages, kafka.Message{ + Topic: svc.ds.Database, + Key: []byte(d.GetTaskId().Hex()), + Value: d.Bytes(), + }) + } + } + _, err = svc.c.WriteMessages(messages...) + if err != nil { + return trace.TraceError(err) + } + return nil +} + +func (svc *KafkaService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { + // N/A + return nil, nil +} + +func (svc *KafkaService) Count(query generic.ListQuery) (n int, err error) { + // N/A + return 0, nil +} + +func NewDataSourceKafkaService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &KafkaService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, err + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultKafkaPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, err + } + + return svc, nil +} + +func (svc *KafkaService) Index(fields []string) { + // TODO: implement me +} + +func (svc *KafkaService) SetTime(t time.Time) { + svc.t = t +} + +func (svc *KafkaService) GetTime() (t time.Time) { + return svc.t +} diff --git a/ds/mongo.go b/ds/mongo.go new file mode 100644 index 0000000..709b1e8 --- /dev/null +++ b/ds/mongo.go @@ -0,0 +1,118 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + "github.com/crawlab-team/crawlab-core/utils" + utils2 "github.com/crawlab-team/crawlab-core/utils" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab-db/mongo" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type MongoService struct { + // dependencies + modelSvc service.ModelService + + // internals + dc *models.DataCollection // models.DataCollection + ds *models.DataSource // models.DataSource + c *mongo2.Client + db *mongo2.Database + col *mongo.Col + t time.Time +} + +func (svc *MongoService) Insert(records ...interface{}) (err error) { + _, err = svc.col.InsertMany(records) + return err +} + +func (svc *MongoService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { + var docs []models.Result + if err := svc.col.Find(utils.GetMongoQuery(query), utils.GetMongoOpts(opts)).All(&docs); err != nil { + return nil, err + } + for i := range docs { + results = append(results, &docs[i]) + } + return results, nil +} + +func (svc *MongoService) Count(query generic.ListQuery) (n int, err error) { + return svc.col.Count(utils.GetMongoQuery(query)) +} + +func NewDataSourceMongoService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &MongoService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, err + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultMongoPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, err + } + + // mongo client + svc.c, err = utils2.GetMongoClient(svc.ds) + if err != nil { + return nil, err + } + + // mongo database + svc.db = mongo.GetMongoDb(svc.ds.Database, mongo.WithDbClient(svc.c)) + + // mongo col + svc.col = mongo.GetMongoColWithDb(svc.dc.Name, svc.db) + + return svc, nil +} + +func (svc *MongoService) Index(fields []string) { + // TODO: implement me +} + +func (svc *MongoService) SetTime(t time.Time) { + svc.t = t +} + +func (svc *MongoService) GetTime() (t time.Time) { + return svc.t +} diff --git a/ds/mssql.go b/ds/mssql.go new file mode 100644 index 0000000..a561899 --- /dev/null +++ b/ds/mssql.go @@ -0,0 +1,71 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + "github.com/crawlab-team/crawlab-core/utils" + utils2 "github.com/crawlab-team/crawlab-core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type MssqlService struct { + SqlService +} + +func NewDataSourceMssqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &MssqlService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, trace.TraceError(err) + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, trace.TraceError(err) + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultMssqlPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, trace.TraceError(err) + } + + // session + svc.s, err = utils2.GetMssqlSession(svc.ds) + if err != nil { + return nil, trace.TraceError(err) + } + + // collection + svc.col = svc.s.Collection(svc.dc.Name) + + return svc, nil +} diff --git a/ds/mssql_test.go b/ds/mssql_test.go new file mode 100644 index 0000000..229197c --- /dev/null +++ b/ds/mssql_test.go @@ -0,0 +1,8 @@ +package ds + +import "testing" + +func TestNewDataSourceMssqlService(t *testing.T) { + t.Run("insert", func(t *testing.T) { + }) +} diff --git a/ds/mysql.go b/ds/mysql.go new file mode 100644 index 0000000..65a4f28 --- /dev/null +++ b/ds/mysql.go @@ -0,0 +1,69 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + "github.com/crawlab-team/crawlab-core/utils" + utils2 "github.com/crawlab-team/crawlab-core/utils" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type MysqlService struct { + SqlService +} + +func NewDataSourceMysqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &MysqlService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, err + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultMysqlPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, err + } + + // session + svc.s, err = utils2.GetMysqlSession(svc.ds) + if err != nil { + return nil, err + } + + // collection + svc.col = svc.s.Collection(svc.dc.Name) + + return svc, nil +} diff --git a/ds/options.go b/ds/options.go new file mode 100644 index 0000000..920e053 --- /dev/null +++ b/ds/options.go @@ -0,0 +1,14 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab-core/interfaces" + "time" +) + +type DataSourceServiceOption func(svc interfaces.DataSourceService) + +func WithMonitorInterval(duration time.Duration) DataSourceServiceOption { + return func(svc interfaces.DataSourceService) { + svc.SetMonitorInterval(duration) + } +} diff --git a/ds/postgresql.go b/ds/postgresql.go new file mode 100644 index 0000000..c8931ae --- /dev/null +++ b/ds/postgresql.go @@ -0,0 +1,71 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + "github.com/crawlab-team/crawlab-core/utils" + utils2 "github.com/crawlab-team/crawlab-core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type PostgresqlService struct { + SqlService +} + +func NewDataSourcePostgresqlService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &PostgresqlService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, trace.TraceError(err) + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, trace.TraceError(err) + } + } + + // data source defaults + if svc.ds.Host == "" { + svc.ds.Host = constants.DefaultHost + } + if svc.ds.Port == "" { + svc.ds.Port = constants.DefaultPostgresqlPort + } + + // data source password + pwd, err := svc.modelSvc.GetPasswordById(svc.ds.Id) + if err == nil { + svc.ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return nil, err + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, trace.TraceError(err) + } + + // session + svc.s, err = utils2.GetPostgresqlSession(svc.ds) + if err != nil { + return nil, trace.TraceError(err) + } + + // collection + svc.col = svc.s.Collection(svc.dc.Name) + + return svc, nil +} diff --git a/ds/service.go b/ds/service.go new file mode 100644 index 0000000..dcd6618 --- /dev/null +++ b/ds/service.go @@ -0,0 +1,306 @@ +package ds + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab-core/constants" + constants2 "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/delegate" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + "github.com/crawlab-team/crawlab-core/result" + "github.com/crawlab-team/crawlab-core/utils" + utils2 "github.com/crawlab-team/crawlab-core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.uber.org/dig" + "sync" + "time" +) + +type Service struct { + // dependencies + modelSvc service.ModelService + + // internals + timeout time.Duration + monitorInterval time.Duration + stopped bool +} + +func (svc *Service) Init() { + // result service registry + reg := result.GetResultServiceRegistry() + + // register result services + reg.Register(constants.DataSourceTypeMongo, NewDataSourceMongoService) + reg.Register(constants.DataSourceTypeMysql, NewDataSourceMysqlService) + reg.Register(constants.DataSourceTypePostgresql, NewDataSourcePostgresqlService) + reg.Register(constants.DataSourceTypeMssql, NewDataSourceMssqlService) + reg.Register(constants.DataSourceTypeSqlite, NewDataSourceSqliteService) + reg.Register(constants.DataSourceTypeCockroachdb, NewDataSourceCockroachdbService) + reg.Register(constants.DataSourceTypeElasticSearch, NewDataSourceElasticsearchService) + reg.Register(constants.DataSourceTypeKafka, NewDataSourceKafkaService) +} + +func (svc *Service) Start() { + // start monitoring + go svc.Monitor() +} + +func (svc *Service) Wait() { + utils.DefaultWait() +} + +func (svc *Service) Stop() { + svc.stopped = true +} + +func (svc *Service) ChangePassword(id primitive.ObjectID, password string) (err error) { + p, err := svc.modelSvc.GetPasswordById(id) + if err == nil { + // exists, save + encryptedPassword, err := utils.EncryptAES(password) + if err != nil { + return err + } + p.Password = encryptedPassword + if err := delegate.NewModelDelegate(p).Save(); err != nil { + return err + } + return nil + } else if err.Error() == mongo.ErrNoDocuments.Error() { + // not exists, add + encryptedPassword, err := utils.EncryptAES(password) + if err != nil { + return err + } + p = &models.Password{ + Id: id, + Password: encryptedPassword, + } + if err := delegate.NewModelDelegate(p).Add(); err != nil { + return err + } + return nil + } else { + // error + return err + } +} + +func (svc *Service) Monitor() { + for { + // return if stopped + if svc.stopped { + return + } + + // monitor + if err := svc.monitor(); err != nil { + trace.PrintError(err) + } + + // wait + time.Sleep(svc.monitorInterval) + } +} + +func (svc *Service) CheckStatus(id primitive.ObjectID) (err error) { + ds, err := svc.modelSvc.GetDataSourceById(id) + if err != nil { + return err + } + return svc.checkStatus(ds, true) +} + +func (svc *Service) SetTimeout(duration time.Duration) { + svc.timeout = duration +} + +func (svc *Service) SetMonitorInterval(duration time.Duration) { + svc.monitorInterval = duration +} + +func (svc *Service) monitor() (err error) { + // start + tic := time.Now() + log.Debugf("[DataSourceService] start monitoring") + + // data source list + dsList, err := svc.modelSvc.GetDataSourceList(nil, nil) + if err != nil { + return err + } + + // waiting group + wg := sync.WaitGroup{} + wg.Add(len(dsList)) + + // iterate data source list + for _, ds := range dsList { + // async operation + go func(ds models.DataSource) { + // check status and save + _ = svc.checkStatus(&ds, true) + + // release + wg.Done() + }(ds) + } + + // wait + wg.Wait() + + // finish + toc := time.Now() + log.Debugf("[DataSourceService] finished monitoring. elapsed: %d ms", (toc.Sub(tic)).Milliseconds()) + + return nil +} + +func (svc *Service) checkStatus(ds *models.DataSource, save bool) (err error) { + // password + if ds.Password == "" { + pwd, err := svc.modelSvc.GetPasswordById(ds.Id) + if err == nil { + ds.Password, err = utils.DecryptAES(pwd.Password) + if err != nil { + return err + } + } else if err.Error() != mongo.ErrNoDocuments.Error() { + return trace.TraceError(err) + } + } + + // check status + if err := svc._checkStatus(ds); err != nil { + ds.Status = constants2.DataSourceStatusOffline + ds.Error = err.Error() + } else { + ds.Status = constants2.DataSourceStatusOnline + ds.Error = "" + } + + // save + if save { + return svc._save(ds) + } + + return nil +} + +func (svc *Service) _save(ds *models.DataSource) (err error) { + log.Debugf("[DataSourceService] saving data source: name=%s, type=%s, status=%s, error=%s", ds.Name, ds.Type, ds.Status, ds.Error) + return delegate.NewModelDelegate(ds).Save() +} + +func (svc *Service) _checkStatus(ds *models.DataSource) (err error) { + switch ds.Type { + case constants.DataSourceTypeMongo: + _, err := utils2.GetMongoClientWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + case constants.DataSourceTypeMysql: + s, err := utils2.GetMysqlSessionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + s.Close() + } + case constants.DataSourceTypePostgresql: + s, err := utils2.GetPostgresqlSessionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + s.Close() + } + case constants.DataSourceTypeMssql: + s, err := utils2.GetMssqlSessionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + s.Close() + } + case constants.DataSourceTypeSqlite: + s, err := utils2.GetSqliteSessionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + s.Close() + } + case constants.DataSourceTypeCockroachdb: + s, err := utils2.GetCockroachdbSessionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if s != nil { + s.Close() + } + case constants.DataSourceTypeElasticSearch: + _, err := utils2.GetElasticsearchClientWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + case constants.DataSourceTypeKafka: + c, err := utils2.GetKafkaConnectionWithTimeout(ds, svc.timeout) + if err != nil { + return err + } + if c != nil { + c.Close() + } + default: + log.Warnf("[DataSourceService] invalid data source type: %s", ds.Type) + } + return nil +} + +func NewDataSourceService(opts ...DataSourceServiceOption) (svc2 interfaces.DataSourceService, err error) { + // service + svc := &Service{ + monitorInterval: 15 * time.Second, + timeout: 10 * time.Second, + } + + // apply options + for _, opt := range opts { + opt(svc) + } + + // dependency injection + c := dig.New() + if err := c.Provide(service.NewService); err != nil { + return nil, trace.TraceError(err) + } + if err := c.Invoke(func(modelSvc service.ModelService) { + svc.modelSvc = modelSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // initialize + svc.Init() + + return svc, nil +} + +var _dsSvc interfaces.DataSourceService + +func GetDataSourceService() (svc interfaces.DataSourceService, err error) { + if _dsSvc != nil { + return _dsSvc, nil + } + svc, err = NewDataSourceService() + if err != nil { + return nil, err + } + _dsSvc = svc + return svc, nil +} diff --git a/ds/sql.go b/ds/sql.go new file mode 100644 index 0000000..cb2ebbf --- /dev/null +++ b/ds/sql.go @@ -0,0 +1,76 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab-core/entity" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + utils2 "github.com/crawlab-team/crawlab-core/utils" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/go-trace" + "github.com/upper/db/v4" + "time" +) + +type SqlService struct { + // dependencies + modelSvc service.ModelService + + // internals + ds *models.DataSource + dc *models.DataCollection + s db.Session + col db.Collection + t time.Time +} + +func (svc *SqlService) Insert(records ...interface{}) (err error) { + for _, d := range records { + var r entity.Result + switch d.(type) { + case entity.Result: + r = d.(entity.Result) + default: + continue + } + _r := r.Flatten() + if _, err = svc.col.Insert(_r); err != nil { + trace.PrintError(err) + continue + } + } + return nil +} + +func (svc *SqlService) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { + var docs []entity.Result + if err := svc.col.Find(utils2.GetSqlQuery(query)). + Offset(opts.Skip). + Limit(opts.Limit).All(&docs); err != nil { + return nil, trace.TraceError(err) + } + for i := range docs { + d := docs[i].ToJSON() + results = append(results, &d) + } + return results, nil +} + +func (svc *SqlService) Count(query generic.ListQuery) (n int, err error) { + nInt64, err := svc.col.Find(utils2.GetSqlQuery(query)).Count() + if err != nil { + return n, err + } + return int(nInt64), nil +} + +func (svc *SqlService) Index(fields []string) { + // TODO: implement me +} + +func (svc *SqlService) SetTime(t time.Time) { + svc.t = t +} + +func (svc *SqlService) GetTime() (t time.Time) { + return svc.t +} diff --git a/ds/sql_options.go b/ds/sql_options.go new file mode 100644 index 0000000..e7d2687 --- /dev/null +++ b/ds/sql_options.go @@ -0,0 +1,6 @@ +package ds + +type SqlOptions struct { + DefaultHost string + DefaultPort string +} diff --git a/ds/sqlite.go b/ds/sqlite.go new file mode 100644 index 0000000..349de42 --- /dev/null +++ b/ds/sqlite.go @@ -0,0 +1,52 @@ +package ds + +import ( + "github.com/crawlab-team/crawlab-core/interfaces" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-core/models/service" + utils2 "github.com/crawlab-team/crawlab-core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type SqliteService struct { + SqlService +} + +func NewDataSourceSqliteService(colId primitive.ObjectID, dsId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &SqliteService{} + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, trace.TraceError(err) + } + + // data source + if dsId.IsZero() { + svc.ds = &models.DataSource{} + } else { + svc.ds, err = svc.modelSvc.GetDataSourceById(dsId) + if err != nil { + return nil, trace.TraceError(err) + } + } + + // data collection + svc.dc, err = svc.modelSvc.GetDataCollectionById(colId) + if err != nil { + return nil, trace.TraceError(err) + } + + // session + svc.s, err = utils2.GetSqliteSession(svc.ds) + if err != nil { + return nil, trace.TraceError(err) + } + + // collection + svc.col = svc.s.Collection(svc.dc.Name) + + return svc, nil +} diff --git a/entity/es.go b/entity/es.go new file mode 100644 index 0000000..d3b5614 --- /dev/null +++ b/entity/es.go @@ -0,0 +1,54 @@ +package entity + +/* ElasticsearchResponseData JSON format +{ + "took" : 6, + "timed_out" : false, + "_shards" : { + "total" : 1, + "successful" : 1, + "skipped" : 0, + "failed" : 0 + }, + "hits" : { + "total" : { + "value" : 60, + "relation" : "eq" + }, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "test_table", + "_id" : "c39ad9a2-9a37-49fb-b7ea-f1b55913e0af", + "_score" : 1.0, + "_source" : { + "_tid" : "62524ac7f5f99e7ef594de64", + "author" : "James Baldwin", + "tags" : [ + "love" + ], + "text" : "“Love does not begin and end the way we seem to think it does. Love is a battle, love is a war; love is a growing up.”" + } + } + ] + } +} +*/ + +type ElasticsearchResponseData struct { + Took int64 `json:"took"` + Timeout bool `json:"timeout"` + Hits struct { + Total struct { + Value int64 `json:"value"` + Relation string `json:"relation"` + } `json:"total"` + MaxScore float64 `json:"max_score"` + Hits []struct { + Index string `json:"_index"` + Id string `json:"_id"` + Score float64 `json:"_score"` + Source interface{} `json:"_source"` + } `json:"hits"` + } `json:"hits"` +} diff --git a/errors/base.go b/errors/base.go index ebf36d8..b53f924 100644 --- a/errors/base.go +++ b/errors/base.go @@ -24,6 +24,7 @@ const ( ErrorPrefixProcess = "process" ErrorPrefixGit = "git" ErrorPrefixResult = "result" + ErrorPrefixDataSource = "data_source" ) type ErrorPrefix string diff --git a/errors/ds.go b/errors/ds.go new file mode 100644 index 0000000..f955016 --- /dev/null +++ b/errors/ds.go @@ -0,0 +1,15 @@ +package errors + +func NewDataSourceError(msg string) (err error) { + return NewError(ErrorPrefixDataSource, msg) +} + +var ( + ErrorDataSourceInvalidType = NewDataSourceError("invalid type") + ErrorDataSourceNotExists = NewDataSourceError("not exists") + ErrorDataSourceNotExistsInContext = NewDataSourceError("not exists in context") + ErrorDataSourceAlreadyExists = NewDataSourceError("already exists") + ErrorDataSourceMismatch = NewDataSourceError("mismatch") + ErrorDataSourceMissingRequiredFields = NewDataSourceError("missing required fields") + ErrorDataSourceUnauthorized = NewDataSourceError("unauthorized") +) diff --git a/go.mod b/go.mod index 8d09637..ad7b0f4 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/elastic/go-elasticsearch/v8 v8.7.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.5.1 github.com/gavv/httpexpect/v2 v2.2.0 @@ -36,19 +37,19 @@ require ( github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.0 github.com/satori/go.uuid v1.2.0 + github.com/segmentio/kafka-go v0.4.39 github.com/shirou/gopsutil v3.21.11+incompatible github.com/smartystreets/goconvey v1.6.4 github.com/spf13/cobra v1.3.0 github.com/spf13/viper v1.10.0 - github.com/stretchr/testify v1.7.2 + github.com/stretchr/testify v1.8.0 github.com/thoas/go-funk v0.9.1 github.com/tklauser/go-sysconf v0.3.9 // indirect + github.com/upper/db/v4 v4.6.0 github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.3.0 go.mongodb.org/mongo-driver v1.8.0 go.uber.org/dig v1.10.0 - golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect - golang.org/x/net v0.0.0-20220630215102-69896b714898 // indirect golang.org/x/sys v0.0.0-20220702020025-31831981b65f // indirect golang.org/x/tools v0.1.11 // indirect google.golang.org/grpc v1.42.0 diff --git a/go.sum b/go.sum index 1a1c26f..0862765 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -122,46 +124,47 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crawlab-team/crawlab-db v0.6.0-1/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= -github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221020034454-40c7a5c6cbe2 h1:dmC9D/BSHv6yh8khtQ7cDYFh8Yzr8NmhbAMBS36h9dA= -github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221020034454-40c7a5c6cbe2/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 h1:1CXWC3lYcVWcgPRc3PNKzZ3fcfX5WZ/V8xwzHEMUFHQ= github.com/crawlab-team/crawlab-db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= -github.com/crawlab-team/crawlab-fs v0.6.0-beta.20211101.1940/go.mod h1:dA1G6xeiClbTMkjRuoagGrcKfQ97jJZRAhZUSwrKdoI= -github.com/crawlab-team/crawlab-fs v0.6.0-beta.20211101.1940.0.20220912130912-a47819e0c7c9 h1:h1SCAinaakI8rtL2kh3ciTSfiHaC7RyD/HK1d3Mil7U= -github.com/crawlab-team/crawlab-fs v0.6.0-beta.20211101.1940.0.20220912130912-a47819e0c7c9/go.mod h1:y9YhLLR3GuPrDuPKe7ZuiHCITK9K2IcI8nlznF8YIEc= github.com/crawlab-team/crawlab-fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73 h1:xIgfVPa3ZJWC72Y57oHS41n4jRtGZPn1YDEYBgMj2EU= github.com/crawlab-team/crawlab-fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73/go.mod h1:y9YhLLR3GuPrDuPKe7ZuiHCITK9K2IcI8nlznF8YIEc= github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c h1:jX0iax3WHwomWGQVWrCTy8a4zYDsKKyuspP3+04XCcU= github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c/go.mod h1:Bq2Pm967EYWbjhP5Ghc4DV2LZgbOLMzLftJXDJYz/gs= -github.com/crawlab-team/crawlab-log v0.1.1-0.20221001084727-6c3b35136f2b h1:qStgbyujyIm/h1qeD+cu8knmaD+n+upsFvCsqzJW9EE= -github.com/crawlab-team/crawlab-log v0.1.1-0.20221001084727-6c3b35136f2b/go.mod h1:/m9GNWcbIlIVOka5SbNd8Ps84VEVbUoLOGTCRfLOkOc= -github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211113.2048.0.20220902052958-e3ff72bd9062 h1:tKj/nQPbvM6+vKzc/Ue3LRkdnVZ5JAhRq1+jDpX2LK8= -github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211113.2048.0.20220902052958-e3ff72bd9062/go.mod h1:6fIrMZn4BbLXNmvRBJTWfrhYydfNOUNnjn5PKfunPO0= github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211113.2048.0.20221024150201-467fe22630f8 h1:t6p0azaPDParcglEMi4oLfxjE0TXWQDF8qhZbqSlhaM= github.com/crawlab-team/crawlab-vcs v0.6.0-beta.20211113.2048.0.20221024150201-467fe22630f8/go.mod h1:6fIrMZn4BbLXNmvRBJTWfrhYydfNOUNnjn5PKfunPO0= github.com/crawlab-team/go-trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= github.com/crawlab-team/go-trace v0.1.1 h1:AecgAOld+ZrSVvujyhK3zoaOmViGKHSCT8/weJ7adB8= github.com/crawlab-team/go-trace v0.1.1/go.mod h1:4U+pWgLhRuD3pbXHonwcaHcW+y8AUqyOfKoZnvKwCug= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e h1:Gwg9kKNZUAI4bSssomlzXCN01Q3MapgwQOCeOxGX/NU= github.com/crawlab-team/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e/go.mod h1:FImmp7V0VcIdTRM68F3PQUqewzuShvUjYBhAHRjD1Aw= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= +github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/elastic-transport-go/v8 v8.2.0 h1:hkK5IIs/15mpSXzd5THWVlWTKJyMw6cbCWM3T/B2S5E= +github.com/elastic/elastic-transport-go/v8 v8.2.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= +github.com/elastic/go-elasticsearch/v8 v8.7.0 h1:ZvbT1YHppBC0QxGnMmaDUxoDa26clwhRaB3Gp5E3UcY= +github.com/elastic/go-elasticsearch/v8 v8.7.0/go.mod h1:lVb8SvJV8McVkdswpL8YR5QKIkhlWaoSq60YpHilOLI= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= @@ -212,8 +215,10 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df/go.mod h1:GJr+FCSXshIwgHBtLglIg9M2l2kQSi6QjVAngtzI08Y= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= @@ -227,15 +232,21 @@ github.com/go-playground/validator/v10 v10.11.0 h1:0W+xRM511GY47Yy3bZUbJVitCNg2B github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.9.8 h1:DxXB6MLd6yyel7CLph8EwNIonUtVZd3Ue5iRcL4DQCE= github.com/goccy/go-json v0.9.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -386,6 +397,57 @@ github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= +github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= +github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= +github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= +github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ= github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -417,10 +479,12 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -429,13 +493,19 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k= @@ -459,6 +529,7 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -468,6 +539,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= +github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -480,6 +553,7 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -495,8 +569,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nleeper/goment v1.4.4 h1:GlMTpxvhueljArSunzYjN9Ri4SOmpn0Vh2hg2z/IIl8= -github.com/nleeper/goment v1.4.4/go.mod h1:zDl5bAyDhqxwQKAvkSXMRLOdCowrdZz53ofRJc4VhTo= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -518,6 +590,8 @@ github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.0.2 h1:+jQXlF3scKIcSEKkdHzXhCTDLPFi5r1wnK6yPS+49Gw= github.com/pelletier/go-toml/v2 v2.0.2/go.mod h1:MovirKjgVRESsAvNZlAjtFwV867yGuwRkXbG66OzopI= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -545,6 +619,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 h1:ewTtJ72GFy2e0e8uyiDwMG3pKCS5mBh+hdSTYsPKEP8= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= @@ -556,6 +632,9 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -567,14 +646,24 @@ github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIE github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= +github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= +github.com/segmentio/kafka-go v0.4.39 h1:75smaomhvkYRwtuOwqLsdhgCG30B82NsbdkdDfFbvrw= +github.com/segmentio/kafka-go v0.4.39/go.mod h1:T0MLgygYvmqmBvC+s8aCcbVNfJN4znVne5j0Pzowp/Q= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -606,6 +695,8 @@ github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cma github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -613,8 +704,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= @@ -632,14 +724,14 @@ github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= -github.com/tkuchiki/go-timezone v0.2.0 h1:yyZVHtQRVZ+wvlte5HXvSpBkR0dPYnPEIgq9qqAqltk= -github.com/tkuchiki/go-timezone v0.2.0/go.mod h1:b1Ean9v2UXtxSq4TZF0i/TU9NuoWa9hOzOKoGCV2zqY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/upper/db/v4 v4.6.0 h1:0VmASnqrl/XN8Ehoq++HBgZ4zRD5j3GXygW8FhP0C5I= +github.com/upper/db/v4 v4.6.0/go.mod h1:2mnRcPf+RcCXmVcD+o04LYlyu3UuF7ubamJia7CkN6s= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw= @@ -658,6 +750,10 @@ github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= +github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= @@ -683,6 +779,7 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ztrue/tracerr v0.3.0 h1:lDi6EgEYhPYPnKcjsYzmWw4EkFEoA/gfe+I9Y5f+h6Y= github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -699,21 +796,31 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY= go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029175232-7e6ffbd03851/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -722,13 +829,17 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -786,6 +897,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -816,8 +928,8 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220630215102-69896b714898 h1:K7wO6V1IrczY9QOQ2WkVpw4JQSwCd52UsxVEirZUfiw= -golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220706163947-c90051bbdb60 h1:8NSylCMxLW4JvserAndSgFL7aPli6A68yf0bYFTcWCM= +golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -860,6 +972,7 @@ golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190225065934-cc5685c2db12/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -867,6 +980,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -920,6 +1034,7 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220702020025-31831981b65f h1:xdsejrW/0Wf2diT5CPp3XmKUNbr7Xvw8kYilQ+6qjRY= golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -949,6 +1064,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -956,9 +1072,12 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -968,6 +1087,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1001,6 +1121,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1168,9 +1290,12 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= +gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= @@ -1199,6 +1324,24 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +modernc.org/b v1.0.2/go.mod h1:fVGfCIzkZw5RsuF2A2WHbJmY7FiMIq30nP4s52uWsoY= +modernc.org/db v1.0.3/go.mod h1:L4ltUg8tu2pkSJk+fKaRrXs/3EdW79ZKYQ5PfVDT53U= +modernc.org/file v1.0.3/go.mod h1:CNj/pwOfCtCbqiHcXDUlHBB2vWrzdaDCWdcnjtS1+XY= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= +modernc.org/golex v1.0.1/go.mod h1:QCA53QtsT1NdGkaZZkF5ezFwk4IXh4BGNafAARTC254= +modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= +modernc.org/internal v1.0.2/go.mod h1:bycJAcev709ZU/47nil584PeBD+kbu8nv61ozeMso9E= +modernc.org/lex v1.0.0/go.mod h1:G6rxMTy3cH2iA0iXL/HRRv4Znu8MK4higxph/lE7ypk= +modernc.org/lexer v1.0.0/go.mod h1:F/Dld0YKYdZCLQ7bD0USbWL4YKCyTDRDHiDTOs0q0vk= +modernc.org/lldb v1.0.2/go.mod h1:ovbKqyzA9H/iPwHkAOH0qJbIQVT9rlijecenxDwVUi0= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/ql v1.4.0/go.mod h1:q4c29Bgdx+iAtxx47ODW5Xo2X0PDkjSCK9NdQl6KFxc= +modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/zappy v1.0.3/go.mod h1:w/Akq8ipfols/xZJdR5IYiQNOqC80qz2mVvsEwEbkiI= moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo= moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e/go.mod h1:nejbQVfXh96n9dSF6cH3Jsk/QI1Z2oEL7sSI2ifXFNA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/interfaces/data_source_service.go b/interfaces/data_source_service.go new file mode 100644 index 0000000..f5309df --- /dev/null +++ b/interfaces/data_source_service.go @@ -0,0 +1,14 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type DataSourceService interface { + ChangePassword(id primitive.ObjectID, password string) (err error) + Monitor() + CheckStatus(id primitive.ObjectID) (err error) + SetTimeout(duration time.Duration) + SetMonitorInterval(duration time.Duration) +} diff --git a/routes/router.go b/routes/router.go index 2d0d41c..f8e8d4f 100644 --- a/routes/router.go +++ b/routes/router.go @@ -176,6 +176,9 @@ func registerRoutesAuthGroup(svc *RouterService, groups *RouterGroups) { // filter svc.RegisterActionControllerToGroup(groups.AuthGroup, "/filters", controllers.FilterController) + + // data sources + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/data-sources", controllers.DataSourceController) } func registerRoutesFilterGroup(svc *RouterService, groups *RouterGroups) { diff --git a/utils/cockroachdb.go b/utils/cockroachdb.go new file mode 100644 index 0000000..324c3e3 --- /dev/null +++ b/utils/cockroachdb.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/mssql" + "time" +) + +func GetCockroachdbSession(ds *models.DataSource) (s db.Session, err error) { + return getCockroachdbSession(context.Background(), ds) +} + +func GetCockroachdbSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getCockroachdbSession(ctx, ds) +} + +func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultCockroachdbPort + } + + // connect settings + settings := mssql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mssql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/utils/es.go b/utils/es.go new file mode 100644 index 0000000..c3d290e --- /dev/null +++ b/utils/es.go @@ -0,0 +1,159 @@ +package utils + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/go-trace" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +func GetElasticsearchClient(ds *models.DataSource) (c *elasticsearch.Client, err error) { + return getElasticsearchClient(context.Background(), ds) +} + +func GetElasticsearchClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *elasticsearch.Client, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getElasticsearchClient(ctx, ds) +} + +func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elasticsearch.Client, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultElasticsearchPort + } + + // es hosts + addresses := []string{ + fmt.Sprintf("http://%s:%s", host, port), + } + + // retry backoff + rb := backoff.NewExponentialBackOff() + + // es client options + cfg := elasticsearch.Config{ + Addresses: addresses, + Username: ds.Username, + Password: ds.Password, + //CloudID: "", + //APIKey: "", + //ServiceToken: "", + //CertificateFingerprint: "", + //Header: nil, + //CACert: nil, + //RetryOnStatus: nil, + //DisableRetry: false, + //EnableRetryOnTimeout: false, + //MaxRetries: 0, + //CompressRequestBody: false, + //DiscoverNodesOnStart: false, + //DiscoverNodesInterval: 0, + //EnableMetrics: false, + //EnableDebugLogger: false, + //EnableCompatibilityMode: false, + //DisableMetaHeader: false, + //UseResponseCheckOnly: false, + RetryBackoff: func(i int) time.Duration { + if i == 1 { + rb.Reset() + } + return rb.NextBackOff() + }, + //Transport: nil, + //Logger: nil, + //Selector: nil, + //ConnectionPoolFunc: nil, + } + + // es client + done := make(chan struct{}) + go func() { + c, err = elasticsearch.NewClient(cfg) + if err != nil { + return + } + var res *esapi.Response + res, err = c.Info() + fmt.Println(res) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return c, err +} + +func GetElasticsearchQuery(query generic.ListQuery) (buf *bytes.Buffer) { + q := map[string]interface{}{} + if len(query) > 0 { + match := getElasticsearchQueryMatch(query) + q["query"] = map[string]interface{}{ + "match": match, + } + } + buf = &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(q); err != nil { + trace.PrintError(err) + } + return buf +} + +func GetElasticsearchQueryWithOptions(query generic.ListQuery, opts *generic.ListOptions) (buf *bytes.Buffer) { + q := map[string]interface{}{ + "size": opts.Limit, + "from": opts.Skip, + // TODO: sort + } + if len(query) > 0 { + match := getElasticsearchQueryMatch(query) + q["query"] = map[string]interface{}{ + "match": match, + } + } + buf = &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(q); err != nil { + trace.PrintError(err) + } + return buf +} + +func getElasticsearchQueryMatch(query generic.ListQuery) (match map[string]interface{}) { + match = map[string]interface{}{} + for _, c := range query { + switch c.Value.(type) { + case primitive.ObjectID: + c.Value = c.Value.(primitive.ObjectID).Hex() + } + switch c.Op { + case generic.OpEqual: + match[c.Key] = c.Value + default: + match[c.Key] = map[string]interface{}{ + c.Op: c.Value, + } + } + } + return match +} diff --git a/utils/kafka.go b/utils/kafka.go new file mode 100644 index 0000000..9eb9065 --- /dev/null +++ b/utils/kafka.go @@ -0,0 +1,41 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/segmentio/kafka-go" + "time" +) + +func GetKafkaConnection(ds *models.DataSource) (c *kafka.Conn, err error) { + return getKafkaConnection(context.Background(), ds) +} + +func GetKafkaConnectionWithTimeout(ds *models.DataSource, timeout time.Duration) (c *kafka.Conn, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getKafkaConnection(ctx, ds) +} + +func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Conn, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultKafkaPort + } + + // kafka connection address + network := "tcp" + address := fmt.Sprintf("%s:%s", host, port) + topic := ds.Database + partition := 0 // TODO: parameterize + + // kafka connection + return kafka.DialLeader(ctx, network, address, topic, partition) +} diff --git a/utils/mongo.go b/utils/mongo.go index d6052bc..db382a2 100644 --- a/utils/mongo.go +++ b/utils/mongo.go @@ -1,9 +1,14 @@ package utils import ( + "context" + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/models/models" "github.com/crawlab-team/crawlab-db/generic" "github.com/crawlab-team/crawlab-db/mongo" "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" ) func GetMongoQuery(query generic.ListQuery) (res bson.M) { @@ -38,3 +43,52 @@ func GetMongoOpts(opts *generic.ListOptions) (res *mongo.FindOptions) { Sort: sort, } } + +func GetMongoClient(ds *models.DataSource) (c *mongo2.Client, err error) { + return getMongoClient(context.Background(), ds) +} + +func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *mongo2.Client, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMongoClient(ctx, ds) +} + +func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Client, err error) { + // normalize settings + if ds.Host == "" { + ds.Host = constants.DefaultHost + } + if ds.Port == "" { + ds.Port = constants.DefaultMongoPort + } + + // options + var opts []mongo.ClientOption + opts = append(opts, mongo.WithContext(ctx)) + opts = append(opts, mongo.WithUri(ds.Url)) + opts = append(opts, mongo.WithHost(ds.Host)) + opts = append(opts, mongo.WithPort(ds.Port)) + opts = append(opts, mongo.WithDb(ds.Database)) + opts = append(opts, mongo.WithUsername(ds.Username)) + opts = append(opts, mongo.WithPassword(ds.Password)) + opts = append(opts, mongo.WithHosts(ds.Hosts)) + + // extra + if ds.Extra != nil { + // auth source + authSource, ok := ds.Extra["auth_source"] + if ok { + opts = append(opts, mongo.WithAuthSource(authSource)) + } + + // auth mechanism + authMechanism, ok := ds.Extra["auth_mechanism"] + if ok { + opts = append(opts, mongo.WithAuthMechanism(authMechanism)) + } + } + + // client + return mongo.GetMongoClient(opts...) +} diff --git a/utils/mssql.go b/utils/mssql.go new file mode 100644 index 0000000..8f51a78 --- /dev/null +++ b/utils/mssql.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/mssql" + "time" +) + +func GetMssqlSession(ds *models.DataSource) (s db.Session, err error) { + return getMssqlSession(context.Background(), ds) +} + +func GetMssqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMssqlSession(ctx, ds) +} + +func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultMssqlPort + } + + // connect settings + settings := mssql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mssql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/utils/mysql.go b/utils/mysql.go new file mode 100644 index 0000000..ffe8638 --- /dev/null +++ b/utils/mysql.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/mysql" + "time" +) + +func GetMysqlSession(ds *models.DataSource) (s db.Session, err error) { + return getMysqlSession(context.Background(), ds) +} + +func GetMysqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMysqlSession(ctx, ds) +} + +func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultMysqlPort + } + + // connect settings + settings := mysql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mysql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/utils/postgresql.go b/utils/postgresql.go new file mode 100644 index 0000000..f55f50f --- /dev/null +++ b/utils/postgresql.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab-core/constants" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/postgresql" + "time" +) + +func GetPostgresqlSession(ds *models.DataSource) (s db.Session, err error) { + return getPostgresqlSession(context.Background(), ds) +} + +func GetPostgresqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getPostgresqlSession(ctx, ds) +} + +func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultPostgresqlPort + } + + // connect settings + settings := postgresql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = postgresql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/utils/sql.go b/utils/sql.go new file mode 100644 index 0000000..f231d75 --- /dev/null +++ b/utils/sql.go @@ -0,0 +1,27 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab-db/generic" + "github.com/upper/db/v4" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func GetSqlQuery(query generic.ListQuery) (res db.Cond) { + res = db.Cond{} + for _, c := range query { + switch c.Value.(type) { + case primitive.ObjectID: + c.Value = c.Value.(primitive.ObjectID).Hex() + } + switch c.Op { + case generic.OpEqual: + res[c.Key] = c.Value + default: + res[c.Key] = db.Cond{ + c.Op: c.Value, + } + } + } + // TODO: sort + return res +} diff --git a/utils/sqlite.go b/utils/sqlite.go new file mode 100644 index 0000000..2258d0d --- /dev/null +++ b/utils/sqlite.go @@ -0,0 +1,45 @@ +package utils + +import ( + "context" + "github.com/crawlab-team/crawlab-core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/sqlite" + "time" +) + +func GetSqliteSession(ds *models.DataSource) (s db.Session, err error) { + return getSqliteSession(context.Background(), ds) +} + +func GetSqliteSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getSqliteSession(ctx, ds) +} + +func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // connect settings + settings := sqlite.ConnectionURL{ + Database: ds.Database, + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = sqlite.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +}