From c0b952e4c7b25214edce0d193350f2589068183c Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Tue, 21 Apr 2020 14:48:56 +0200 Subject: [PATCH 01/19] feat: add cdn uservice --- engine/api/services/const.go | 1 + engine/cdn/cdn.go | 104 +++++++++++++++++++++++++++++++++++ engine/cdn/cdn_router.go | 21 +++++++ engine/cdn/status_handler.go | 24 ++++++++ engine/cdn/types.go | 24 ++++++++ engine/cmd_config.go | 9 +++ engine/cmd_start.go | 36 ++++++++---- engine/config.go | 38 +++++++------ engine/types.go | 2 + 9 files changed, 230 insertions(+), 29 deletions(-) create mode 100644 engine/cdn/cdn.go create mode 100644 engine/cdn/cdn_router.go create mode 100644 engine/cdn/status_handler.go create mode 100644 engine/cdn/types.go diff --git a/engine/api/services/const.go b/engine/api/services/const.go index f9a5a4b503..b0b14971dd 100644 --- a/engine/api/services/const.go +++ b/engine/api/services/const.go @@ -8,6 +8,7 @@ const ( TypeVCS = "vcs" TypeAPI = "api" TypeUI = "ui" + TypeCDN = "cdn" TypeHatchery = "hatchery" TypeDBMigrate = "dbmigrate" ) diff --git a/engine/cdn/cdn.go b/engine/cdn/cdn.go new file mode 100644 index 0000000000..de229106ab --- /dev/null +++ b/engine/cdn/cdn.go @@ -0,0 +1,104 @@ +package cdn + +import ( + "context" + "fmt" + "net/http" + + "github.com/gorilla/mux" + + "github.com/ovh/cds/engine/api" + "github.com/ovh/cds/engine/api/services" + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/cdsclient" + "github.com/ovh/cds/sdk/log" +) + +// New returns a new service +func New() *Service { + s := new(Service) + s.Router = &api.Router{ + Mux: mux.NewRouter(), + } + return s +} + +func (s *Service) Init(config interface{}) (cdsclient.ServiceConfig, error) { + var cfg cdsclient.ServiceConfig + sConfig, ok := config.(Configuration) + if !ok { + return cfg, sdk.WithStack(fmt.Errorf("invalid CDN service configuration")) + } + + cfg.Host = sConfig.API.HTTP.URL + cfg.Token = sConfig.API.Token + cfg.InsecureSkipVerifyTLS = sConfig.API.HTTP.Insecure + cfg.RequestSecondsTimeout = sConfig.API.RequestTimeout + return cfg, nil +} + +// ApplyConfiguration apply an object of type CDN.Configuration after checking it +func (s *Service) ApplyConfiguration(config interface{}) error { + if err := s.CheckConfiguration(config); err != nil { + return err + } + var ok bool + s.Cfg, ok = config.(Configuration) + if !ok { + return fmt.Errorf("invalid configuration") + } + + s.ServiceName = s.Cfg.Name + s.ServiceType = services.TypeCDN + s.HTTPURL = s.Cfg.URL + s.MaxHeartbeatFailures = s.Cfg.API.MaxHeartbeatFailures + return nil +} + +// CheckConfiguration checks the validity of the configuration object +func (s *Service) CheckConfiguration(config interface{}) error { + sConfig, ok := config.(Configuration) + if !ok { + return fmt.Errorf("invalid configuration") + } + + if sConfig.URL == "" { + return fmt.Errorf("your CDS configuration seems to be empty. Please use environment variables, file or Consul to set your configuration") + } + if sConfig.Name == "" { + return fmt.Errorf("please enter a name in your CDN configuration") + } + + return nil +} + +// Serve will start the http api server +func (s *Service) Serve(c context.Context) error { + ctx, cancel := context.WithCancel(c) + defer cancel() + + //Init the http server + s.initRouter(ctx) + server := &http.Server{ + Addr: fmt.Sprintf("%s:%d", s.Cfg.HTTP.Addr, s.Cfg.HTTP.Port), + Handler: s.Router.Mux, + MaxHeaderBytes: 1 << 20, + } + + //Gracefully shutdown the http server + go func() { + select { + case <-ctx.Done(): + log.Info(ctx, "CDN> Shutdown HTTP Server") + _ = server.Shutdown(ctx) + } + }() + + //Start the http server + log.Info(ctx, "CDN> Starting HTTP Server on port %d", s.Cfg.HTTP.Port) + if err := server.ListenAndServe(); err != nil { + log.Fatalf("CDN> Cannot start cds-cdn: %s", err) + } + + return ctx.Err() +} diff --git a/engine/cdn/cdn_router.go b/engine/cdn/cdn_router.go new file mode 100644 index 0000000000..57ba3dfc64 --- /dev/null +++ b/engine/cdn/cdn_router.go @@ -0,0 +1,21 @@ +package cdn + +import ( + "context" + + "github.com/ovh/cds/engine/api" + "github.com/ovh/cds/engine/service" +) + +func (s *Service) initRouter(ctx context.Context) { + r := s.Router + r.Background = ctx + r.URL = s.Cfg.URL + r.SetHeaderFunc = api.DefaultHeaders + r.Middlewares = append(r.Middlewares, service.CheckRequestSignatureMiddleware(s.ParsedAPIPublicKey)) + + r.Handle("/mon/version", nil, r.GET(api.VersionHandler, api.Auth(false))) + r.Handle("/mon/status", nil, r.GET(s.statusHandler, api.Auth(false))) + r.Handle("/mon/metrics", nil, r.GET(service.GetPrometheustMetricsHandler(s), api.Auth(false))) + r.Handle("/mon/metrics/all", nil, r.GET(service.GetMetricsHandler, api.Auth(false))) +} diff --git a/engine/cdn/status_handler.go b/engine/cdn/status_handler.go new file mode 100644 index 0000000000..294a431058 --- /dev/null +++ b/engine/cdn/status_handler.go @@ -0,0 +1,24 @@ +package cdn + +import ( + "context" + "net/http" + + "github.com/ovh/cds/engine/service" + "github.com/ovh/cds/sdk" +) + +func (s *Service) statusHandler() service.Handler { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { + var status = http.StatusOK + return service.WriteJSON(w, s.Status(ctx), status) + } +} + +func (s *Service) Status(ctx context.Context) sdk.MonitoringStatus { + m := s.CommonMonitoring() + + status := sdk.MonitoringStatusOK + m.Lines = append(m.Lines, sdk.MonitoringStatusLine{Component: "CDN", Value: status, Status: status}) + return m +} diff --git a/engine/cdn/types.go b/engine/cdn/types.go new file mode 100644 index 0000000000..a13ee23967 --- /dev/null +++ b/engine/cdn/types.go @@ -0,0 +1,24 @@ +package cdn + +import ( + "github.com/ovh/cds/engine/api" + "github.com/ovh/cds/engine/service" +) + +// Service is the stuct representing a hooks µService +type Service struct { + service.Common + Cfg Configuration + Router *api.Router +} + +// Configuration is the hooks configuration structure +type Configuration struct { + Name string `toml:"name" default:"cds-cdn" comment:"Name of this CDS CDN Service\n Enter a name to enable this service" json:"name"` + HTTP struct { + Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"` + Port int `toml:"port" default:"8089" json:"port"` + } `toml:"http" comment:"######################\n CDS CDN HTTP Configuration \n######################" json:"http"` + URL string `default:"http://localhost:8087" json:"url"` + API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################" json:"api"` +} diff --git a/engine/cmd_config.go b/engine/cmd_config.go index f89a267a96..f1632251a6 100644 --- a/engine/cmd_config.go +++ b/engine/cmd_config.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/ovh/cds/engine/cdn" "io/ioutil" "os" "strconv" @@ -203,6 +204,14 @@ var configCheckCmd = &cobra.Command{ } } + if conf.CDN != nil && conf.CDN.API.HTTP.URL != "" { + fmt.Printf("checking cdn configuration...\n") + if err := cdn.New().CheckConfiguration(*conf.CDN); err != nil { + fmt.Printf("cdn Configuration: %v\n", err) + hasError = true + } + } + if !hasError { fmt.Println("Configuration file OK") } diff --git a/engine/cmd_start.go b/engine/cmd_start.go index f873bee7d2..0ebe11d3ab 100644 --- a/engine/cmd_start.go +++ b/engine/cmd_start.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/ovh/cds/engine/cdn" "os" "os/signal" "sort" @@ -79,9 +80,12 @@ This component operates CDS workflow repositories #### VCS This component operates CDS VCS connectivity +#### CDN +This component operates CDS CDN to handle storage + Start all of this with a single command: - $ engine start [api] [hatchery:local] [hatchery:marathon] [hatchery:openstack] [hatchery:swarm] [hatchery:vsphere] [elasticsearch] [hooks] [vcs] [repositories] [migrate] [ui] + $ engine start [api] [hatchery:local] [hatchery:marathon] [hatchery:openstack] [hatchery:swarm] [hatchery:vsphere] [elasticsearch] [hooks] [vcs] [repositories] [cdn] [migrate] [ui] All the services are using the same configuration file format. @@ -133,7 +137,7 @@ See $ engine config command for more details. for _, a := range args { fmt.Printf("Starting service %s\n", a) switch a { - case "api": + case services.TypeAPI: if conf.API == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } @@ -141,7 +145,7 @@ See $ engine config command for more details. names = append(names, conf.API.Name) types = append(types, services.TypeAPI) - case "ui": + case services.TypeUI: if conf.UI == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } @@ -157,7 +161,7 @@ See $ engine config command for more details. names = append(names, conf.DatabaseMigrate.Name) types = append(types, services.TypeDBMigrate) - case "hatchery:local": + case services.TypeHatchery + ":local": if conf.Hatchery.Local == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } @@ -165,7 +169,7 @@ See $ engine config command for more details. names = append(names, conf.Hatchery.Local.Name) types = append(types, services.TypeHatchery) - case "hatchery:kubernetes": + case services.TypeHatchery + ":kubernetes": if conf.Hatchery.Kubernetes == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } @@ -173,7 +177,7 @@ See $ engine config command for more details. names = append(names, conf.Hatchery.Kubernetes.Name) types = append(types, services.TypeHatchery) - case "hatchery:marathon": + case services.TypeHatchery + ":marathon": if conf.Hatchery.Marathon == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } @@ -181,7 +185,7 @@ See $ engine config command for more details. names = append(names, conf.Hatchery.Marathon.Name) types = append(types, services.TypeHatchery) - case "hatchery:openstack": + case services.TypeHatchery + ":openstack": if conf.Hatchery.Openstack == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } @@ -189,7 +193,7 @@ See $ engine config command for more details. names = append(names, conf.Hatchery.Openstack.Name) types = append(types, services.TypeAPI) - case "hatchery:swarm": + case services.TypeHatchery + ":swarm": if conf.Hatchery.Swarm == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } @@ -197,7 +201,7 @@ See $ engine config command for more details. names = append(names, conf.Hatchery.Swarm.Name) types = append(types, services.TypeHatchery) - case "hatchery:vsphere": + case services.TypeHatchery + ":vsphere": if conf.Hatchery.VSphere == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } @@ -205,7 +209,7 @@ See $ engine config command for more details. names = append(names, conf.Hatchery.VSphere.Name) types = append(types, services.TypeHatchery) - case "hooks": + case services.TypeHooks: if conf.Hooks == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } @@ -213,7 +217,15 @@ See $ engine config command for more details. names = append(names, conf.Hooks.Name) types = append(types, services.TypeHooks) - case "vcs": + case services.TypeCDN: + if conf.CDN == nil { + sdk.Exit("Unable to start: missing service %s configuration", a) + } + serviceConfs = append(serviceConfs, serviceConf{arg: a, service: cdn.New(), cfg: *conf.CDN}) + names = append(names, conf.Hooks.Name) + types = append(types, services.TypeCDN) + + case services.TypeVCS: if conf.VCS == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } @@ -221,7 +233,7 @@ See $ engine config command for more details. names = append(names, conf.VCS.Name) types = append(types, services.TypeVCS) - case "repositories": + case services.TypeRepositories: if conf.Repositories == nil { sdk.Exit("Unable to start: missing service %s configuration", a) } diff --git a/engine/config.go b/engine/config.go index 0f745413de..894ceddad6 100644 --- a/engine/config.go +++ b/engine/config.go @@ -3,6 +3,7 @@ package main import ( "bytes" "fmt" + "github.com/ovh/cds/engine/cdn" "io" "os" "sort" @@ -63,7 +64,7 @@ func configBootstrap(args []string) Configuration { } for _, a := range args { switch a { - case "api": + case services.TypeAPI: conf.API = &api.Configuration{} conf.API.Name = "cds-api-" + namesgenerator.GetRandomNameCDS(0) defaults.SetDefaults(conf.API) @@ -77,7 +78,7 @@ func configBootstrap(args []string) Configuration { HealthURL: "https://ovh.github.io", Type: "doc", }) - case "ui": + case services.TypeUI: conf.UI = &ui.Configuration{} conf.UI.Name = "cds-ui-" + namesgenerator.GetRandomNameCDS(0) defaults.SetDefaults(conf.UI) @@ -85,23 +86,23 @@ func configBootstrap(args []string) Configuration { conf.DatabaseMigrate = &migrateservice.Configuration{} defaults.SetDefaults(conf.DatabaseMigrate) conf.DatabaseMigrate.Name = "cds-migrate-" + namesgenerator.GetRandomNameCDS(0) - case "hatchery:local": + case services.TypeHatchery + ":local": conf.Hatchery.Local = &local.HatcheryConfiguration{} defaults.SetDefaults(conf.Hatchery.Local) conf.Hatchery.Local.Name = "cds-hatchery-local-" + namesgenerator.GetRandomNameCDS(0) - case "hatchery:kubernetes": + case services.TypeHatchery + ":kubernetes": conf.Hatchery.Kubernetes = &kubernetes.HatcheryConfiguration{} defaults.SetDefaults(conf.Hatchery.Kubernetes) conf.Hatchery.Kubernetes.Name = "cds-hatchery-kubernetes-" + namesgenerator.GetRandomNameCDS(0) - case "hatchery:marathon": + case services.TypeHatchery + ":marathon": conf.Hatchery.Marathon = &marathon.HatcheryConfiguration{} defaults.SetDefaults(conf.Hatchery.Marathon) conf.Hatchery.Marathon.Name = "cds-hatchery-marathon-" + namesgenerator.GetRandomNameCDS(0) - case "hatchery:openstack": + case services.TypeHatchery + ":openstack": conf.Hatchery.Openstack = &openstack.HatcheryConfiguration{} defaults.SetDefaults(conf.Hatchery.Openstack) conf.Hatchery.Openstack.Name = "cds-hatchery-openstack-" + namesgenerator.GetRandomNameCDS(0) - case "hatchery:swarm": + case services.TypeHatchery + ":swarm": conf.Hatchery.Swarm = &swarm.HatcheryConfiguration{} defaults.SetDefaults(conf.Hatchery.Swarm) conf.Hatchery.Swarm.DockerEngines = map[string]swarm.DockerEngineConfiguration{ @@ -110,15 +111,15 @@ func configBootstrap(args []string) Configuration { }, } conf.Hatchery.Swarm.Name = "cds-hatchery-swarm-" + namesgenerator.GetRandomNameCDS(0) - case "hatchery:vsphere": + case services.TypeHatchery + ":vsphere": conf.Hatchery.VSphere = &vsphere.HatcheryConfiguration{} defaults.SetDefaults(conf.Hatchery.VSphere) conf.Hatchery.VSphere.Name = "cds-hatchery-vsphere-" + namesgenerator.GetRandomNameCDS(0) - case "hooks": + case services.TypeHooks: conf.Hooks = &hooks.Configuration{} defaults.SetDefaults(conf.Hooks) conf.Hooks.Name = "cds-hooks-" + namesgenerator.GetRandomNameCDS(0) - case "vcs": + case services.TypeVCS: conf.VCS = &vcs.Configuration{} defaults.SetDefaults(conf.VCS) var github vcs.GithubServerConfiguration @@ -132,18 +133,21 @@ func configBootstrap(args []string) Configuration { var gerrit vcs.GerritServerConfiguration defaults.SetDefaults(&gerrit) conf.VCS.Servers = map[string]vcs.ServerConfiguration{ - "github": vcs.ServerConfiguration{URL: "https://github.com", Github: &github}, - "bitbucket": vcs.ServerConfiguration{URL: "https://mybitbucket.com", Bitbucket: &bitbucket}, - "bitbucketcloud": vcs.ServerConfiguration{BitbucketCloud: &bitbucketcloud}, - "gitlab": vcs.ServerConfiguration{URL: "https://gitlab.com", Gitlab: &gitlab}, - "gerrit": vcs.ServerConfiguration{URL: "http://localhost:8080", Gerrit: &gerrit}, + "github": {URL: "https://github.com", Github: &github}, + "bitbucket": {URL: "https://mybitbucket.com", Bitbucket: &bitbucket}, + "bitbucketcloud": {BitbucketCloud: &bitbucketcloud}, + "gitlab": {URL: "https://gitlab.com", Gitlab: &gitlab}, + "gerrit": {URL: "http://localhost:8080", Gerrit: &gerrit}, } conf.VCS.Name = "cds-vcs-" + namesgenerator.GetRandomNameCDS(0) - case "repositories": + case services.TypeRepositories: conf.Repositories = &repositories.Configuration{} defaults.SetDefaults(conf.Repositories) conf.Repositories.Name = "cds-repositories-" + namesgenerator.GetRandomNameCDS(0) - case "elasticsearch": + case services.TypeCDN: + conf.CDN = &cdn.Configuration{} + defaults.SetDefaults(conf.CDN) + case services.TypeElasticsearch: conf.ElasticSearch = &elasticsearch.Configuration{} defaults.SetDefaults(conf.ElasticSearch) default: diff --git a/engine/types.go b/engine/types.go index 732d0f8b5f..6ff22fbc4d 100644 --- a/engine/types.go +++ b/engine/types.go @@ -3,6 +3,7 @@ package main import ( "github.com/ovh/cds/engine/api" "github.com/ovh/cds/engine/api/observability" + "github.com/ovh/cds/engine/cdn" "github.com/ovh/cds/engine/elasticsearch" "github.com/ovh/cds/engine/hatchery/kubernetes" "github.com/ovh/cds/engine/hatchery/local" @@ -36,6 +37,7 @@ type Configuration struct { UI *ui.Configuration `toml:"ui" comment:"#####################\n UI Configuration \n####################" json:"ui"` Hatchery *HatcheryConfiguration `toml:"hatchery" json:"hatchery"` Hooks *hooks.Configuration `toml:"hooks" comment:"######################\n CDS Hooks Settings \n######################" json:"hooks"` + CDN *cdn.Configuration `toml:"cdn" comment:"######################\n CDS cdn Settings \n######################" json:"cdn"` VCS *vcs.Configuration `toml:"vcs" comment:"######################\n CDS VCS Settings \n######################" json:"vcs"` Repositories *repositories.Configuration `toml:"repositories" comment:"######################\n CDS Repositories Settings \n######################" json:"repositories"` ElasticSearch *elasticsearch.Configuration `toml:"elasticsearch" comment:"######################\n CDS ElasticSearch Settings \n This is use for CDS timeline and is optional\n######################" json:"elasticsearch"` From b1b399ee61f50d49f04c633aaab0d4dc5a230cb8 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Thu, 23 Apr 2020 08:22:22 +0200 Subject: [PATCH 02/19] wip --- contrib/grpcplugins/action/archive/go.mod | 2 +- contrib/grpcplugins/action/archive/go.sum | 3 +++ contrib/grpcplugins/action/clair/go.sum | 3 +++ contrib/grpcplugins/action/download/go.mod | 2 +- contrib/grpcplugins/action/download/go.sum | 3 +++ contrib/grpcplugins/action/group-tmpl/go.sum | 3 +++ .../grpcplugins/action/kafka-publish/go.mod | 2 +- .../grpcplugins/action/kafka-publish/go.sum | 3 +++ contrib/grpcplugins/action/marathon/go.mod | 2 +- contrib/grpcplugins/action/marathon/go.sum | 3 +++ .../action/npm-audit-parser/go.sum | 3 +++ contrib/grpcplugins/action/ssh-cmd/go.sum | 3 +++ contrib/grpcplugins/action/tmpl/go.sum | 3 +++ contrib/grpcplugins/action/venom/go.mod | 2 +- contrib/grpcplugins/action/venom/go.sum | 3 +++ engine/api/workflow_queue.go | 8 +++++++ engine/cdn/cdn.go | 2 ++ engine/cdn/cdn_log.go | 5 +++++ engine/worker/internal/take.go | 13 +++++++++++ engine/worker/internal/types.go | 7 ++++-- go.mod | 1 + go.sum | 2 ++ sdk/jws/jws.go | 22 +++++++++++++++++-- sdk/worker.go | 10 +++++---- 24 files changed, 97 insertions(+), 13 deletions(-) create mode 100644 engine/cdn/cdn_log.go diff --git a/contrib/grpcplugins/action/archive/go.mod b/contrib/grpcplugins/action/archive/go.mod index 47a4870e1e..1e6e83c11b 100644 --- a/contrib/grpcplugins/action/archive/go.mod +++ b/contrib/grpcplugins/action/archive/go.mod @@ -11,5 +11,5 @@ require ( github.com/golang/snappy v0.0.1 // indirect github.com/mholt/archiver v3.1.1+incompatible github.com/ovh/cds v0.0.0-00010101000000-000000000000 - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.5.1 ) diff --git a/contrib/grpcplugins/action/archive/go.sum b/contrib/grpcplugins/action/archive/go.sum index bbb1daea30..d66cd51d91 100644 --- a/contrib/grpcplugins/action/archive/go.sum +++ b/contrib/grpcplugins/action/archive/go.sum @@ -118,6 +118,7 @@ github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMW github.com/fsamin/go-repo v0.1.3/go.mod h1:FjkhUwT3u2bCq7HcT+3hfMxkQhiKlfdvw91+fDHNRxo= github.com/fsamin/go-repo v0.1.4/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= github.com/fsamin/go-repo v0.1.5/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= +github.com/fsamin/go-repo v0.1.7/go.mod h1:JRLbo6sPXvAwwGs5RgifFk1pXiefeGf0hyHifEg1Vw4= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -473,6 +474,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tebeka/go2xunit v1.4.10/go.mod h1:wmc9jKT7KlU4QLU6DNTaIXNnYNOjKKNlp6mjOS0UrqY= github.com/tevino/abool v0.0.0-20170917061928-9b9efcf221b5/go.mod h1:f1SCnEOt6sc3fOJfPQDRDzHOtSXuTtnz0ImG9kPRDV0= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -636,6 +638,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/clair/go.sum b/contrib/grpcplugins/action/clair/go.sum index 8bc8e9ee04..2a3dbcc7b2 100644 --- a/contrib/grpcplugins/action/clair/go.sum +++ b/contrib/grpcplugins/action/clair/go.sum @@ -121,6 +121,7 @@ github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMW github.com/fsamin/go-repo v0.1.3/go.mod h1:FjkhUwT3u2bCq7HcT+3hfMxkQhiKlfdvw91+fDHNRxo= github.com/fsamin/go-repo v0.1.4/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= github.com/fsamin/go-repo v0.1.5/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= +github.com/fsamin/go-repo v0.1.7/go.mod h1:JRLbo6sPXvAwwGs5RgifFk1pXiefeGf0hyHifEg1Vw4= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -476,6 +477,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tebeka/go2xunit v1.4.10/go.mod h1:wmc9jKT7KlU4QLU6DNTaIXNnYNOjKKNlp6mjOS0UrqY= @@ -637,6 +639,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/download/go.mod b/contrib/grpcplugins/action/download/go.mod index 9d5dbda5d4..3852cda00d 100644 --- a/contrib/grpcplugins/action/download/go.mod +++ b/contrib/grpcplugins/action/download/go.mod @@ -7,5 +7,5 @@ go 1.13 require ( github.com/golang/protobuf v1.3.2 github.com/ovh/cds v0.0.0-00010101000000-000000000000 - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.5.1 ) diff --git a/contrib/grpcplugins/action/download/go.sum b/contrib/grpcplugins/action/download/go.sum index 384ce1224a..894f1041fe 100644 --- a/contrib/grpcplugins/action/download/go.sum +++ b/contrib/grpcplugins/action/download/go.sum @@ -112,6 +112,7 @@ github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMW github.com/fsamin/go-repo v0.1.3/go.mod h1:FjkhUwT3u2bCq7HcT+3hfMxkQhiKlfdvw91+fDHNRxo= github.com/fsamin/go-repo v0.1.4/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= github.com/fsamin/go-repo v0.1.5/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= +github.com/fsamin/go-repo v0.1.7/go.mod h1:JRLbo6sPXvAwwGs5RgifFk1pXiefeGf0hyHifEg1Vw4= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -458,6 +459,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tebeka/go2xunit v1.4.10/go.mod h1:wmc9jKT7KlU4QLU6DNTaIXNnYNOjKKNlp6mjOS0UrqY= github.com/tevino/abool v0.0.0-20170917061928-9b9efcf221b5/go.mod h1:f1SCnEOt6sc3fOJfPQDRDzHOtSXuTtnz0ImG9kPRDV0= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -618,6 +620,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/group-tmpl/go.sum b/contrib/grpcplugins/action/group-tmpl/go.sum index 384ce1224a..894f1041fe 100644 --- a/contrib/grpcplugins/action/group-tmpl/go.sum +++ b/contrib/grpcplugins/action/group-tmpl/go.sum @@ -112,6 +112,7 @@ github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMW github.com/fsamin/go-repo v0.1.3/go.mod h1:FjkhUwT3u2bCq7HcT+3hfMxkQhiKlfdvw91+fDHNRxo= github.com/fsamin/go-repo v0.1.4/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= github.com/fsamin/go-repo v0.1.5/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= +github.com/fsamin/go-repo v0.1.7/go.mod h1:JRLbo6sPXvAwwGs5RgifFk1pXiefeGf0hyHifEg1Vw4= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -458,6 +459,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tebeka/go2xunit v1.4.10/go.mod h1:wmc9jKT7KlU4QLU6DNTaIXNnYNOjKKNlp6mjOS0UrqY= github.com/tevino/abool v0.0.0-20170917061928-9b9efcf221b5/go.mod h1:f1SCnEOt6sc3fOJfPQDRDzHOtSXuTtnz0ImG9kPRDV0= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -618,6 +620,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/kafka-publish/go.mod b/contrib/grpcplugins/action/kafka-publish/go.mod index 78ffc9ef4c..30d4ba3848 100644 --- a/contrib/grpcplugins/action/kafka-publish/go.mod +++ b/contrib/grpcplugins/action/kafka-publish/go.mod @@ -16,6 +16,6 @@ require ( github.com/klauspost/cpuid v1.2.1 // indirect github.com/ovh/cds v0.0.0-00010101000000-000000000000 github.com/phayes/permbits v0.0.0-20190612203442-39d7c581d2ee - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.5.1 gopkg.in/urfave/cli.v1 v1.20.0 ) diff --git a/contrib/grpcplugins/action/kafka-publish/go.sum b/contrib/grpcplugins/action/kafka-publish/go.sum index 97c8add215..60687b9e02 100644 --- a/contrib/grpcplugins/action/kafka-publish/go.sum +++ b/contrib/grpcplugins/action/kafka-publish/go.sum @@ -163,6 +163,7 @@ github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMW github.com/fsamin/go-repo v0.1.3/go.mod h1:FjkhUwT3u2bCq7HcT+3hfMxkQhiKlfdvw91+fDHNRxo= github.com/fsamin/go-repo v0.1.4/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= github.com/fsamin/go-repo v0.1.5/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= +github.com/fsamin/go-repo v0.1.7/go.mod h1:JRLbo6sPXvAwwGs5RgifFk1pXiefeGf0hyHifEg1Vw4= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be h1:UhjSvwE1gxUYfekK9BXZ/LL55we9Avg+2Pt0PIlMYCk= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -607,6 +608,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tebeka/go2xunit v1.4.10/go.mod h1:wmc9jKT7KlU4QLU6DNTaIXNnYNOjKKNlp6mjOS0UrqY= github.com/tevino/abool v0.0.0-20170917061928-9b9efcf221b5/go.mod h1:f1SCnEOt6sc3fOJfPQDRDzHOtSXuTtnz0ImG9kPRDV0= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= @@ -788,6 +790,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/marathon/go.mod b/contrib/grpcplugins/action/marathon/go.mod index dda84cb55a..1d1f3f9ef5 100644 --- a/contrib/grpcplugins/action/marathon/go.mod +++ b/contrib/grpcplugins/action/marathon/go.mod @@ -13,6 +13,6 @@ require ( github.com/golang/protobuf v1.3.2 github.com/ovh/cds v0.0.0-00010101000000-000000000000 github.com/ovh/cds/sdk/interpolate v0.0.0-20191126072910-b8d81d038865 - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.5.1 github.com/xeipuuv/gojsonschema v1.2.0 ) diff --git a/contrib/grpcplugins/action/marathon/go.sum b/contrib/grpcplugins/action/marathon/go.sum index 69f3331aed..65625bbedf 100644 --- a/contrib/grpcplugins/action/marathon/go.sum +++ b/contrib/grpcplugins/action/marathon/go.sum @@ -156,6 +156,7 @@ github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMW github.com/fsamin/go-repo v0.1.3/go.mod h1:FjkhUwT3u2bCq7HcT+3hfMxkQhiKlfdvw91+fDHNRxo= github.com/fsamin/go-repo v0.1.4/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= github.com/fsamin/go-repo v0.1.5/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= +github.com/fsamin/go-repo v0.1.7/go.mod h1:JRLbo6sPXvAwwGs5RgifFk1pXiefeGf0hyHifEg1Vw4= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be h1:UhjSvwE1gxUYfekK9BXZ/LL55we9Avg+2Pt0PIlMYCk= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -587,6 +588,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tebeka/go2xunit v1.4.10/go.mod h1:wmc9jKT7KlU4QLU6DNTaIXNnYNOjKKNlp6mjOS0UrqY= github.com/tevino/abool v0.0.0-20170917061928-9b9efcf221b5/go.mod h1:f1SCnEOt6sc3fOJfPQDRDzHOtSXuTtnz0ImG9kPRDV0= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= @@ -770,6 +772,7 @@ google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1 h1:mzQIVyOPSXJaQWi1m6AFCjrCEPIwQBSOn48Ri8ZpzAg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/npm-audit-parser/go.sum b/contrib/grpcplugins/action/npm-audit-parser/go.sum index 384ce1224a..894f1041fe 100644 --- a/contrib/grpcplugins/action/npm-audit-parser/go.sum +++ b/contrib/grpcplugins/action/npm-audit-parser/go.sum @@ -112,6 +112,7 @@ github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMW github.com/fsamin/go-repo v0.1.3/go.mod h1:FjkhUwT3u2bCq7HcT+3hfMxkQhiKlfdvw91+fDHNRxo= github.com/fsamin/go-repo v0.1.4/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= github.com/fsamin/go-repo v0.1.5/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= +github.com/fsamin/go-repo v0.1.7/go.mod h1:JRLbo6sPXvAwwGs5RgifFk1pXiefeGf0hyHifEg1Vw4= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -458,6 +459,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tebeka/go2xunit v1.4.10/go.mod h1:wmc9jKT7KlU4QLU6DNTaIXNnYNOjKKNlp6mjOS0UrqY= github.com/tevino/abool v0.0.0-20170917061928-9b9efcf221b5/go.mod h1:f1SCnEOt6sc3fOJfPQDRDzHOtSXuTtnz0ImG9kPRDV0= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -618,6 +620,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/ssh-cmd/go.sum b/contrib/grpcplugins/action/ssh-cmd/go.sum index d7306627b2..f0f12a11c2 100644 --- a/contrib/grpcplugins/action/ssh-cmd/go.sum +++ b/contrib/grpcplugins/action/ssh-cmd/go.sum @@ -112,6 +112,7 @@ github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMW github.com/fsamin/go-repo v0.1.3/go.mod h1:FjkhUwT3u2bCq7HcT+3hfMxkQhiKlfdvw91+fDHNRxo= github.com/fsamin/go-repo v0.1.4/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= github.com/fsamin/go-repo v0.1.5/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= +github.com/fsamin/go-repo v0.1.7/go.mod h1:JRLbo6sPXvAwwGs5RgifFk1pXiefeGf0hyHifEg1Vw4= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -458,6 +459,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tebeka/go2xunit v1.4.10/go.mod h1:wmc9jKT7KlU4QLU6DNTaIXNnYNOjKKNlp6mjOS0UrqY= github.com/tevino/abool v0.0.0-20170917061928-9b9efcf221b5/go.mod h1:f1SCnEOt6sc3fOJfPQDRDzHOtSXuTtnz0ImG9kPRDV0= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -619,6 +621,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/tmpl/go.sum b/contrib/grpcplugins/action/tmpl/go.sum index 384ce1224a..894f1041fe 100644 --- a/contrib/grpcplugins/action/tmpl/go.sum +++ b/contrib/grpcplugins/action/tmpl/go.sum @@ -112,6 +112,7 @@ github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMW github.com/fsamin/go-repo v0.1.3/go.mod h1:FjkhUwT3u2bCq7HcT+3hfMxkQhiKlfdvw91+fDHNRxo= github.com/fsamin/go-repo v0.1.4/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= github.com/fsamin/go-repo v0.1.5/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= +github.com/fsamin/go-repo v0.1.7/go.mod h1:JRLbo6sPXvAwwGs5RgifFk1pXiefeGf0hyHifEg1Vw4= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -458,6 +459,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tebeka/go2xunit v1.4.10/go.mod h1:wmc9jKT7KlU4QLU6DNTaIXNnYNOjKKNlp6mjOS0UrqY= github.com/tevino/abool v0.0.0-20170917061928-9b9efcf221b5/go.mod h1:f1SCnEOt6sc3fOJfPQDRDzHOtSXuTtnz0ImG9kPRDV0= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -618,6 +620,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/venom/go.mod b/contrib/grpcplugins/action/venom/go.mod index 1dc121a667..5f544c9331 100644 --- a/contrib/grpcplugins/action/venom/go.mod +++ b/contrib/grpcplugins/action/venom/go.mod @@ -12,7 +12,7 @@ require ( github.com/ovh/go-ovh v0.0.0-20181109152953-ba5adb4cf014 // indirect github.com/ovh/venom v0.26.0 github.com/sclevine/agouti v3.0.0+incompatible // indirect - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.5.1 github.com/yesnault/go-imap v0.0.0-20160710142244-eb9bbb66bd7b // indirect gopkg.in/ini.v1 v1.51.0 // indirect gopkg.in/testfixtures.v2 v2.6.0 // indirect diff --git a/contrib/grpcplugins/action/venom/go.sum b/contrib/grpcplugins/action/venom/go.sum index cc911ae5a2..28e8bcb749 100644 --- a/contrib/grpcplugins/action/venom/go.sum +++ b/contrib/grpcplugins/action/venom/go.sum @@ -120,6 +120,7 @@ github.com/fsamin/go-dump v1.0.9/go.mod h1:ZgKd2aOXAFFbbFuUgvQhu7mwTlI3d3qnTICMW github.com/fsamin/go-repo v0.1.3/go.mod h1:FjkhUwT3u2bCq7HcT+3hfMxkQhiKlfdvw91+fDHNRxo= github.com/fsamin/go-repo v0.1.4/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= github.com/fsamin/go-repo v0.1.5/go.mod h1:V7Te54EYyamyQIp6UKfONBbXBlPF4u3C1bvcH7OtdLM= +github.com/fsamin/go-repo v0.1.7/go.mod h1:JRLbo6sPXvAwwGs5RgifFk1pXiefeGf0hyHifEg1Vw4= github.com/fsamin/go-shredder v0.0.0-20180118184739-b2488aedb5be/go.mod h1:kuiNcf1lKxl4isIY6bHxbBatpLD43c2RKWIV/AGlhXY= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -496,6 +497,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tebeka/go2xunit v1.4.10/go.mod h1:wmc9jKT7KlU4QLU6DNTaIXNnYNOjKKNlp6mjOS0UrqY= github.com/tevino/abool v0.0.0-20170917061928-9b9efcf221b5/go.mod h1:f1SCnEOt6sc3fOJfPQDRDzHOtSXuTtnz0ImG9kPRDV0= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -659,6 +661,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/engine/api/workflow_queue.go b/engine/api/workflow_queue.go index 9ea2de0103..53930550dc 100644 --- a/engine/api/workflow_queue.go +++ b/engine/api/workflow_queue.go @@ -26,6 +26,7 @@ import ( "github.com/ovh/cds/engine/api/workflow" "github.com/ovh/cds/engine/service" "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/jws" "github.com/ovh/cds/sdk/log" ) @@ -180,6 +181,13 @@ func takeJob(ctx context.Context, dbFunc func() *gorp.DbMap, store cache.Store, wnjri.Secrets = append(wnjri.Secrets, secretsKeys...) wnjri.NodeJobRun.Parameters = append(wnjri.NodeJobRun.Parameters, params...) + wnjri.SigningKey, err = jws.NewRandomSymmetricKey(32) + // get cdn url + wnjri.GelfServiceAddr = "http://localhost:8089/logs" + + if err != nil { + return nil, err + } if err := tx.Commit(); err != nil { return nil, sdk.WithStack(err) } diff --git a/engine/cdn/cdn.go b/engine/cdn/cdn.go index de229106ab..311bb7d3b9 100644 --- a/engine/cdn/cdn.go +++ b/engine/cdn/cdn.go @@ -85,6 +85,8 @@ func (s *Service) Serve(c context.Context) error { MaxHeaderBytes: 1 << 20, } + s.initGelfServer() + //Gracefully shutdown the http server go func() { select { diff --git a/engine/cdn/cdn_log.go b/engine/cdn/cdn_log.go new file mode 100644 index 0000000000..4e6a466a5a --- /dev/null +++ b/engine/cdn/cdn_log.go @@ -0,0 +1,5 @@ +package cdn + +func (s *Service) initGelfServer() { + +} diff --git a/engine/worker/internal/take.go b/engine/worker/internal/take.go index 173f4238d3..dd239706ec 100644 --- a/engine/worker/internal/take.go +++ b/engine/worker/internal/take.go @@ -5,6 +5,9 @@ import ( "strings" "time" + "github.com/sirupsen/logrus" + "gopkg.in/Graylog2/go-gelf.v2/gelf" + "github.com/ovh/cds/engine/worker/pkg/workerruntime" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" @@ -30,6 +33,16 @@ func (w *CurrentWorker) Take(ctx context.Context, job sdk.WorkflowNodeJobRun) er w.currentJob.secrets = info.Secrets // Reset build variables w.currentJob.newVariables = nil + w.currentJob.signingKey = info.SigningKey + + log.Info(ctx, "Setup step logger") + w.logger.stepLogger = logrus.New() + gelfWriter, err := gelf.NewTCPWriter(info.GelfServiceAddr) + if err != nil { + return sdk.WithStack(err) + } + w.logger.stepLogger.SetOutput(gelfWriter) + log.Info(ctx, "Setup step logger done") start := time.Now() diff --git a/engine/worker/internal/types.go b/engine/worker/internal/types.go index e62dc1d07e..65e6542bcb 100644 --- a/engine/worker/internal/types.go +++ b/engine/worker/internal/types.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/sirupsen/logrus" "os" "strings" "time" @@ -27,8 +28,9 @@ type CurrentWorker struct { basedir afero.Fs manualExit bool logger struct { - logChan chan sdk.Log - llist *list.List + logChan chan sdk.Log + llist *list.List + stepLogger *logrus.Logger } httpPort int32 register struct { @@ -42,6 +44,7 @@ type CurrentWorker struct { params []sdk.Parameter secrets []sdk.Variable context context.Context + signingKey string } status struct { Name string `json:"name"` diff --git a/go.mod b/go.mod index 6ce8c1120c..d762ea9484 100644 --- a/go.mod +++ b/go.mod @@ -193,6 +193,7 @@ require ( google.golang.org/genproto v0.0.0-20190817000702-55e96fffbd48 // indirect google.golang.org/grpc v1.23.0 gopkg.in/AlecAivazis/survey.v1 v1.7.1 + gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/bsm/sarama-cluster.v2 v2.1.4 gopkg.in/go-playground/assert.v1 v1.2.1 // indirect diff --git a/go.sum b/go.sum index 7ebd74366f..6d50f6a612 100644 --- a/go.sum +++ b/go.sum @@ -702,6 +702,8 @@ google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1 h1:mzQIVyOPSXJaQWi1m6AFCjrCEPIwQBSOn48Ri8ZpzAg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0 h1:Xg23ydYYJLmb9AK3XdcEpplHZd1MpN3X2ZeeMoBClmY= +gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/sdk/jws/jws.go b/sdk/jws/jws.go index cb9448ee98..6d33ff3fd2 100644 --- a/sdk/jws/jws.go +++ b/sdk/jws/jws.go @@ -8,12 +8,25 @@ import ( "encoding/json" "encoding/pem" "errors" + "fmt" "gopkg.in/square/go-jose.v2" "github.com/ovh/cds/sdk" ) +func NewRandomSymmetricKey(size int) (string, error) { + if size <= 0 || size%8 != 0 { + return "", sdk.WithStack(fmt.Errorf("invalid key size")) + } + + k := make([]byte, size) + if _, err := rand.Read(k); err != nil { + return "", sdk.WithStack(err) + } + return string(k), nil +} + // NewRandomRSAKey generates a public/private key pair func NewRandomRSAKey() (*rsa.PrivateKey, error) { // Generate a public/private key pair to use for this example. @@ -63,6 +76,11 @@ func NewSigner(privateKey *rsa.PrivateKey) (jose.Signer, error) { return jose.NewSigner(jose.SigningKey{Algorithm: jose.PS512, Key: privateKey}, nil) } +// NewSigner instantiate a signer using HMAC using SHA-512 with the given private key. +func NewHMacSigner(secret string) (jose.Signer, error) { + return jose.NewSigner(jose.SigningKey{Algorithm: jose.HS512, Key: secret}, nil) +} + // Sign a json marshalled content and returns a protected JWS object using the full serialization format. func Sign(signer jose.Signer, content interface{}) (string, error) { btes, err := json.Marshal(content) @@ -82,12 +100,12 @@ func Sign(signer jose.Signer, content interface{}) (string, error) { // Verify parses the serialized, protected JWS object, than verifying the signature on the payload // and unmarshal the payload into i -func Verify(publicKey *rsa.PublicKey, s string, i interface{}) error { +func Verify(key interface{}, s string, i interface{}) error { object, err := jose.ParseSigned(s) if err != nil { return sdk.WithStack(err) } - output, err := object.Verify(publicKey) + output, err := object.Verify(key) if err != nil { return sdk.WithStack(err) } diff --git a/sdk/worker.go b/sdk/worker.go index 29b9fcf83d..8db647c6cb 100644 --- a/sdk/worker.go +++ b/sdk/worker.go @@ -75,8 +75,10 @@ func TemplateEnvs(args WorkerArgs, envs map[string]string) (map[string]string, e // WorkflowNodeJobRunData is returned to worker in answer to postTakeWorkflowJobHandler type WorkflowNodeJobRunData struct { - NodeJobRun WorkflowNodeJobRun - Secrets []Variable - Number int64 - SubNumber int64 + NodeJobRun WorkflowNodeJobRun + Secrets []Variable + Number int64 + SubNumber int64 + SigningKey string + GelfServiceAddr string } From 46a3f87b2e1c72e3565c028972391c577fdfa89f Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Fri, 24 Apr 2020 16:39:23 +0200 Subject: [PATCH 03/19] wip --- engine/api/api.go | 9 ++ engine/api/database/gorpmapping/encryption.go | 3 + engine/api/worker.go | 4 +- engine/api/worker/dao.go | 64 ++++++-- engine/api/worker/gorp_model.go | 22 +++ engine/api/worker/heartbeat.go | 2 +- engine/api/worker/registration.go | 2 +- engine/api/worker/worker_test.go | 4 +- engine/api/workflow_queue.go | 23 +-- engine/api/workflow_queue_test.go | 4 +- engine/cdn/cdn.go | 20 ++- engine/cdn/cdn_log.go | 139 +++++++++++++++++- engine/cdn/cdn_router.go | 21 ++- engine/cdn/types.go | 17 ++- engine/sql/200_worker.sql | 10 ++ engine/worker/internal/take.go | 32 ++-- engine/worker/internal/types.go | 46 ++++-- go.mod | 2 +- sdk/jws/jws.go | 8 +- sdk/log/log.go | 3 + sdk/worker.go | 7 + 21 files changed, 359 insertions(+), 83 deletions(-) create mode 100644 engine/api/worker/gorp_model.go create mode 100644 engine/sql/200_worker.sql diff --git a/engine/api/api.go b/engine/api/api.go index 3d1cf2a527..ae155d5cce 100644 --- a/engine/api/api.go +++ b/engine/api/api.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/ovh/cds/engine/cdn" "io" "io/ioutil" "net/http" @@ -192,6 +193,7 @@ type Configuration struct { StepMaxSize int64 `toml:"stepMaxSize" default:"15728640" comment:"Max step logs size in bytes (default: 15MB)" json:"stepMaxSize"` ServiceMaxSize int64 `toml:"serviceMaxSize" default:"15728640" comment:"Max service logs size in bytes (default: 15MB)" json:"serviceMaxSize"` } `toml:"log" json:"log" comment:"###########################\n Log settings.\n##########################"` + CDN cdn.Configuration `toml:"cdn" json:"cdn" comment:"###########################\n CDN settings.\n##########################"` } // ServiceConfiguration is the configuration of external service @@ -864,6 +866,13 @@ func (a *API) Serve(ctx context.Context) error { log.Error(ctx, "api> heap dump uploaded to %s", s) }() + cdsService := &cdn.Service{ + Cfg: a.Config.CDN, + Db: a.mustDB(), + Cache: a.Cache, + } + cdsService.RunTcpLogServer(ctx) + log.Info(ctx, "Starting CDS API HTTP Server on %s:%d", a.Config.HTTP.Addr, a.Config.HTTP.Port) if err := s.ListenAndServe(); err != nil { return fmt.Errorf("Cannot start HTTP server: %v", err) diff --git a/engine/api/database/gorpmapping/encryption.go b/engine/api/database/gorpmapping/encryption.go index 057b26d401..dc206b0cbf 100644 --- a/engine/api/database/gorpmapping/encryption.go +++ b/engine/api/database/gorpmapping/encryption.go @@ -1,8 +1,10 @@ package gorpmapping import ( + "context" "encoding/json" "fmt" + "github.com/ovh/cds/sdk/log" "reflect" "strconv" "strings" @@ -110,6 +112,7 @@ func updateEncryptedData(db gorp.SqlExecutor, i interface{}) error { } query := fmt.Sprintf("UPDATE %s SET %s WHERE %s = %v", table, strings.Join(updateSlice, ","), key, id) + log.Warning(context.TODO(), ">>>>%s", query) res, err := db.Exec(query, encryptedContentArgs...) if err != nil { return sdk.WithStack(err) diff --git a/engine/api/worker.go b/engine/api/worker.go index 2c47cea6ad..61533a77b6 100644 --- a/engine/api/worker.go +++ b/engine/api/worker.go @@ -213,7 +213,7 @@ func (api *API) workerWaitingHandler() service.Handler { return nil } - if err := worker.SetStatus(api.mustDB(), wk.ID, sdk.StatusWaiting); err != nil { + if err := worker.SetStatus(ctx, api.mustDB(), wk.ID, sdk.StatusWaiting); err != nil { return sdk.WrapError(err, "cannot update worker %s", wk.ID) } return nil @@ -254,7 +254,7 @@ func DisableWorker(ctx context.Context, db *gorp.DbMap, id string) error { log.Info(ctx, "DisableWorker> Worker %s crashed while building %d !", name, jobID.Int64) } - if err := worker.SetStatus(tx, id, sdk.StatusDisabled); err != nil { + if err := worker.SetStatus(ctx, tx, id, sdk.StatusDisabled); err != nil { cause := sdk.Cause(err) if cause == worker.ErrNoWorker || cause == sql.ErrNoRows { return sdk.WrapError(sdk.ErrWrongRequest, "DisableWorker> worker %s does not exists", id) diff --git a/engine/api/worker/dao.go b/engine/api/worker/dao.go index 35cd061d49..8036b72393 100644 --- a/engine/api/worker/dao.go +++ b/engine/api/worker/dao.go @@ -11,8 +11,13 @@ import ( "github.com/ovh/cds/sdk" ) -func Insert(db gorp.SqlExecutor, w *sdk.Worker) error { - return gorpmapping.Insert(db, w) +func Insert(ctx context.Context, db gorp.SqlExecutor, w *sdk.Worker) error { + dbData := &dbWorker{Worker: *w} + if err := gorpmapping.InsertAndSign(ctx, db, dbData); err != nil { + return err + } + *w = dbData.Worker + return nil } // Delete remove worker from database, it also removes the associated access_token @@ -93,27 +98,56 @@ func LoadDeadWorkers(ctx context.Context, db gorp.SqlExecutor, timeout float64, } // SetStatus sets job_run_id and status to building on given worker -func SetStatus(db gorp.SqlExecutor, workerID string, status string) error { - query := `UPDATE worker SET status = $1 WHERE id = $2` +func SetStatus(ctx context.Context, db gorp.SqlExecutor, workerID string, status string) error { + w, err := LoadByID(ctx, db, workerID) + if err != nil { + return err + } + w.Status = status if status == sdk.StatusBuilding || status == sdk.StatusWaiting { - query = `UPDATE worker SET status = $1, job_run_id = NULL WHERE id = $2` + w.JobRunID = nil } - - if _, err := db.Exec(query, status, workerID); err != nil { - return sdk.WithStack(err) + dbData := &dbWorker{Worker: *w} + if err := gorpmapping.UpdateAndSign(ctx, db, dbData); err != nil { + return err } return nil } // SetToBuilding sets job_run_id and status to building on given worker -func SetToBuilding(db gorp.SqlExecutor, workerID string, jobRunID int64) error { - query := `UPDATE worker SET status = $1, job_run_id = $2 WHERE id = $3` - - res, errE := db.Exec(query, sdk.StatusBuilding, jobRunID, workerID) - if errE != nil { - return sdk.WithStack(errE) +func SetToBuilding(ctx context.Context, db gorp.SqlExecutor, workerID string, jobRunID int64, key []byte) error { + w, err := LoadByID(ctx, db, workerID) + if err != nil { + return err } + w.Status = sdk.StatusBuilding + w.JobRunID = &jobRunID + w.PrivateKey = key - _, err := res.RowsAffected() + dbData := &dbWorker{Worker: *w} + if err := gorpmapping.UpdateAndSign(ctx, db, dbData); err != nil { + return err + } return err } + +// LoadWorkerWithDecryptKey load worker with decrypted private key +func LoadWorkerWithDecryptKey(ctx context.Context, db gorp.SqlExecutor, workerID string) (sdk.Worker, error) { + var work dbWorker + query := gorpmapping.NewQuery(`SELECT * FROM worker WHERE id = $1`).Args(workerID) + found, err := gorpmapping.Get(ctx, db, query, &work, gorpmapping.GetOptions.WithDecryption) + if err != nil { + return sdk.Worker{}, err + } + if !found { + return sdk.Worker{}, sdk.ErrNotFound + } + isValid, err := gorpmapping.CheckSignature(work, work.Signature) + if err != nil { + return sdk.Worker{}, err + } + if !isValid { + return sdk.Worker{}, sdk.ErrInvalidData + } + return work.Worker, err +} diff --git a/engine/api/worker/gorp_model.go b/engine/api/worker/gorp_model.go new file mode 100644 index 0000000000..3e748823eb --- /dev/null +++ b/engine/api/worker/gorp_model.go @@ -0,0 +1,22 @@ +package worker + +import ( + "github.com/ovh/cds/engine/api/database/gorpmapping" + "github.com/ovh/cds/sdk" +) + +type dbWorker struct { + gorpmapping.SignedEntity + sdk.Worker +} + +func init() { + gorpmapping.Register(gorpmapping.New(dbWorker{}, "worker", false, "id")) +} + +func (e dbWorker) Canonical() gorpmapping.CanonicalForms { + var _ = []interface{}{e.ID, e.Name} + return gorpmapping.CanonicalForms{ + "{{print .ID}}{{.Name}}", + } +} diff --git a/engine/api/worker/heartbeat.go b/engine/api/worker/heartbeat.go index f689d3d04a..029c3fa921 100644 --- a/engine/api/worker/heartbeat.go +++ b/engine/api/worker/heartbeat.go @@ -19,7 +19,7 @@ func DisableDeadWorkers(ctx context.Context, db *gorp.DbMap) error { } for i := range workers { log.Debug("Disable worker %s[%s] LastBeat:%v status:%s", workers[i].Name, workers[i].ID, workers[i].LastBeat, workers[i].Status) - if errD := SetStatus(db, workers[i].ID, sdk.StatusDisabled); errD != nil { + if errD := SetStatus(ctx, db, workers[i].ID, sdk.StatusDisabled); errD != nil { log.Warning(ctx, "Cannot disable worker %v: %v", workers[i].ID, errD) } } diff --git a/engine/api/worker/registration.go b/engine/api/worker/registration.go index 01463cc7ac..a98c09117b 100644 --- a/engine/api/worker/registration.go +++ b/engine/api/worker/registration.go @@ -98,7 +98,7 @@ func RegisterWorker(ctx context.Context, db gorp.SqlExecutor, store cache.Store, w.Uptodate = registrationForm.Version == sdk.VERSION - if err := Insert(db, w); err != nil { + if err := Insert(ctx, db, w); err != nil { return nil, err } diff --git a/engine/api/worker/worker_test.go b/engine/api/worker/worker_test.go index 07de358205..a34427aff0 100644 --- a/engine/api/worker/worker_test.go +++ b/engine/api/worker/worker_test.go @@ -38,7 +38,7 @@ func TestDAO(t *testing.T) { Status: sdk.StatusWaiting, } - if err := worker.Insert(db, w); err != nil { + if err := worker.Insert(context.TODO(), db, w); err != nil { t.Fatalf("Cannot insert worker %+v: %v", w, err) } @@ -57,7 +57,7 @@ func TestDAO(t *testing.T) { assert.Equal(t, "foofoo", wk.ID) } - test.NoError(t, worker.SetStatus(db, wk.ID, sdk.StatusBuilding)) + test.NoError(t, worker.SetStatus(context.TODO(), db, wk.ID, sdk.StatusBuilding)) test.NoError(t, worker.RefreshWorker(db, wk.ID)) } diff --git a/engine/api/workflow_queue.go b/engine/api/workflow_queue.go index 53930550dc..994a4dbb75 100644 --- a/engine/api/workflow_queue.go +++ b/engine/api/workflow_queue.go @@ -99,6 +99,7 @@ func (api *API) postTakeWorkflowJobHandler() service.Handler { return sdk.WrapError(err, "cannot takeJob nodeJobRunID:%d", id) } + pbji.GelfServiceAddr = fmt.Sprintf("%s:%d", api.Config.CDN.TCP.Addr, api.Config.CDN.TCP.Port) workflow.ResyncNodeRunsWithCommits(ctx, api.mustDB(), api.Cache, *p, report) go WorkflowSendEvent(context.Background(), api.mustDB(), api.Cache, *p, report) @@ -132,10 +133,16 @@ func takeJob(ctx context.Context, dbFunc func() *gorp.DbMap, store cache.Store, return nil, sdk.WrapError(err, "cannot take job %d", id) } + workerKey, err := jws.NewRandomSymmetricKey(32) + if err != nil { + return nil, err + } + // Change worker status - if err := worker.SetToBuilding(tx, wk.ID, job.ID); err != nil { + if err := worker.SetToBuilding(ctx, tx, wk.ID, job.ID, workerKey); err != nil { return nil, sdk.WrapError(err, "cannot update worker %s status", wk.Name) } + wnjri.SigningKey = string(workerKey) // Load the node run noderun, err := workflow.LoadNodeRunByID(tx, job.WorkflowNodeRunID, workflow.LoadRunOptions{}) @@ -181,10 +188,6 @@ func takeJob(ctx context.Context, dbFunc func() *gorp.DbMap, store cache.Store, wnjri.Secrets = append(wnjri.Secrets, secretsKeys...) wnjri.NodeJobRun.Parameters = append(wnjri.NodeJobRun.Parameters, params...) - wnjri.SigningKey, err = jws.NewRandomSymmetricKey(32) - // get cdn url - wnjri.GelfServiceAddr = "http://localhost:8089/logs" - if err != nil { return nil, err } @@ -495,7 +498,7 @@ func postJobResult(ctx context.Context, dbFunc func(context.Context) *gorp.DbMap // ^ build variables are now updated on job run and on node //Update worker status - if err := worker.SetStatus(tx, wr.ID, sdk.StatusWaiting); err != nil { + if err := worker.SetStatus(ctx, tx, wr.ID, sdk.StatusWaiting); err != nil { return nil, sdk.WrapError(err, "cannot update worker %s status", wr.ID) } @@ -534,15 +537,15 @@ func (api *API) postWorkflowJobLogsHandler() service.Handler { return sdk.WrapError(err, "invalid id") } + if ok := isWorker(ctx); !ok { + return sdk.WithStack(sdk.ErrForbidden) + } + pbJob, err := workflow.LoadNodeJobRun(ctx, api.mustDB(), api.Cache, id) if err != nil { return sdk.WrapError(err, "cannot get job run %d", id) } - if ok := isWorker(ctx); !ok { - return sdk.WithStack(sdk.ErrForbidden) - } - // Checks that the token used by the worker cas access to one of the execgroups grantedGroupIDs := append(getAPIConsumer(ctx).GetGroupIDs(), group.SharedInfraGroup.ID) if !pbJob.ExecGroups.HasOneOf(grantedGroupIDs...) { diff --git a/engine/api/workflow_queue_test.go b/engine/api/workflow_queue_test.go index ab19684827..e82560858f 100644 --- a/engine/api/workflow_queue_test.go +++ b/engine/api/workflow_queue_test.go @@ -1004,7 +1004,7 @@ func TestPostVulnerabilityReportHandler(t *testing.T) { } testRegisterWorker(t, api, router, &ctx) ctx.worker.JobRunID = &wrDB.WorkflowNodeRuns[w.WorkflowData.Node.ID][0].Stages[0].RunJobs[0].ID - assert.NoError(t, worker.SetToBuilding(db, ctx.worker.ID, *ctx.worker.JobRunID)) + assert.NoError(t, worker.SetToBuilding(context.TODO(), db, ctx.worker.ID, *ctx.worker.JobRunID, nil)) request := sdk.VulnerabilityWorkerReport{ Vulnerabilities: []sdk.Vulnerability{ @@ -1326,7 +1326,7 @@ func TestInsertNewCodeCoverageReport(t *testing.T) { } testRegisterWorker(t, api, router, &ctx) ctx.worker.JobRunID = &wrr.WorkflowNodeRuns[w.WorkflowData.Node.ID][0].Stages[0].RunJobs[0].ID - assert.NoError(t, worker.SetToBuilding(db, ctx.worker.ID, *ctx.worker.JobRunID)) + assert.NoError(t, worker.SetToBuilding(context.TODO(), db, ctx.worker.ID, *ctx.worker.JobRunID, nil)) uri := router.GetRoute("POST", api.postWorkflowJobCoverageResultsHandler, vars) test.NotEmpty(t, uri) diff --git a/engine/cdn/cdn.go b/engine/cdn/cdn.go index 311bb7d3b9..53d1c4a09d 100644 --- a/engine/cdn/cdn.go +++ b/engine/cdn/cdn.go @@ -5,9 +5,6 @@ import ( "fmt" "net/http" - "github.com/gorilla/mux" - - "github.com/ovh/cds/engine/api" "github.com/ovh/cds/engine/api/services" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/cdsclient" @@ -17,9 +14,11 @@ import ( // New returns a new service func New() *Service { s := new(Service) - s.Router = &api.Router{ - Mux: mux.NewRouter(), - } + /* + s.Router = &api.Router{ + Mux: mux.NewRouter(), + } + */ return s } @@ -77,16 +76,16 @@ func (s *Service) Serve(c context.Context) error { ctx, cancel := context.WithCancel(c) defer cancel() + s.RunTcpLogServer(ctx) + //Init the http server s.initRouter(ctx) server := &http.Server{ - Addr: fmt.Sprintf("%s:%d", s.Cfg.HTTP.Addr, s.Cfg.HTTP.Port), - Handler: s.Router.Mux, + Addr: fmt.Sprintf("%s:%d", s.Cfg.HTTP.Addr, s.Cfg.HTTP.Port), + //Handler: s.Router.Mux, MaxHeaderBytes: 1 << 20, } - s.initGelfServer() - //Gracefully shutdown the http server go func() { select { @@ -101,6 +100,5 @@ func (s *Service) Serve(c context.Context) error { if err := server.ListenAndServe(); err != nil { log.Fatalf("CDN> Cannot start cds-cdn: %s", err) } - return ctx.Err() } diff --git a/engine/cdn/cdn_log.go b/engine/cdn/cdn_log.go index 4e6a466a5a..48ddfc5d96 100644 --- a/engine/cdn/cdn_log.go +++ b/engine/cdn/cdn_log.go @@ -1,5 +1,142 @@ package cdn -func (s *Service) initGelfServer() { +import ( + "bufio" + "context" + "fmt" + "net" + "time" + "github.com/ovh/cds/engine/api/worker" + "github.com/ovh/cds/engine/api/workflow" + "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/jws" + "github.com/ovh/cds/sdk/log" + "github.com/ovh/cds/sdk/log/hook" +) + +var ( + workers = make(map[string]sdk.Worker) +) + +func (s *Service) RunTcpLogServer(ctx context.Context) { + log.Info(ctx, "Starting tcp server %s:%d", s.Cfg.TCP.Addr, s.Cfg.TCP.Port) + listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.Cfg.TCP.Addr, s.Cfg.TCP.Port)) + if err != nil { + log.Fatalf("unable to start tcp log server: %v", err) + } + + //Gracefully shutdown the tcp server + go func() { + select { + case <-ctx.Done(): + log.Info(ctx, "CDN> Shutdown tcp log Server") + _ = listener.Close() + } + }() + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + log.Error(ctx, "unable to accept connection: %v", err) + return + } + go s.handleConnection(ctx, conn) + } + }() +} + +func (s *Service) handleConnection(ctx context.Context, conn net.Conn) { + defer func() { + _ = conn.Close() + }() + + timeoutDuration := 5 * time.Second + bufReader := bufio.NewReader(conn) + + for { + // Set a deadline for reading. Read operation will fail if no data is received after deadline. + if err := conn.SetReadDeadline(time.Now().Add(timeoutDuration)); err != nil { + log.Error(ctx, "unable to set read deadline on connection") + return + } + bytes, err := bufReader.ReadBytes(byte(0)) + if err != nil { + log.Info(ctx, "client left") + return + } + log.Warning(ctx, "Message IN CDNNNNNNNNNN") + var m *hook.Message + if err := m.UnmarshalJSON(bytes); err != nil { + log.Error(ctx, "cdn.log > unable to unmarshal log message: %v", err) + continue + } + + sig, ok := m.Extra[log.ExtraFieldSignature] + if !ok || sig == "" { + log.Error(ctx, "cdn.log > signature not found on log message %+v", m) + continue + } + + stepOrderI, ok := m.Extra[log.ExtraFieldStepOrder] + if !ok { + log.Error(ctx, "cdn.log > missing step order extra field") + continue + } + stepOrder, ok := stepOrderI.(int64) + if !ok { + log.Error(ctx, "cdn.log > unable to read step order extra field") + continue + } + + // Get worker datas + var workerSign sdk.WorkerSignature + if err := jws.UnsafeParse(sig.(string), &workerSign); err != nil { + log.Error(ctx, "cdn.log > unable to unsafe parse log signature: %v", err) + continue + } + workerData, ok := workers[workerSign.WorkerID] + if !ok { + var err error + workerData, err = s.getWorker(ctx, workerSign.WorkerID) + if err != nil { + log.Error(ctx, "cdn.log > unable to retrieve worker data from api: %v", err) + continue + } + } + if err := jws.Verify(workerData.PrivateKey, sig.(string), &workerSign); err != nil { + log.Error(ctx, "cdn.log > unable to verify signature") + continue + } + + pbJob, err := workflow.LoadNodeJobRun(ctx, s.Db, s.Cache, workerSign.JobID) + if err != nil { + log.Error(ctx, "cdn.log > unable to verify signature") + continue + } + + logDate := time.Unix(0, int64(m.Time*1e9)) + logs := sdk.Log{ + JobID: pbJob.ID, + LastModified: &logDate, + NodeRunID: pbJob.WorkflowNodeRunID, + Start: &logDate, + StepOrder: stepOrder, + Val: m.Full, + } + if err := workflow.AddLog(s.Db, pbJob, &logs, s.Cfg.Log.StepMaxSize); err != nil { + log.Error(ctx, "cdn.log > unable to insert log") + continue + } + } +} + +func (s *Service) getWorker(ctx context.Context, workerID string) (sdk.Worker, error) { + w, err := worker.LoadWorkerWithDecryptKey(ctx, s.Db, workerID) + if err != nil { + return sdk.Worker{}, err + } + workers[w.ID] = w + return w, nil } diff --git a/engine/cdn/cdn_router.go b/engine/cdn/cdn_router.go index 57ba3dfc64..9f343b9ad4 100644 --- a/engine/cdn/cdn_router.go +++ b/engine/cdn/cdn_router.go @@ -2,20 +2,17 @@ package cdn import ( "context" - - "github.com/ovh/cds/engine/api" - "github.com/ovh/cds/engine/service" ) func (s *Service) initRouter(ctx context.Context) { - r := s.Router - r.Background = ctx - r.URL = s.Cfg.URL - r.SetHeaderFunc = api.DefaultHeaders - r.Middlewares = append(r.Middlewares, service.CheckRequestSignatureMiddleware(s.ParsedAPIPublicKey)) + //r := s.Router + //r.Background = ctx + //r.URL = s.Cfg.URL + //r.SetHeaderFunc = api.DefaultHeaders + //r.Middlewares = append(r.Middlewares, service.CheckRequestSignatureMiddleware(s.ParsedAPIPublicKey)) - r.Handle("/mon/version", nil, r.GET(api.VersionHandler, api.Auth(false))) - r.Handle("/mon/status", nil, r.GET(s.statusHandler, api.Auth(false))) - r.Handle("/mon/metrics", nil, r.GET(service.GetPrometheustMetricsHandler(s), api.Auth(false))) - r.Handle("/mon/metrics/all", nil, r.GET(service.GetMetricsHandler, api.Auth(false))) + //r.Handle("/mon/version", nil, r.GET(api.VersionHandler, api.Auth(false))) + //r.Handle("/mon/status", nil, r.GET(s.statusHandler, api.Auth(false))) + //r.Handle("/mon/metrics", nil, r.GET(service.GetPrometheustMetricsHandler(s), api.Auth(false))) + //r.Handle("/mon/metrics/all", nil, r.GET(service.GetMetricsHandler, api.Auth(false))) } diff --git a/engine/cdn/types.go b/engine/cdn/types.go index a13ee23967..0cb918e1e8 100644 --- a/engine/cdn/types.go +++ b/engine/cdn/types.go @@ -1,24 +1,35 @@ package cdn import ( - "github.com/ovh/cds/engine/api" + "github.com/go-gorp/gorp" + "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/service" ) // Service is the stuct representing a hooks µService type Service struct { service.Common - Cfg Configuration - Router *api.Router + Cfg Configuration + //Router *api.Router + Db *gorp.DbMap + Cache cache.Store } // Configuration is the hooks configuration structure type Configuration struct { Name string `toml:"name" default:"cds-cdn" comment:"Name of this CDS CDN Service\n Enter a name to enable this service" json:"name"` + TCP struct { + Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"` + Port int `toml:"port" default:"8089" json:"port"` + } `toml:"tcp" comment:"######################\n CDS CDN TCP Configuration \n######################" json:"tcp"` HTTP struct { Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"` Port int `toml:"port" default:"8089" json:"port"` } `toml:"http" comment:"######################\n CDS CDN HTTP Configuration \n######################" json:"http"` URL string `default:"http://localhost:8087" json:"url"` API service.APIServiceConfiguration `toml:"api" comment:"######################\n CDS API Settings \n######################" json:"api"` + Log struct { + StepMaxSize int64 `toml:"stepMaxSize" default:"15728640" comment:"Max step logs size in bytes (default: 15MB)" json:"stepMaxSize"` + ServiceMaxSize int64 `toml:"serviceMaxSize" default:"15728640" comment:"Max service logs size in bytes (default: 15MB)" json:"serviceMaxSize"` + } `toml:"log" json:"log" comment:"###########################\n Log settings.\n##########################"` } diff --git a/engine/sql/200_worker.sql b/engine/sql/200_worker.sql new file mode 100644 index 0000000000..9a99b46e60 --- /dev/null +++ b/engine/sql/200_worker.sql @@ -0,0 +1,10 @@ +-- +migrate Up +ALTER TABLE "worker" ADD COLUMN IF NOT EXISTS cypher_private_key BYTEA; +ALTER TABLE "worker" ADD COLUMN IF NOT EXISTS sig BYTEA; +ALTER TABLE "worker" ADD COLUMN IF NOT EXISTS signer TEXT; + + +-- +migrate Down +ALTER TABLE "worker" DROP COLUMN cypher_hmac_key; +ALTER TABLE "worker" DROP COLUMN sig; +ALTER TABLE "worker" DROP COLUMN signer; diff --git a/engine/worker/internal/take.go b/engine/worker/internal/take.go index dd239706ec..a8e87bd534 100644 --- a/engine/worker/internal/take.go +++ b/engine/worker/internal/take.go @@ -6,11 +6,12 @@ import ( "time" "github.com/sirupsen/logrus" - "gopkg.in/Graylog2/go-gelf.v2/gelf" "github.com/ovh/cds/engine/worker/pkg/workerruntime" "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/jws" "github.com/ovh/cds/sdk/log" + loghook "github.com/ovh/cds/sdk/log/hook" ) func (w *CurrentWorker) Take(ctx context.Context, job sdk.WorkflowNodeJobRun) error { @@ -33,16 +34,29 @@ func (w *CurrentWorker) Take(ctx context.Context, job sdk.WorkflowNodeJobRun) er w.currentJob.secrets = info.Secrets // Reset build variables w.currentJob.newVariables = nil - w.currentJob.signingKey = info.SigningKey - log.Info(ctx, "Setup step logger") - w.logger.stepLogger = logrus.New() - gelfWriter, err := gelf.NewTCPWriter(info.GelfServiceAddr) - if err != nil { - return sdk.WithStack(err) + if info.SigningKey != "" { + signer, err := jws.NewHMacSigner(info.SigningKey) + if err != nil { + return sdk.WithStack(err) + } + w.currentJob.signer = signer + } + + if info.GelfServiceAddr != "" { + log.Info(ctx, "Setup step logger") + w.logger.stepLogger = logrus.New() + graylogcfg := &loghook.Config{ + Addr: info.GelfServiceAddr, + Protocol: "tcp", + } + extra := map[string]interface{}{} + hook, err := loghook.NewHook(graylogcfg, extra) + if err != nil { + return sdk.WithStack(err) + } + w.logger.stepLogger.AddHook(hook) } - w.logger.stepLogger.SetOutput(gelfWriter) - log.Info(ctx, "Setup step logger done") start := time.Now() diff --git a/engine/worker/internal/types.go b/engine/worker/internal/types.go index 65e6542bcb..da04780b35 100644 --- a/engine/worker/internal/types.go +++ b/engine/worker/internal/types.go @@ -5,17 +5,18 @@ import ( "context" "encoding/json" "fmt" - "github.com/sirupsen/logrus" "os" "strings" "time" "github.com/ovh/cds/engine/worker/pkg/workerruntime" - + "github.com/sirupsen/logrus" "github.com/spf13/afero" + "gopkg.in/square/go-jose.v2" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/cdsclient" + "github.com/ovh/cds/sdk/jws" "github.com/ovh/cds/sdk/log" ) @@ -44,7 +45,7 @@ type CurrentWorker struct { params []sdk.Parameter secrets []sdk.Variable context context.Context - signingKey string + signer jose.Signer } status struct { Name string `json:"name"` @@ -81,15 +82,42 @@ func (wk *CurrentWorker) Parameters() []sdk.Parameter { func (wk *CurrentWorker) SendLog(ctx context.Context, level workerruntime.Level, s string) { jobID, _ := workerruntime.JobID(ctx) stepOrder, err := workerruntime.StepOrder(ctx) - if !strings.HasSuffix(s, "\n") { - s += "\n" + if wk.logger.stepLogger != nil { + if !strings.HasSuffix(s, "\n") { + s += "\n" + } + if err != nil { + log.Error(ctx, "SendLog> %v", err) + } + if err := wk.sendLog(jobID, fmt.Sprintf("[%s] ", level)+s, stepOrder, false); err != nil { + log.Error(ctx, "SendLog> %v", err) + } + return } - if err != nil { - log.Error(ctx, "SendLog> %v", err) + var logLevel logrus.Level + switch level { + case workerruntime.LevelDebug: + logLevel = logrus.DebugLevel + case workerruntime.LevelInfo: + logLevel = logrus.InfoLevel + case workerruntime.LevelWarn: + logLevel = logrus.WarnLevel + case workerruntime.LevelError: + logLevel = logrus.ErrorLevel + default: } - if err := wk.sendLog(jobID, fmt.Sprintf("[%s] ", level)+s, stepOrder, false); err != nil { - log.Error(ctx, "SendLog> %v", err) + + dataToSign := sdk.WorkerSignature{ + WorkerID: wk.id, + JobID: wk.currentJob.wJob.ID, + Timestamp: time.Now().UnixNano(), } + signature, err := jws.Sign(wk.currentJob.signer, dataToSign) + if err != nil { + log.Error(ctx, "unable to sign logs: %v", err) + } + wk.logger.stepLogger.WithField("Signature", signature).WithField("StepOrder", stepOrder).Log(logLevel, s) + } func (wk *CurrentWorker) Name() string { diff --git a/go.mod b/go.mod index d762ea9484..0c5d3ce3f7 100644 --- a/go.mod +++ b/go.mod @@ -193,7 +193,7 @@ require ( google.golang.org/genproto v0.0.0-20190817000702-55e96fffbd48 // indirect google.golang.org/grpc v1.23.0 gopkg.in/AlecAivazis/survey.v1 v1.7.1 - gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0 // indirect + gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/bsm/sarama-cluster.v2 v2.1.4 gopkg.in/go-playground/assert.v1 v1.2.1 // indirect diff --git a/sdk/jws/jws.go b/sdk/jws/jws.go index 6d33ff3fd2..10df0c705b 100644 --- a/sdk/jws/jws.go +++ b/sdk/jws/jws.go @@ -15,16 +15,16 @@ import ( "github.com/ovh/cds/sdk" ) -func NewRandomSymmetricKey(size int) (string, error) { +func NewRandomSymmetricKey(size int) ([]byte, error) { if size <= 0 || size%8 != 0 { - return "", sdk.WithStack(fmt.Errorf("invalid key size")) + return nil, sdk.WithStack(fmt.Errorf("invalid key size")) } k := make([]byte, size) if _, err := rand.Read(k); err != nil { - return "", sdk.WithStack(err) + return nil, sdk.WithStack(err) } - return string(k), nil + return k, nil } // NewRandomRSAKey generates a public/private key pair diff --git a/sdk/log/log.go b/sdk/log/log.go index 22464dbdf0..efc7b8e629 100644 --- a/sdk/log/log.go +++ b/sdk/log/log.go @@ -33,6 +33,9 @@ const ( HeaderRequestID = "Request-ID" ContextLoggingRequestIDKey = "ctx-logging-request-id" ContextLoggingFuncKey = "ctx-logging-func" + + ExtraFieldSignature = "Signature" + ExtraFieldStepOrder = "StepOrder" ) var ( diff --git a/sdk/worker.go b/sdk/worker.go index 8db647c6cb..bdb8255a5a 100644 --- a/sdk/worker.go +++ b/sdk/worker.go @@ -20,6 +20,13 @@ type Worker struct { Version string `json:"version" cli:"version" db:"version"` OS string `json:"os" cli:"os" db:"os"` Arch string `json:"arch" cli:"arch" db:"arch"` + PrivateKey []byte `json:"-" cli:"-" db:"cypher_private_key" gorpmapping:"encrypted,ID,Name"` +} + +type WorkerSignature struct { + WorkerID string + JobID int64 + Timestamp int64 } // WorkerRegistrationForm represents the arguments needed to register a worker From 99be8c316681bddf4a22bc778cfe81a5a9a0af21 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Mon, 27 Apr 2020 15:41:37 +0200 Subject: [PATCH 04/19] wip --- engine/api/database/gorpmapping/encryption.go | 4 +-- engine/api/worker/dao.go | 34 +++++++++++++------ engine/api/worker/gorp_model.go | 18 ++++++++++ engine/api/worker/init.go | 6 ---- engine/api/workflow_queue.go | 3 +- engine/cdn/cdn_log.go | 33 ++++++++++++------ engine/worker/internal/take.go | 7 +++- engine/worker/internal/types.go | 2 +- sdk/jws/jws.go | 8 +++-- sdk/log/hook/hook.go | 3 +- sdk/log/log.go | 4 +-- .../node/pipeline/step/step.log.component.ts | 4 +-- 12 files changed, 85 insertions(+), 41 deletions(-) diff --git a/engine/api/database/gorpmapping/encryption.go b/engine/api/database/gorpmapping/encryption.go index dc206b0cbf..6beb9dda47 100644 --- a/engine/api/database/gorpmapping/encryption.go +++ b/engine/api/database/gorpmapping/encryption.go @@ -110,8 +110,8 @@ func updateEncryptedData(db gorp.SqlExecutor, i interface{}) error { updateSlice = append(updateSlice, encryptedColumns[f]+" = $"+strconv.Itoa(c)) c++ } - - query := fmt.Sprintf("UPDATE %s SET %s WHERE %s = %v", table, strings.Join(updateSlice, ","), key, id) + encryptedContentArgs = append(encryptedContentArgs, id) + query := fmt.Sprintf("UPDATE %s SET %s WHERE %s = $%d", table, strings.Join(updateSlice, ","), key, c) log.Warning(context.TODO(), ">>>>%s", query) res, err := db.Exec(query, encryptedContentArgs...) if err != nil { diff --git a/engine/api/worker/dao.go b/engine/api/worker/dao.go index 8036b72393..1dc820a33f 100644 --- a/engine/api/worker/dao.go +++ b/engine/api/worker/dao.go @@ -9,6 +9,7 @@ import ( "github.com/ovh/cds/engine/api/authentication" "github.com/ovh/cds/engine/api/database/gorpmapping" "github.com/ovh/cds/sdk" + "github.com/ovh/cds/sdk/log" ) func Insert(ctx context.Context, db gorp.SqlExecutor, w *sdk.Worker) error { @@ -42,7 +43,7 @@ func Delete(db gorp.SqlExecutor, id string) error { func LoadByConsumerID(ctx context.Context, db gorp.SqlExecutor, id string) (*sdk.Worker, error) { query := gorpmapping.NewQuery("SELECT * FROM worker WHERE auth_consumer_id = $1").Args(id) - var w sdk.Worker + var w dbWorker found, err := gorpmapping.Get(ctx, db, query, &w) if err != nil { return nil, err @@ -50,12 +51,12 @@ func LoadByConsumerID(ctx context.Context, db gorp.SqlExecutor, id string) (*sdk if !found { return nil, sdk.WithStack(sdk.ErrNotFound) } - return &w, nil + return &w.Worker, nil } func LoadByID(ctx context.Context, db gorp.SqlExecutor, id string) (*sdk.Worker, error) { query := gorpmapping.NewQuery("SELECT * FROM worker WHERE id = $1").Args(id) - var w sdk.Worker + var w dbWorker found, err := gorpmapping.Get(ctx, db, query, &w) if err != nil { return nil, err @@ -63,37 +64,49 @@ func LoadByID(ctx context.Context, db gorp.SqlExecutor, id string) (*sdk.Worker, if !found { return nil, sdk.WithStack(sdk.ErrNotFound) } - return &w, nil + return &w.Worker, nil } func LoadAll(ctx context.Context, db gorp.SqlExecutor) ([]sdk.Worker, error) { - var workers []sdk.Worker + var wks []dbWorker query := gorpmapping.NewQuery(`SELECT * FROM worker ORDER BY name ASC`) - if err := gorpmapping.GetAll(ctx, db, query, &workers); err != nil { + if err := gorpmapping.GetAll(ctx, db, query, &wks); err != nil { return nil, err } + workers := make([]sdk.Worker, len(wks)) + for i := range wks { + workers[i] = wks[i].Worker + } return workers, nil } func LoadByHatcheryID(ctx context.Context, db gorp.SqlExecutor, hatcheryID int64) ([]sdk.Worker, error) { - var workers []sdk.Worker + var wks []dbWorker query := gorpmapping.NewQuery(`SELECT * FROM worker WHERE hatchery_id = $1 ORDER BY name ASC`).Args(hatcheryID) - if err := gorpmapping.GetAll(ctx, db, query, &workers); err != nil { + if err := gorpmapping.GetAll(ctx, db, query, &wks); err != nil { return nil, err } + workers := make([]sdk.Worker, len(wks)) + for i := range wks { + workers[i] = wks[i].Worker + } return workers, nil } func LoadDeadWorkers(ctx context.Context, db gorp.SqlExecutor, timeout float64, status []string) ([]sdk.Worker, error) { - var workers []sdk.Worker + var wks []dbWorker query := gorpmapping.NewQuery(`SELECT * FROM worker WHERE status = ANY(string_to_array($1, ',')::text[]) AND now() - last_beat > $2 * INTERVAL '1' SECOND ORDER BY last_beat ASC`).Args(strings.Join(status, ","), timeout) - if err := gorpmapping.GetAll(ctx, db, query, &workers); err != nil { + if err := gorpmapping.GetAll(ctx, db, query, &wks); err != nil { return nil, err } + workers := make([]sdk.Worker, len(wks)) + for i := range wks { + workers[i] = wks[i].Worker + } return workers, nil } @@ -149,5 +162,6 @@ func LoadWorkerWithDecryptKey(ctx context.Context, db gorp.SqlExecutor, workerID if !isValid { return sdk.Worker{}, sdk.ErrInvalidData } + log.Warning(ctx, "My WORKER: %+v", work.Worker) return work.Worker, err } diff --git a/engine/api/worker/gorp_model.go b/engine/api/worker/gorp_model.go index 3e748823eb..411d0d0ba2 100644 --- a/engine/api/worker/gorp_model.go +++ b/engine/api/worker/gorp_model.go @@ -20,3 +20,21 @@ func (e dbWorker) Canonical() gorpmapping.CanonicalForms { "{{print .ID}}{{.Name}}", } } + +func (e dbWorker) GetWorker() sdk.Worker { + return sdk.Worker{ + ID: e.ID, + PrivateKey: e.PrivateKey, + Status: e.Status, + JobRunID: e.JobRunID, + Arch: e.Arch, + ConsumerID: e.ConsumerID, + HatcheryID: e.HatcheryID, + LastBeat: e.LastBeat, + ModelID: e.ModelID, + Name: e.Name, + OS: e.OS, + Uptodate: e.Uptodate, + Version: e.Version, + } +} diff --git a/engine/api/worker/init.go b/engine/api/worker/init.go index 5d5aee2e4e..ec2e49005a 100644 --- a/engine/api/worker/init.go +++ b/engine/api/worker/init.go @@ -7,8 +7,6 @@ import ( "github.com/go-gorp/gorp" "github.com/ovh/cds/engine/api/cache" - "github.com/ovh/cds/engine/api/database/gorpmapping" - "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" ) @@ -39,7 +37,3 @@ func Initialize(c context.Context, DBFunc func() *gorp.DbMap, store cache.Store) } } } - -func init() { - gorpmapping.Register(gorpmapping.New(sdk.Worker{}, "worker", false, "id")) -} diff --git a/engine/api/workflow_queue.go b/engine/api/workflow_queue.go index 994a4dbb75..86d2962727 100644 --- a/engine/api/workflow_queue.go +++ b/engine/api/workflow_queue.go @@ -2,6 +2,7 @@ package api import ( "context" + "encoding/base64" "fmt" "net/http" "strconv" @@ -142,7 +143,7 @@ func takeJob(ctx context.Context, dbFunc func() *gorp.DbMap, store cache.Store, if err := worker.SetToBuilding(ctx, tx, wk.ID, job.ID, workerKey); err != nil { return nil, sdk.WrapError(err, "cannot update worker %s status", wk.Name) } - wnjri.SigningKey = string(workerKey) + wnjri.SigningKey = base64.StdEncoding.EncodeToString(workerKey) // Load the node run noderun, err := workflow.LoadNodeRunByID(tx, job.WorkflowNodeRunID, workflow.LoadRunOptions{}) diff --git a/engine/cdn/cdn_log.go b/engine/cdn/cdn_log.go index 48ddfc5d96..4b8ccb7e71 100644 --- a/engine/cdn/cdn_log.go +++ b/engine/cdn/cdn_log.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "net" + "strings" "time" "github.com/ovh/cds/engine/api/worker" @@ -66,10 +67,15 @@ func (s *Service) handleConnection(ctx context.Context, conn net.Conn) { log.Info(ctx, "client left") return } - log.Warning(ctx, "Message IN CDNNNNNNNNNN") - var m *hook.Message - if err := m.UnmarshalJSON(bytes); err != nil { - log.Error(ctx, "cdn.log > unable to unmarshal log message: %v", err) + if len(bytes) == 0 { + continue + } + if bytes[len(bytes)-1] == byte(0) { + bytes = bytes[:len(bytes)-1] + } + m := hook.Message{} + if err := (&m).UnmarshalJSON(bytes); err != nil { + log.Error(ctx, "cdn.log > unable to unmarshal log message: %s %v", string(bytes), err) continue } @@ -84,11 +90,7 @@ func (s *Service) handleConnection(ctx context.Context, conn net.Conn) { log.Error(ctx, "cdn.log > missing step order extra field") continue } - stepOrder, ok := stepOrderI.(int64) - if !ok { - log.Error(ctx, "cdn.log > unable to read step order extra field") - continue - } + stepOrder := int64(stepOrderI.(float64)) // Get worker datas var workerSign sdk.WorkerSignature @@ -106,7 +108,7 @@ func (s *Service) handleConnection(ctx context.Context, conn net.Conn) { } } if err := jws.Verify(workerData.PrivateKey, sig.(string), &workerSign); err != nil { - log.Error(ctx, "cdn.log > unable to verify signature") + log.Error(ctx, "cdn.log > unable to verify signature: %v", err) continue } @@ -125,10 +127,19 @@ func (s *Service) handleConnection(ctx context.Context, conn net.Conn) { StepOrder: stepOrder, Val: m.Full, } - if err := workflow.AddLog(s.Db, pbJob, &logs, s.Cfg.Log.StepMaxSize); err != nil { + tx, err := s.Db.Begin() + if err != nil { + + } + if !strings.HasSuffix(logs.Val, "\n") { + logs.Val += "\n" + } + if err := workflow.AddLog(tx, pbJob, &logs, s.Cfg.Log.StepMaxSize); err != nil { log.Error(ctx, "cdn.log > unable to insert log") + _ = tx.Rollback() continue } + _ = tx.Commit() } } diff --git a/engine/worker/internal/take.go b/engine/worker/internal/take.go index a8e87bd534..5204fd89e7 100644 --- a/engine/worker/internal/take.go +++ b/engine/worker/internal/take.go @@ -2,6 +2,7 @@ package internal import ( "context" + "encoding/base64" "strings" "time" @@ -36,7 +37,11 @@ func (w *CurrentWorker) Take(ctx context.Context, job sdk.WorkflowNodeJobRun) er w.currentJob.newVariables = nil if info.SigningKey != "" { - signer, err := jws.NewHMacSigner(info.SigningKey) + secretKey := make([]byte, 32) + if _, err := base64.StdEncoding.Decode(secretKey, []byte(info.SigningKey)); err != nil { + return sdk.WithStack(err) + } + signer, err := jws.NewHMacSigner(secretKey) if err != nil { return sdk.WithStack(err) } diff --git a/engine/worker/internal/types.go b/engine/worker/internal/types.go index da04780b35..9cfc00fbd4 100644 --- a/engine/worker/internal/types.go +++ b/engine/worker/internal/types.go @@ -82,7 +82,7 @@ func (wk *CurrentWorker) Parameters() []sdk.Parameter { func (wk *CurrentWorker) SendLog(ctx context.Context, level workerruntime.Level, s string) { jobID, _ := workerruntime.JobID(ctx) stepOrder, err := workerruntime.StepOrder(ctx) - if wk.logger.stepLogger != nil { + if wk.logger.stepLogger == nil { if !strings.HasSuffix(s, "\n") { s += "\n" } diff --git a/sdk/jws/jws.go b/sdk/jws/jws.go index 10df0c705b..77f64c55d2 100644 --- a/sdk/jws/jws.go +++ b/sdk/jws/jws.go @@ -77,8 +77,12 @@ func NewSigner(privateKey *rsa.PrivateKey) (jose.Signer, error) { } // NewSigner instantiate a signer using HMAC using SHA-512 with the given private key. -func NewHMacSigner(secret string) (jose.Signer, error) { - return jose.NewSigner(jose.SigningKey{Algorithm: jose.HS512, Key: secret}, nil) +func NewHMacSigner(secret []byte) (jose.Signer, error) { + sign, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS512, Key: secret}, nil) + if err != nil { + return nil, sdk.WithStack(err) + } + return sign, nil } // Sign a json marshalled content and returns a protected JWS object using the full serialization format. diff --git a/sdk/log/hook/hook.go b/sdk/log/hook/hook.go index 667beec963..d8bd881896 100644 --- a/sdk/log/hook/hook.go +++ b/sdk/log/hook/hook.go @@ -244,7 +244,7 @@ func (hook *Hook) messageFromEntry(entry *logrus.Entry, file string, line int) * // original input. If the input has no newlines, stick the // whole thing in Short. short := p - full := "" + full := p if i := strings.IndexRune(p, '\n'); i > 0 { short = p[:i] full = p @@ -252,7 +252,6 @@ func (hook *Hook) messageFromEntry(entry *logrus.Entry, file string, line int) * // Merge hook extra fields and entry fields extra := hook.merge(hook.Extra, entry.Data) - return &Message{ Version: "1.1", Host: hook.Hostname, diff --git a/sdk/log/log.go b/sdk/log/log.go index efc7b8e629..35696ae04c 100644 --- a/sdk/log/log.go +++ b/sdk/log/log.go @@ -34,8 +34,8 @@ const ( ContextLoggingRequestIDKey = "ctx-logging-request-id" ContextLoggingFuncKey = "ctx-logging-func" - ExtraFieldSignature = "Signature" - ExtraFieldStepOrder = "StepOrder" + ExtraFieldSignature = "_Signature" + ExtraFieldStepOrder = "_StepOrder" ) var ( diff --git a/ui/src/app/views/workflow/run/node/pipeline/step/step.log.component.ts b/ui/src/app/views/workflow/run/node/pipeline/step/step.log.component.ts index 7ddddf86f7..ffae0bee26 100644 --- a/ui/src/app/views/workflow/run/node/pipeline/step/step.log.component.ts +++ b/ui/src/app/views/workflow/run/node/pipeline/step/step.log.component.ts @@ -204,7 +204,6 @@ export class WorkflowStepLogComponent implements OnInit, OnDestroy { } htmlView() { - this.ansiViewSelected = this.ansiViewSelected; this.htmlViewSelected = !this.htmlViewSelected; this.basicView = false; this.splittedLogs = null; @@ -214,7 +213,6 @@ export class WorkflowStepLogComponent implements OnInit, OnDestroy { ansiView() { this.ansiViewSelected = !this.ansiViewSelected; - this.htmlViewSelected = this.htmlViewSelected; this.basicView = false; this.splittedLogs = null; this.parseLogs(); @@ -257,7 +255,7 @@ export class WorkflowStepLogComponent implements OnInit, OnDestroy { this.limitTo = this.splittedLogs.length - 40; this.splittedLogsToDisplay.splice(this.limitFrom, this.limitTo - this.limitFrom); } - + console.log(this.splittedLogsToDisplay); this._cd.markForCheck(); } From 97e5beac2fac78cdb09765618f10692b208649f3 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Tue, 28 Apr 2020 10:25:30 +0200 Subject: [PATCH 05/19] wip --- engine/api/database/gorpmapping/encryption.go | 3 - engine/api/services.go | 7 +- engine/cdn/cdn_log.go | 186 +++++++++++------- engine/cdn/types.go | 8 +- engine/hatchery/kubernetes/services.go | 16 +- engine/hatchery/serve.go | 41 ++++ engine/hatchery/swarm/swarm_util_logs.go | 16 +- engine/service/types.go | 5 + engine/worker/internal/take.go | 15 +- engine/worker/internal/types.go | 9 +- sdk/config.go | 5 + sdk/hatchery/service_log.go | 1 + sdk/jws/jws.go | 6 +- sdk/log/log.go | 37 +++- sdk/services.go | 1 + sdk/worker.go | 6 - 16 files changed, 245 insertions(+), 117 deletions(-) create mode 100644 sdk/hatchery/service_log.go diff --git a/engine/api/database/gorpmapping/encryption.go b/engine/api/database/gorpmapping/encryption.go index 6beb9dda47..d8e12c0c7e 100644 --- a/engine/api/database/gorpmapping/encryption.go +++ b/engine/api/database/gorpmapping/encryption.go @@ -1,10 +1,8 @@ package gorpmapping import ( - "context" "encoding/json" "fmt" - "github.com/ovh/cds/sdk/log" "reflect" "strconv" "strings" @@ -112,7 +110,6 @@ func updateEncryptedData(db gorp.SqlExecutor, i interface{}) error { } encryptedContentArgs = append(encryptedContentArgs, id) query := fmt.Sprintf("UPDATE %s SET %s WHERE %s = $%d", table, strings.Join(updateSlice, ","), key, c) - log.Warning(context.TODO(), ">>>>%s", query) res, err := db.Exec(query, encryptedContentArgs...) if err != nil { return sdk.WithStack(err) diff --git a/engine/api/services.go b/engine/api/services.go index 1de67cec34..64aec19719 100644 --- a/engine/api/services.go +++ b/engine/api/services.go @@ -60,9 +60,9 @@ func (api *API) postServiceRegisterHandler() service.Handler { } if srv != nil && !(srv.Type == data.Type) { return sdk.WrapError(sdk.ErrForbidden, "cannot register service %s of type %s for consumer %s while existing service type is different", data.Name, data.Type, consumer.ID) - } - - // Update or create the service + } + + // Update or create the service if srv != nil { srv.Update(data) if err := services.Update(ctx, tx, srv); err != nil { @@ -87,6 +87,7 @@ func (api *API) postServiceRegisterHandler() service.Handler { } srv.Uptodate = data.Version == sdk.VERSION + srv.LogServer = api.Config.CDN.TCP return service.WriteJSON(w, srv, http.StatusOK) } diff --git a/engine/cdn/cdn_log.go b/engine/cdn/cdn_log.go index 4b8ccb7e71..e11e58a8e5 100644 --- a/engine/cdn/cdn_log.go +++ b/engine/cdn/cdn_log.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/ovh/cds/engine/api/services" "github.com/ovh/cds/engine/api/worker" "github.com/ovh/cds/engine/api/workflow" "github.com/ovh/cds/sdk" @@ -17,7 +18,8 @@ import ( ) var ( - workers = make(map[string]sdk.Worker) + workers = make(map[string]sdk.Worker) + hatcheries = make(map[string][]byte) ) func (s *Service) RunTcpLogServer(ctx context.Context) { @@ -52,95 +54,128 @@ func (s *Service) handleConnection(ctx context.Context, conn net.Conn) { defer func() { _ = conn.Close() }() - - timeoutDuration := 5 * time.Second bufReader := bufio.NewReader(conn) - for { - // Set a deadline for reading. Read operation will fail if no data is received after deadline. - if err := conn.SetReadDeadline(time.Now().Add(timeoutDuration)); err != nil { - log.Error(ctx, "unable to set read deadline on connection") - return - } bytes, err := bufReader.ReadBytes(byte(0)) if err != nil { log.Info(ctx, "client left") return } - if len(bytes) == 0 { - continue - } - if bytes[len(bytes)-1] == byte(0) { - bytes = bytes[:len(bytes)-1] - } - m := hook.Message{} - if err := (&m).UnmarshalJSON(bytes); err != nil { - log.Error(ctx, "cdn.log > unable to unmarshal log message: %s %v", string(bytes), err) - continue - } + // remove byte(0) + bytes = bytes[:len(bytes)-1] - sig, ok := m.Extra[log.ExtraFieldSignature] - if !ok || sig == "" { - log.Error(ctx, "cdn.log > signature not found on log message %+v", m) + if err := s.handleLogMessage(ctx, bytes); err != nil { + log.Error(ctx, "cdn.log> :%v", err) continue } + } +} - stepOrderI, ok := m.Extra[log.ExtraFieldStepOrder] - if !ok { - log.Error(ctx, "cdn.log > missing step order extra field") - continue - } - stepOrder := int64(stepOrderI.(float64)) +func (s *Service) handleLogMessage(ctx context.Context, messageReceived []byte) error { + m := hook.Message{} + if err := (&m).UnmarshalJSON(messageReceived); err != nil { + return sdk.WrapError(err, "unable to unmarshall gelf message: %s", string(messageReceived)) + } - // Get worker datas - var workerSign sdk.WorkerSignature - if err := jws.UnsafeParse(sig.(string), &workerSign); err != nil { - log.Error(ctx, "cdn.log > unable to unsafe parse log signature: %v", err) - continue - } - workerData, ok := workers[workerSign.WorkerID] - if !ok { - var err error - workerData, err = s.getWorker(ctx, workerSign.WorkerID) - if err != nil { - log.Error(ctx, "cdn.log > unable to retrieve worker data from api: %v", err) - continue - } - } - if err := jws.Verify(workerData.PrivateKey, sig.(string), &workerSign); err != nil { - log.Error(ctx, "cdn.log > unable to verify signature: %v", err) - continue - } + sig, ok := m.Extra[log.ExtraFieldSignature] + if !ok || sig == "" { + return sdk.WithStack(fmt.Errorf("signature not found on log message")) + } - pbJob, err := workflow.LoadNodeJobRun(ctx, s.Db, s.Cache, workerSign.JobID) + // Get worker datas + var signature log.Signature + if err := jws.UnsafeParse(sig.(string), &signature); err != nil { + return err + } + + switch { + case signature.Worker != nil: + return s.handleWorkerLog(ctx, signature.Worker.WorkerID, sig, m) + case signature.Service != nil: + return s.handleServiceLog(ctx, signature.Service.HatcheryName, sig, m) + default: + return sdk.WithStack(sdk.ErrWrongRequest) + } +} + +func (s *Service) handleWorkerLog(ctx context.Context, workerID string, sig interface{}, m hook.Message) error { + var signature log.Signature + workerData, ok := workers[workerID] + if !ok { + var err error + workerData, err = s.getWorker(ctx, workerID) if err != nil { - log.Error(ctx, "cdn.log > unable to verify signature") - continue + return err } + } + if err := jws.Verify(workerData.PrivateKey, sig.(string), &signature); err != nil { + return err + } + if workerData.JobRunID == nil || *workerData.JobRunID != signature.JobID { + return sdk.WithStack(sdk.ErrForbidden) + } - logDate := time.Unix(0, int64(m.Time*1e9)) - logs := sdk.Log{ - JobID: pbJob.ID, - LastModified: &logDate, - NodeRunID: pbJob.WorkflowNodeRunID, - Start: &logDate, - StepOrder: stepOrder, - Val: m.Full, - } - tx, err := s.Db.Begin() - if err != nil { + pbJob, err := workflow.LoadNodeJobRun(ctx, s.Db, s.Cache, signature.JobID) + if err != nil { + return err + } + logDate := time.Unix(0, int64(m.Time*1e9)) + logs := sdk.Log{ + JobID: pbJob.ID, + LastModified: &logDate, + NodeRunID: pbJob.WorkflowNodeRunID, + Start: &logDate, + StepOrder: signature.Worker.StepOrder, + Val: m.Full, + } + if !strings.HasSuffix(logs.Val, "\n") { + logs.Val += "\n" + } + + tx, err := s.Db.Begin() + if err != nil { + return sdk.WithStack(err) + } + defer tx.Rollback() // nolint + + if err := workflow.AddLog(tx, pbJob, &logs, s.Cfg.Log.StepMaxSize); err != nil { + return err + } + return sdk.WithStack(tx.Commit()) +} + +func (s *Service) handleServiceLog(ctx context.Context, hatcheryName string, sig interface{}, m hook.Message) error { + var signature log.Signature + hatcheryPublicKey, ok := hatcheries[hatcheryName] + if !ok { + var err error + hatcheryPublicKey, err = s.getHatcheryPublicKey(ctx, hatcheryName) + if err != nil { + return err } - if !strings.HasSuffix(logs.Val, "\n") { - logs.Val += "\n" - } - if err := workflow.AddLog(tx, pbJob, &logs, s.Cfg.Log.StepMaxSize); err != nil { - log.Error(ctx, "cdn.log > unable to insert log") - _ = tx.Rollback() - continue - } - _ = tx.Commit() } + if err := jws.Verify(hatcheryPublicKey, sig.(string), &signature); err != nil { + return err + } + + nodeRunJob, err := workflow.LoadNodeJobRun(ctx, s.Db, s.Cache, signature.JobID) + if err != nil { + return err + } + + logs := sdk.ServiceLog{ + ServiceRequirementName: signature.Service.RequirementName, + ServiceRequirementID: signature.Service.RequirementID, + WorkflowNodeJobRunID: signature.JobID, + WorkflowNodeRunID: nodeRunJob.WorkflowNodeRunID, + Val: m.Full, + } + + if err := workflow.AddServiceLog(s.Db, nodeRunJob, &logs, s.Cfg.Log.ServiceMaxSize); err != nil { + return err + } + return nil } func (s *Service) getWorker(ctx context.Context, workerID string) (sdk.Worker, error) { @@ -151,3 +186,12 @@ func (s *Service) getWorker(ctx context.Context, workerID string) (sdk.Worker, e workers[w.ID] = w return w, nil } + +func (s *Service) getHatcheryPublicKey(ctx context.Context, hatcheryName string) ([]byte, error) { + h, err := services.LoadByNameAndType(ctx, s.Db, hatcheryName, services.TypeHatchery) + if err != nil { + return nil, err + } + hatcheries[hatcheryName] = h.PublicKey + return h.PublicKey, nil +} diff --git a/engine/cdn/types.go b/engine/cdn/types.go index 0cb918e1e8..cf805f9e27 100644 --- a/engine/cdn/types.go +++ b/engine/cdn/types.go @@ -4,6 +4,7 @@ import ( "github.com/go-gorp/gorp" "github.com/ovh/cds/engine/api/cache" "github.com/ovh/cds/engine/service" + "github.com/ovh/cds/sdk" ) // Service is the stuct representing a hooks µService @@ -17,11 +18,8 @@ type Service struct { // Configuration is the hooks configuration structure type Configuration struct { - Name string `toml:"name" default:"cds-cdn" comment:"Name of this CDS CDN Service\n Enter a name to enable this service" json:"name"` - TCP struct { - Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"` - Port int `toml:"port" default:"8089" json:"port"` - } `toml:"tcp" comment:"######################\n CDS CDN TCP Configuration \n######################" json:"tcp"` + Name string `toml:"name" default:"cds-cdn" comment:"Name of this CDS CDN Service\n Enter a name to enable this service" json:"name"` + TCP sdk.TCPServer `toml:"tcp" comment:"######################\n CDS CDN TCP Configuration \n######################" json:"tcp"` HTTP struct { Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"` Port int `toml:"port" default:"8089" json:"port"` diff --git a/engine/hatchery/kubernetes/services.go b/engine/hatchery/kubernetes/services.go index e340a7392e..f62dcdd4b0 100644 --- a/engine/hatchery/kubernetes/services.go +++ b/engine/hatchery/kubernetes/services.go @@ -14,6 +14,10 @@ import ( ) func (h *HatcheryKubernetes) getServicesLogs(ctx context.Context) error { + if err := h.Common.InitServiceLogger(); err != nil { + return err + } + pods, err := h.k8sClient.CoreV1().Pods(h.Config.Namespace).List(metav1.ListOptions{LabelSelector: LABEL_SERVICE_JOB_ID}) if err != nil { return err @@ -64,13 +68,15 @@ func (h *HatcheryKubernetes) getServicesLogs(ctx context.Context) error { } if len(servicesLogs) > 0 { - // Do call api ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - if err := h.Client.QueueServiceLogs(ctx, servicesLogs); err != nil { - cancel() - return fmt.Errorf("Hatchery> Swarm> Cannot send service logs : %v", err) + defer cancel() + if h.Common.ServiceLogger == nil { + if err := h.Client.QueueServiceLogs(ctx, servicesLogs); err != nil { + return sdk.WithStack(fmt.Errorf("hatchery> Swarm> Cannot send service logs : %v", err)) + } + } else { + h.Common.SendServiceLog(ctx, servicesLogs) } - cancel() } return nil diff --git a/engine/hatchery/serve.go b/engine/hatchery/serve.go index 3577d279e3..029f96b33a 100644 --- a/engine/hatchery/serve.go +++ b/engine/hatchery/serve.go @@ -13,12 +13,15 @@ import ( "time" "github.com/gorilla/mux" + "github.com/sirupsen/logrus" + "gopkg.in/square/go-jose.v2" "github.com/ovh/cds/engine/api" "github.com/ovh/cds/engine/service" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/cdsclient" "github.com/ovh/cds/sdk/hatchery" + "github.com/ovh/cds/sdk/jws" "github.com/ovh/cds/sdk/log" ) @@ -187,6 +190,44 @@ func (c *Common) getPanicDumpListHandler() service.Handler { } } +func (c *Common) InitServiceLogger() error { + tcpServer := c.Common.ServiceInstance.LogServer + var signer jose.Signer + if tcpServer.Addr != "" && tcpServer.Port != 0 { + logger, err := log.New(fmt.Sprintf("%s:%d", tcpServer.Addr, tcpServer.Port)) + if err != nil { + return err + } + signer, err = jws.NewSigner(c.Common.PrivateKey) + if err != nil { + return sdk.WithStack(err) + } + c.Signer = signer + c.ServiceLogger = logger + } + return nil +} + +func (c *Common) SendServiceLog(ctx context.Context, servicesLogs []sdk.ServiceLog) { + for _, s := range servicesLogs { + dataToSign := log.Signature{ + Service: &log.SignatureService{ + HatcheryName: c.ServiceName(), + RequirementID: s.ServiceRequirementID, + RequirementName: s.ServiceRequirementName, + }, + JobID: s.WorkflowNodeJobRunID, + Timestamp: time.Now().UnixNano(), + } + signature, err := jws.Sign(c.Signer, dataToSign) + if err != nil { + log.Error(ctx, "SendServiceLog> unable to sign service log message: %v", err) + continue + } + c.ServiceLogger.WithField("Signature", signature).Log(logrus.InfoLevel, s) + } +} + func (c *Common) getPanicDumpHandler() service.Handler { return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error { vars := mux.Vars(r) diff --git a/engine/hatchery/swarm/swarm_util_logs.go b/engine/hatchery/swarm/swarm_util_logs.go index 4b89064053..2bb9d71fb3 100644 --- a/engine/hatchery/swarm/swarm_util_logs.go +++ b/engine/hatchery/swarm/swarm_util_logs.go @@ -7,11 +7,16 @@ import ( "time" "github.com/docker/docker/api/types" + "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/log" ) func (h *HatcherySwarm) getServicesLogs() error { + if err := h.Common.InitServiceLogger(); err != nil { + return err + } + for _, dockerClient := range h.dockerClients { containers, err := h.getContainers(dockerClient, types.ContainerListOptions{All: true}) if err != nil { @@ -74,15 +79,18 @@ func (h *HatcherySwarm) getServicesLogs() error { Val: string(logs), }) } - - if len(servicesLogs) > 0 { + } + if len(servicesLogs) > 0 { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if h.Common.ServiceLogger == nil { // Do call api - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) if err := h.Client.QueueServiceLogs(ctx, servicesLogs); err != nil { log.Error(ctx, "Hatchery> Swarm> Cannot send service logs : %v", err) } - cancel() + } else { + h.Common.SendServiceLog(ctx, servicesLogs) } + cancel() } } return nil diff --git a/engine/service/types.go b/engine/service/types.go index 2c9b16b44a..bcb61ae011 100644 --- a/engine/service/types.go +++ b/engine/service/types.go @@ -6,6 +6,9 @@ import ( "fmt" "time" + "github.com/sirupsen/logrus" + "gopkg.in/square/go-jose.v2" + "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/cdsclient" ) @@ -102,6 +105,8 @@ type Common struct { ServiceType string ServiceInstance *sdk.Service PrivateKey *rsa.PrivateKey + Signer jose.Signer + ServiceLogger *logrus.Logger } // Service is the interface for a engine service diff --git a/engine/worker/internal/take.go b/engine/worker/internal/take.go index 5204fd89e7..2c4e422048 100644 --- a/engine/worker/internal/take.go +++ b/engine/worker/internal/take.go @@ -6,13 +6,10 @@ import ( "strings" "time" - "github.com/sirupsen/logrus" - "github.com/ovh/cds/engine/worker/pkg/workerruntime" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/jws" "github.com/ovh/cds/sdk/log" - loghook "github.com/ovh/cds/sdk/log/hook" ) func (w *CurrentWorker) Take(ctx context.Context, job sdk.WorkflowNodeJobRun) error { @@ -50,17 +47,11 @@ func (w *CurrentWorker) Take(ctx context.Context, job sdk.WorkflowNodeJobRun) er if info.GelfServiceAddr != "" { log.Info(ctx, "Setup step logger") - w.logger.stepLogger = logrus.New() - graylogcfg := &loghook.Config{ - Addr: info.GelfServiceAddr, - Protocol: "tcp", - } - extra := map[string]interface{}{} - hook, err := loghook.NewHook(graylogcfg, extra) + logger, err := log.New(info.GelfServiceAddr) if err != nil { - return sdk.WithStack(err) + return err } - w.logger.stepLogger.AddHook(hook) + w.logger.stepLogger = logger } start := time.Now() diff --git a/engine/worker/internal/types.go b/engine/worker/internal/types.go index 9cfc00fbd4..0b5a840e95 100644 --- a/engine/worker/internal/types.go +++ b/engine/worker/internal/types.go @@ -107,8 +107,11 @@ func (wk *CurrentWorker) SendLog(ctx context.Context, level workerruntime.Level, default: } - dataToSign := sdk.WorkerSignature{ - WorkerID: wk.id, + dataToSign := log.Signature{ + Worker: &log.SignatureWorker{ + WorkerID: wk.id, + StepOrder: int64(stepOrder), + }, JobID: wk.currentJob.wJob.ID, Timestamp: time.Now().UnixNano(), } @@ -116,7 +119,7 @@ func (wk *CurrentWorker) SendLog(ctx context.Context, level workerruntime.Level, if err != nil { log.Error(ctx, "unable to sign logs: %v", err) } - wk.logger.stepLogger.WithField("Signature", signature).WithField("StepOrder", stepOrder).Log(logLevel, s) + wk.logger.stepLogger.WithField(log.ExtraFieldSignature, signature).Log(logLevel, s) } diff --git a/sdk/config.go b/sdk/config.go index 17d0ff8637..e5e698f901 100644 --- a/sdk/config.go +++ b/sdk/config.go @@ -10,3 +10,8 @@ type ConfigUser struct { URLUI string `json:"url.ui"` URLAPI string `json:"url.api"` } + +type TCPServer struct { + Addr string `toml:"addr" default:"" commented:"true" comment:"Listen address without port, example: 127.0.0.1" json:"addr"` + Port int `toml:"port" default:"8089" json:"port"` +} diff --git a/sdk/hatchery/service_log.go b/sdk/hatchery/service_log.go new file mode 100644 index 0000000000..7c06ac419e --- /dev/null +++ b/sdk/hatchery/service_log.go @@ -0,0 +1 @@ +package hatchery diff --git a/sdk/jws/jws.go b/sdk/jws/jws.go index 77f64c55d2..06d515f1da 100644 --- a/sdk/jws/jws.go +++ b/sdk/jws/jws.go @@ -113,14 +113,14 @@ func Verify(key interface{}, s string, i interface{}) error { if err != nil { return sdk.WithStack(err) } - return json.Unmarshal(output, i) + return sdk.WithStack(json.Unmarshal(output, i)) } func UnsafeParse(s string, i interface{}) error { object, err := jose.ParseSigned(s) if err != nil { - return err + return sdk.WithStack(err) } output := object.UnsafePayloadWithoutVerification() - return json.Unmarshal(output, i) + return sdk.WithStack(json.Unmarshal(output, i)) } diff --git a/sdk/log/log.go b/sdk/log/log.go index 35696ae04c..a8a3e291c4 100644 --- a/sdk/log/log.go +++ b/sdk/log/log.go @@ -8,9 +8,9 @@ import ( "os" "strings" - log "github.com/sirupsen/logrus" - + "github.com/ovh/cds/sdk" loghook "github.com/ovh/cds/sdk/log/hook" + log "github.com/sirupsen/logrus" ) // Conf contains log configuration @@ -242,3 +242,36 @@ func newEntry(ctx context.Context, fields log.Fields) *log.Entry { return entry } + +type Signature struct { + Worker *SignatureWorker + Service *SignatureService + JobID int64 + Timestamp int64 +} + +type SignatureWorker struct { + WorkerID string + StepOrder int64 +} + +type SignatureService struct { + HatcheryName string + RequirementID int64 + RequirementName string +} + +func New(logServerAddr string) (*log.Logger, error) { + newLogger := log.New() + graylogcfg := &loghook.Config{ + Addr: logServerAddr, + Protocol: "tcp", + } + extra := map[string]interface{}{} + hook, err := loghook.NewHook(graylogcfg, extra) + if err != nil { + return nil, sdk.WithStack(err) + } + newLogger.AddHook(hook) + return newLogger, nil +} diff --git a/sdk/services.go b/sdk/services.go index 272ca393d1..31402cdc61 100644 --- a/sdk/services.go +++ b/sdk/services.go @@ -24,6 +24,7 @@ type Service struct { MonitoringStatus MonitoringStatus `json:"monitoring_status" db:"monitoring_status" cli:"-"` Version string `json:"version" db:"-" cli:"version"` Uptodate bool `json:"up_to_date" db:"-"` + LogServer TCPServer `json:"tcp" db:"-"` } // Update service field from new data. diff --git a/sdk/worker.go b/sdk/worker.go index bdb8255a5a..7342a07ba2 100644 --- a/sdk/worker.go +++ b/sdk/worker.go @@ -23,12 +23,6 @@ type Worker struct { PrivateKey []byte `json:"-" cli:"-" db:"cypher_private_key" gorpmapping:"encrypted,ID,Name"` } -type WorkerSignature struct { - WorkerID string - JobID int64 - Timestamp int64 -} - // WorkerRegistrationForm represents the arguments needed to register a worker type WorkerRegistrationForm struct { BinaryCapabilities []string From 153e57e03a0038c43466da21942e3f74580fb485 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Tue, 28 Apr 2020 10:42:26 +0200 Subject: [PATCH 06/19] fix --- engine/hatchery/serve.go | 2 +- engine/worker/internal/take.go | 2 +- sdk/log/log.go | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/engine/hatchery/serve.go b/engine/hatchery/serve.go index 029f96b33a..4e6407e313 100644 --- a/engine/hatchery/serve.go +++ b/engine/hatchery/serve.go @@ -196,7 +196,7 @@ func (c *Common) InitServiceLogger() error { if tcpServer.Addr != "" && tcpServer.Port != 0 { logger, err := log.New(fmt.Sprintf("%s:%d", tcpServer.Addr, tcpServer.Port)) if err != nil { - return err + return sdk.WithStack(err) } signer, err = jws.NewSigner(c.Common.PrivateKey) if err != nil { diff --git a/engine/worker/internal/take.go b/engine/worker/internal/take.go index 2c4e422048..9e110e5b3e 100644 --- a/engine/worker/internal/take.go +++ b/engine/worker/internal/take.go @@ -49,7 +49,7 @@ func (w *CurrentWorker) Take(ctx context.Context, job sdk.WorkflowNodeJobRun) er log.Info(ctx, "Setup step logger") logger, err := log.New(info.GelfServiceAddr) if err != nil { - return err + return sdk.WithStack(err) } w.logger.stepLogger = logger } diff --git a/sdk/log/log.go b/sdk/log/log.go index a8a3e291c4..c97a9d54e8 100644 --- a/sdk/log/log.go +++ b/sdk/log/log.go @@ -8,7 +8,6 @@ import ( "os" "strings" - "github.com/ovh/cds/sdk" loghook "github.com/ovh/cds/sdk/log/hook" log "github.com/sirupsen/logrus" ) @@ -270,7 +269,7 @@ func New(logServerAddr string) (*log.Logger, error) { extra := map[string]interface{}{} hook, err := loghook.NewHook(graylogcfg, extra) if err != nil { - return nil, sdk.WithStack(err) + return nil, fmt.Errorf("unable to add hook: %v", err) } newLogger.AddHook(hook) return newLogger, nil From cccc53948c7a275d37088c84480a63d22f07bab7 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Tue, 28 Apr 2020 14:14:29 +0200 Subject: [PATCH 07/19] fix: hatchery plublic key --- engine/cdn/cdn_log.go | 17 +++++++++++------ engine/hatchery/swarm/swarm.go | 3 +++ engine/hatchery/swarm/swarm_util_logs.go | 4 ---- sdk/log/log.go | 3 +-- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/engine/cdn/cdn_log.go b/engine/cdn/cdn_log.go index e11e58a8e5..5e7bf28b57 100644 --- a/engine/cdn/cdn_log.go +++ b/engine/cdn/cdn_log.go @@ -3,6 +3,7 @@ package cdn import ( "bufio" "context" + "crypto/rsa" "fmt" "net" "strings" @@ -19,7 +20,7 @@ import ( var ( workers = make(map[string]sdk.Worker) - hatcheries = make(map[string][]byte) + hatcheries = make(map[string]*rsa.PublicKey) ) func (s *Service) RunTcpLogServer(ctx context.Context) { @@ -77,9 +78,9 @@ func (s *Service) handleLogMessage(ctx context.Context, messageReceived []byte) return sdk.WrapError(err, "unable to unmarshall gelf message: %s", string(messageReceived)) } - sig, ok := m.Extra[log.ExtraFieldSignature] + sig, ok := m.Extra["_"+log.ExtraFieldSignature] if !ok || sig == "" { - return sdk.WithStack(fmt.Errorf("signature not found on log message")) + return sdk.WithStack(fmt.Errorf("signature not found on log message: %+v", m)) } // Get worker datas @@ -187,11 +188,15 @@ func (s *Service) getWorker(ctx context.Context, workerID string) (sdk.Worker, e return w, nil } -func (s *Service) getHatcheryPublicKey(ctx context.Context, hatcheryName string) ([]byte, error) { +func (s *Service) getHatcheryPublicKey(ctx context.Context, hatcheryName string) (*rsa.PublicKey, error) { h, err := services.LoadByNameAndType(ctx, s.Db, hatcheryName, services.TypeHatchery) if err != nil { return nil, err } - hatcheries[hatcheryName] = h.PublicKey - return h.PublicKey, nil + publicKey, err := jws.NewPublicKeyFromPEM(h.PublicKey) + if err != nil { + return nil, sdk.WithStack(err) + } + hatcheries[hatcheryName] = publicKey + return publicKey, nil } diff --git a/engine/hatchery/swarm/swarm.go b/engine/hatchery/swarm/swarm.go index f94b1cfb37..2a2f8b55a9 100644 --- a/engine/hatchery/swarm/swarm.go +++ b/engine/hatchery/swarm/swarm.go @@ -172,6 +172,9 @@ func (h *HatcherySwarm) InitHatchery(ctx context.Context) error { return fmt.Errorf("no docker engine available") } } + if err := h.Common.InitServiceLogger(); err != nil { + return err + } sdk.GoRoutine(context.Background(), "swarm", func(ctx context.Context) { h.routines(ctx) }) diff --git a/engine/hatchery/swarm/swarm_util_logs.go b/engine/hatchery/swarm/swarm_util_logs.go index 2bb9d71fb3..ca34e4a001 100644 --- a/engine/hatchery/swarm/swarm_util_logs.go +++ b/engine/hatchery/swarm/swarm_util_logs.go @@ -13,10 +13,6 @@ import ( ) func (h *HatcherySwarm) getServicesLogs() error { - if err := h.Common.InitServiceLogger(); err != nil { - return err - } - for _, dockerClient := range h.dockerClients { containers, err := h.getContainers(dockerClient, types.ContainerListOptions{All: true}) if err != nil { diff --git a/sdk/log/log.go b/sdk/log/log.go index c97a9d54e8..22b6a44a83 100644 --- a/sdk/log/log.go +++ b/sdk/log/log.go @@ -33,8 +33,7 @@ const ( ContextLoggingRequestIDKey = "ctx-logging-request-id" ContextLoggingFuncKey = "ctx-logging-func" - ExtraFieldSignature = "_Signature" - ExtraFieldStepOrder = "_StepOrder" + ExtraFieldSignature = "Signature" ) var ( From 6fdc96bb4b55b4454a30e0ba407ac843e36f7f3a Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Tue, 28 Apr 2020 15:23:57 +0200 Subject: [PATCH 08/19] wip --- engine/api/worker/dao.go | 18 +++++++- engine/cdn/cdn_log.go | 57 ++++++++++++++++++------ engine/hatchery/serve.go | 1 + engine/hatchery/swarm/swarm_util_logs.go | 3 ++ sdk/log.go | 2 + sdk/log/log.go | 2 + 6 files changed, 67 insertions(+), 16 deletions(-) diff --git a/engine/api/worker/dao.go b/engine/api/worker/dao.go index 1dc820a33f..755375f0ea 100644 --- a/engine/api/worker/dao.go +++ b/engine/api/worker/dao.go @@ -144,8 +144,8 @@ func SetToBuilding(ctx context.Context, db gorp.SqlExecutor, workerID string, jo return err } -// LoadWorkerWithDecryptKey load worker with decrypted private key -func LoadWorkerWithDecryptKey(ctx context.Context, db gorp.SqlExecutor, workerID string) (sdk.Worker, error) { +// LoadWorkerByIDWithDecryptKey load worker with decrypted private key +func LoadWorkerByIDWithDecryptKey(ctx context.Context, db gorp.SqlExecutor, workerID string) (sdk.Worker, error) { var work dbWorker query := gorpmapping.NewQuery(`SELECT * FROM worker WHERE id = $1`).Args(workerID) found, err := gorpmapping.Get(ctx, db, query, &work, gorpmapping.GetOptions.WithDecryption) @@ -165,3 +165,17 @@ func LoadWorkerWithDecryptKey(ctx context.Context, db gorp.SqlExecutor, workerID log.Warning(ctx, "My WORKER: %+v", work.Worker) return work.Worker, err } + +// LoadWorkerByName load worker by name +func LoadWorkerByName(ctx context.Context, db gorp.SqlExecutor, workerName string) (sdk.Worker, error) { + var work dbWorker + query := gorpmapping.NewQuery(`SELECT * FROM worker WHERE name = $1`).Args(workerName) + found, err := gorpmapping.Get(ctx, db, query, &work) + if err != nil { + return sdk.Worker{}, err + } + if !found { + return sdk.Worker{}, sdk.ErrNotFound + } + return work.Worker, err +} diff --git a/engine/cdn/cdn_log.go b/engine/cdn/cdn_log.go index 5e7bf28b57..8a97f051d9 100644 --- a/engine/cdn/cdn_log.go +++ b/engine/cdn/cdn_log.go @@ -9,6 +9,8 @@ import ( "strings" "time" + gocache "github.com/patrickmn/go-cache" + "github.com/ovh/cds/engine/api/services" "github.com/ovh/cds/engine/api/worker" "github.com/ovh/cds/engine/api/workflow" @@ -19,8 +21,7 @@ import ( ) var ( - workers = make(map[string]sdk.Worker) - hatcheries = make(map[string]*rsa.PublicKey) + logCache = gocache.New(20*time.Minute, 30*time.Minute) ) func (s *Service) RunTcpLogServer(ctx context.Context) { @@ -93,7 +94,7 @@ func (s *Service) handleLogMessage(ctx context.Context, messageReceived []byte) case signature.Worker != nil: return s.handleWorkerLog(ctx, signature.Worker.WorkerID, sig, m) case signature.Service != nil: - return s.handleServiceLog(ctx, signature.Service.HatcheryName, sig, m) + return s.handleServiceLog(ctx, signature.Service.HatcheryID, signature.Service.HatcheryName, signature.Service.WorkerName, sig, m) default: return sdk.WithStack(sdk.ErrWrongRequest) } @@ -101,13 +102,16 @@ func (s *Service) handleLogMessage(ctx context.Context, messageReceived []byte) func (s *Service) handleWorkerLog(ctx context.Context, workerID string, sig interface{}, m hook.Message) error { var signature log.Signature - workerData, ok := workers[workerID] + var workerData sdk.Worker + cacheData, ok := logCache.Get(fmt.Sprintf("worker-%s", workerID)) if !ok { var err error workerData, err = s.getWorker(ctx, workerID) if err != nil { return err } + } else { + workerData = cacheData.(sdk.Worker) } if err := jws.Verify(workerData.PrivateKey, sig.(string), &signature); err != nil { return err @@ -146,20 +150,40 @@ func (s *Service) handleWorkerLog(ctx context.Context, workerID string, sig inte return sdk.WithStack(tx.Commit()) } -func (s *Service) handleServiceLog(ctx context.Context, hatcheryName string, sig interface{}, m hook.Message) error { +func (s *Service) handleServiceLog(ctx context.Context, hatcheryID int64, hatcheryName string, workerName string, sig interface{}, m hook.Message) error { var signature log.Signature - hatcheryPublicKey, ok := hatcheries[hatcheryName] + + var pk *rsa.PublicKey + cacheData, ok := logCache.Get(fmt.Sprintf("hatchery-key-%d", hatcheryID)) if !ok { var err error - hatcheryPublicKey, err = s.getHatcheryPublicKey(ctx, hatcheryName) + pk, err = s.getHatchery(ctx, hatcheryID, hatcheryName) if err != nil { return err } + } else { + pk = cacheData.(*rsa.PublicKey) } - if err := jws.Verify(hatcheryPublicKey, sig.(string), &signature); err != nil { + + if err := jws.Verify(pk, sig.(string), &signature); err != nil { return err } + // Verified that worker has been spawn by this hatchery + workerCacheKey := fmt.Sprintf("service-worker-%s", workerName) + _, ok = logCache.Get(workerCacheKey) + if !ok { + // Verify that the worker has been spawn by this hatchery + w, err := worker.LoadWorkerByName(ctx, s.Db, workerName) + if err != nil { + return err + } + if w.HatcheryID != signature.Service.HatcheryID { + return sdk.WrapError(sdk.ErrWrongRequest, "hatchery and worker does not match") + } + logCache.Set(workerCacheKey, true, gocache.DefaultExpiration) + } + nodeRunJob, err := workflow.LoadNodeJobRun(ctx, s.Db, s.Cache, signature.JobID) if err != nil { return err @@ -180,23 +204,28 @@ func (s *Service) handleServiceLog(ctx context.Context, hatcheryName string, sig } func (s *Service) getWorker(ctx context.Context, workerID string) (sdk.Worker, error) { - w, err := worker.LoadWorkerWithDecryptKey(ctx, s.Db, workerID) + w, err := worker.LoadWorkerByIDWithDecryptKey(ctx, s.Db, workerID) if err != nil { return sdk.Worker{}, err } - workers[w.ID] = w + logCache.Set(fmt.Sprintf("worker-%s", w.ID), w, gocache.DefaultExpiration) return w, nil } -func (s *Service) getHatcheryPublicKey(ctx context.Context, hatcheryName string) (*rsa.PublicKey, error) { +func (s *Service) getHatchery(ctx context.Context, hatcheryID int64, hatcheryName string) (*rsa.PublicKey, error) { h, err := services.LoadByNameAndType(ctx, s.Db, hatcheryName, services.TypeHatchery) if err != nil { return nil, err } - publicKey, err := jws.NewPublicKeyFromPEM(h.PublicKey) + if h.ID != hatcheryID { + return nil, sdk.WithStack(sdk.ErrWrongRequest) + } + + // Verify signature + pk, err := jws.NewPublicKeyFromPEM(h.PublicKey) if err != nil { return nil, sdk.WithStack(err) } - hatcheries[hatcheryName] = publicKey - return publicKey, nil + logCache.Set(fmt.Sprintf("hatchery-key-%d", hatcheryID), pk, gocache.DefaultExpiration) + return pk, nil } diff --git a/engine/hatchery/serve.go b/engine/hatchery/serve.go index 4e6407e313..b86dc1e4bb 100644 --- a/engine/hatchery/serve.go +++ b/engine/hatchery/serve.go @@ -215,6 +215,7 @@ func (c *Common) SendServiceLog(ctx context.Context, servicesLogs []sdk.ServiceL HatcheryName: c.ServiceName(), RequirementID: s.ServiceRequirementID, RequirementName: s.ServiceRequirementName, + WorkerName: s.WorkerName, }, JobID: s.WorkflowNodeJobRunID, Timestamp: time.Now().UnixNano(), diff --git a/engine/hatchery/swarm/swarm_util_logs.go b/engine/hatchery/swarm/swarm_util_logs.go index ca34e4a001..bf664371df 100644 --- a/engine/hatchery/swarm/swarm_util_logs.go +++ b/engine/hatchery/swarm/swarm_util_logs.go @@ -25,6 +25,7 @@ func (h *HatcherySwarm) getServicesLogs() error { if !isWorkflowService { continue } + workerName := cnt.Labels["service_worker"] ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) logsOpts := types.ContainerLogsOptions{ Details: true, @@ -73,6 +74,8 @@ func (h *HatcherySwarm) getServicesLogs() error { ServiceRequirementID: reqServiceID, ServiceRequirementName: cnt.Labels["service_req_name"], Val: string(logs), + WorkerName: workerName, + HatcheryID: h.Service().ID, }) } } diff --git a/sdk/log.go b/sdk/log.go index 82c56db508..40bcae25f6 100644 --- a/sdk/log.go +++ b/sdk/log.go @@ -40,4 +40,6 @@ type ServiceLog struct { ServiceRequirementID int64 `json:"requirement_id" db:"-"` ServiceRequirementName string `json:"requirement_service_name" db:"requirement_service_name"` Val string `json:"val,omitempty" db:"value"` + WorkerName string `json:"worker_name"` + HatcheryID int64 `json:"hatchery_id"` } diff --git a/sdk/log/log.go b/sdk/log/log.go index 22b6a44a83..617830653e 100644 --- a/sdk/log/log.go +++ b/sdk/log/log.go @@ -254,9 +254,11 @@ type SignatureWorker struct { } type SignatureService struct { + HatcheryID int64 HatcheryName string RequirementID int64 RequirementName string + WorkerName string } func New(logServerAddr string) (*log.Logger, error) { From 41092ed79b21fc70d2190400f5339c49274846a9 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Tue, 28 Apr 2020 16:14:44 +0200 Subject: [PATCH 09/19] fix: check hatchery --- engine/cdn/cdn_log.go | 3 ++- engine/hatchery/kubernetes/services.go | 1 + engine/hatchery/serve.go | 1 + engine/hatchery/swarm/swarm_util_logs.go | 1 - sdk/log.go | 1 - 5 files changed, 4 insertions(+), 3 deletions(-) diff --git a/engine/cdn/cdn_log.go b/engine/cdn/cdn_log.go index 8a97f051d9..f01c0aebaf 100644 --- a/engine/cdn/cdn_log.go +++ b/engine/cdn/cdn_log.go @@ -67,7 +67,7 @@ func (s *Service) handleConnection(ctx context.Context, conn net.Conn) { bytes = bytes[:len(bytes)-1] if err := s.handleLogMessage(ctx, bytes); err != nil { - log.Error(ctx, "cdn.log> :%v", err) + log.Error(ctx, "cdn.log> %v", err) continue } } @@ -217,6 +217,7 @@ func (s *Service) getHatchery(ctx context.Context, hatcheryID int64, hatcheryNam if err != nil { return nil, err } + if h.ID != hatcheryID { return nil, sdk.WithStack(sdk.ErrWrongRequest) } diff --git a/engine/hatchery/kubernetes/services.go b/engine/hatchery/kubernetes/services.go index f62dcdd4b0..882e12743d 100644 --- a/engine/hatchery/kubernetes/services.go +++ b/engine/hatchery/kubernetes/services.go @@ -63,6 +63,7 @@ func (h *HatcheryKubernetes) getServicesLogs(ctx context.Context) error { ServiceRequirementID: reqServiceID, ServiceRequirementName: subsStr[0][2], Val: string(logs), + WorkerName: pod.ObjectMeta.Name, }) } } diff --git a/engine/hatchery/serve.go b/engine/hatchery/serve.go index b86dc1e4bb..906d029023 100644 --- a/engine/hatchery/serve.go +++ b/engine/hatchery/serve.go @@ -212,6 +212,7 @@ func (c *Common) SendServiceLog(ctx context.Context, servicesLogs []sdk.ServiceL for _, s := range servicesLogs { dataToSign := log.Signature{ Service: &log.SignatureService{ + HatcheryID: c.Service().ID, HatcheryName: c.ServiceName(), RequirementID: s.ServiceRequirementID, RequirementName: s.ServiceRequirementName, diff --git a/engine/hatchery/swarm/swarm_util_logs.go b/engine/hatchery/swarm/swarm_util_logs.go index bf664371df..12389cfef3 100644 --- a/engine/hatchery/swarm/swarm_util_logs.go +++ b/engine/hatchery/swarm/swarm_util_logs.go @@ -75,7 +75,6 @@ func (h *HatcherySwarm) getServicesLogs() error { ServiceRequirementName: cnt.Labels["service_req_name"], Val: string(logs), WorkerName: workerName, - HatcheryID: h.Service().ID, }) } } diff --git a/sdk/log.go b/sdk/log.go index 40bcae25f6..38802ea91d 100644 --- a/sdk/log.go +++ b/sdk/log.go @@ -41,5 +41,4 @@ type ServiceLog struct { ServiceRequirementName string `json:"requirement_service_name" db:"requirement_service_name"` Val string `json:"val,omitempty" db:"value"` WorkerName string `json:"worker_name"` - HatcheryID int64 `json:"hatchery_id"` } From c96f16e90899f00fd36a990f32b0e1af073a7483 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Tue, 28 Apr 2020 16:30:47 +0200 Subject: [PATCH 10/19] auto review --- contrib/grpcplugins/action/archive/go.sum | 1 - contrib/grpcplugins/action/clair/go.sum | 1 - contrib/grpcplugins/action/download/go.sum | 1 - contrib/grpcplugins/action/group-tmpl/go.sum | 1 - .../grpcplugins/action/kafka-publish/go.sum | 1 - contrib/grpcplugins/action/marathon/go.sum | 1 - .../grpcplugins/action/npm-audit-parser/go.sum | 1 - contrib/grpcplugins/action/ssh-cmd/go.sum | 1 - contrib/grpcplugins/action/tmpl/go.sum | 1 - contrib/grpcplugins/action/venom/go.sum | 1 - engine/api/api.go | 2 +- engine/api/worker/dao.go | 2 -- engine/api/worker/gorp_model.go | 18 ------------------ engine/cmd_config.go | 2 +- engine/cmd_start.go | 4 ++-- engine/config.go | 2 +- engine/hatchery/kubernetes/kubernetes.go | 3 +++ engine/hatchery/kubernetes/services.go | 4 ---- go.mod | 1 - go.sum | 2 -- .../node/pipeline/step/step.log.component.ts | 1 - 21 files changed, 8 insertions(+), 43 deletions(-) diff --git a/contrib/grpcplugins/action/archive/go.sum b/contrib/grpcplugins/action/archive/go.sum index d66cd51d91..de49ffc712 100644 --- a/contrib/grpcplugins/action/archive/go.sum +++ b/contrib/grpcplugins/action/archive/go.sum @@ -638,7 +638,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/clair/go.sum b/contrib/grpcplugins/action/clair/go.sum index 2a3dbcc7b2..7d85dfe14a 100644 --- a/contrib/grpcplugins/action/clair/go.sum +++ b/contrib/grpcplugins/action/clair/go.sum @@ -639,7 +639,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/download/go.sum b/contrib/grpcplugins/action/download/go.sum index 894f1041fe..82207fd02a 100644 --- a/contrib/grpcplugins/action/download/go.sum +++ b/contrib/grpcplugins/action/download/go.sum @@ -620,7 +620,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/group-tmpl/go.sum b/contrib/grpcplugins/action/group-tmpl/go.sum index 894f1041fe..82207fd02a 100644 --- a/contrib/grpcplugins/action/group-tmpl/go.sum +++ b/contrib/grpcplugins/action/group-tmpl/go.sum @@ -620,7 +620,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/kafka-publish/go.sum b/contrib/grpcplugins/action/kafka-publish/go.sum index 60687b9e02..461c052044 100644 --- a/contrib/grpcplugins/action/kafka-publish/go.sum +++ b/contrib/grpcplugins/action/kafka-publish/go.sum @@ -790,7 +790,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/marathon/go.sum b/contrib/grpcplugins/action/marathon/go.sum index 65625bbedf..d2fdc02d16 100644 --- a/contrib/grpcplugins/action/marathon/go.sum +++ b/contrib/grpcplugins/action/marathon/go.sum @@ -772,7 +772,6 @@ google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1 h1:mzQIVyOPSXJaQWi1m6AFCjrCEPIwQBSOn48Ri8ZpzAg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/npm-audit-parser/go.sum b/contrib/grpcplugins/action/npm-audit-parser/go.sum index 894f1041fe..82207fd02a 100644 --- a/contrib/grpcplugins/action/npm-audit-parser/go.sum +++ b/contrib/grpcplugins/action/npm-audit-parser/go.sum @@ -620,7 +620,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/ssh-cmd/go.sum b/contrib/grpcplugins/action/ssh-cmd/go.sum index f0f12a11c2..d243ff9b64 100644 --- a/contrib/grpcplugins/action/ssh-cmd/go.sum +++ b/contrib/grpcplugins/action/ssh-cmd/go.sum @@ -621,7 +621,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/tmpl/go.sum b/contrib/grpcplugins/action/tmpl/go.sum index 894f1041fe..82207fd02a 100644 --- a/contrib/grpcplugins/action/tmpl/go.sum +++ b/contrib/grpcplugins/action/tmpl/go.sum @@ -620,7 +620,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/contrib/grpcplugins/action/venom/go.sum b/contrib/grpcplugins/action/venom/go.sum index 28e8bcb749..c48ef0b28d 100644 --- a/contrib/grpcplugins/action/venom/go.sum +++ b/contrib/grpcplugins/action/venom/go.sum @@ -661,7 +661,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/engine/api/api.go b/engine/api/api.go index a9922b0fe3..f9b1086585 100644 --- a/engine/api/api.go +++ b/engine/api/api.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "github.com/ovh/cds/engine/cdn" "io" "io/ioutil" "net/http" @@ -50,6 +49,7 @@ import ( "github.com/ovh/cds/engine/api/worker" "github.com/ovh/cds/engine/api/workermodel" "github.com/ovh/cds/engine/api/workflow" + "github.com/ovh/cds/engine/cdn" "github.com/ovh/cds/engine/service" "github.com/ovh/cds/sdk" "github.com/ovh/cds/sdk/cdsclient" diff --git a/engine/api/worker/dao.go b/engine/api/worker/dao.go index 755375f0ea..17f2a14140 100644 --- a/engine/api/worker/dao.go +++ b/engine/api/worker/dao.go @@ -9,7 +9,6 @@ import ( "github.com/ovh/cds/engine/api/authentication" "github.com/ovh/cds/engine/api/database/gorpmapping" "github.com/ovh/cds/sdk" - "github.com/ovh/cds/sdk/log" ) func Insert(ctx context.Context, db gorp.SqlExecutor, w *sdk.Worker) error { @@ -162,7 +161,6 @@ func LoadWorkerByIDWithDecryptKey(ctx context.Context, db gorp.SqlExecutor, work if !isValid { return sdk.Worker{}, sdk.ErrInvalidData } - log.Warning(ctx, "My WORKER: %+v", work.Worker) return work.Worker, err } diff --git a/engine/api/worker/gorp_model.go b/engine/api/worker/gorp_model.go index 411d0d0ba2..3e748823eb 100644 --- a/engine/api/worker/gorp_model.go +++ b/engine/api/worker/gorp_model.go @@ -20,21 +20,3 @@ func (e dbWorker) Canonical() gorpmapping.CanonicalForms { "{{print .ID}}{{.Name}}", } } - -func (e dbWorker) GetWorker() sdk.Worker { - return sdk.Worker{ - ID: e.ID, - PrivateKey: e.PrivateKey, - Status: e.Status, - JobRunID: e.JobRunID, - Arch: e.Arch, - ConsumerID: e.ConsumerID, - HatcheryID: e.HatcheryID, - LastBeat: e.LastBeat, - ModelID: e.ModelID, - Name: e.Name, - OS: e.OS, - Uptodate: e.Uptodate, - Version: e.Version, - } -} diff --git a/engine/cmd_config.go b/engine/cmd_config.go index f1632251a6..c630964f4d 100644 --- a/engine/cmd_config.go +++ b/engine/cmd_config.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "github.com/ovh/cds/engine/cdn" "io/ioutil" "os" "strconv" @@ -12,6 +11,7 @@ import ( toml "github.com/yesnault/go-toml" "github.com/ovh/cds/engine/api" + "github.com/ovh/cds/engine/cdn" "github.com/ovh/cds/engine/hatchery/kubernetes" "github.com/ovh/cds/engine/hatchery/local" "github.com/ovh/cds/engine/hatchery/marathon" diff --git a/engine/cmd_start.go b/engine/cmd_start.go index 0ebe11d3ab..fdd89761d1 100644 --- a/engine/cmd_start.go +++ b/engine/cmd_start.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/ovh/cds/engine/cdn" "os" "os/signal" "sort" @@ -15,6 +14,7 @@ import ( "github.com/ovh/cds/engine/api" "github.com/ovh/cds/engine/api/observability" "github.com/ovh/cds/engine/api/services" + "github.com/ovh/cds/engine/cdn" "github.com/ovh/cds/engine/elasticsearch" "github.com/ovh/cds/engine/hatchery/kubernetes" "github.com/ovh/cds/engine/hatchery/local" @@ -222,7 +222,7 @@ See $ engine config command for more details. sdk.Exit("Unable to start: missing service %s configuration", a) } serviceConfs = append(serviceConfs, serviceConf{arg: a, service: cdn.New(), cfg: *conf.CDN}) - names = append(names, conf.Hooks.Name) + names = append(names, conf.CDN.Name) types = append(types, services.TypeCDN) case services.TypeVCS: diff --git a/engine/config.go b/engine/config.go index 894ceddad6..0c4d6c1d99 100644 --- a/engine/config.go +++ b/engine/config.go @@ -3,7 +3,6 @@ package main import ( "bytes" "fmt" - "github.com/ovh/cds/engine/cdn" "io" "os" "sort" @@ -21,6 +20,7 @@ import ( "github.com/ovh/cds/engine/api/database" "github.com/ovh/cds/engine/api/database/gorpmapping" "github.com/ovh/cds/engine/api/services" + "github.com/ovh/cds/engine/cdn" "github.com/ovh/cds/engine/elasticsearch" "github.com/ovh/cds/engine/hatchery/kubernetes" "github.com/ovh/cds/engine/hatchery/local" diff --git a/engine/hatchery/kubernetes/kubernetes.go b/engine/hatchery/kubernetes/kubernetes.go index db40b8ba9a..05173edd8d 100644 --- a/engine/hatchery/kubernetes/kubernetes.go +++ b/engine/hatchery/kubernetes/kubernetes.go @@ -43,6 +43,9 @@ func New() *HatcheryKubernetes { // InitHatchery register local hatchery with its worker model func (h *HatcheryKubernetes) InitHatchery(ctx context.Context) error { + if err := h.Common.InitServiceLogger(); err != nil { + return err + } sdk.GoRoutine(context.Background(), "hatchery kubernetes routines", func(ctx context.Context) { h.routines(ctx) }) diff --git a/engine/hatchery/kubernetes/services.go b/engine/hatchery/kubernetes/services.go index 882e12743d..cb149eba61 100644 --- a/engine/hatchery/kubernetes/services.go +++ b/engine/hatchery/kubernetes/services.go @@ -14,10 +14,6 @@ import ( ) func (h *HatcheryKubernetes) getServicesLogs(ctx context.Context) error { - if err := h.Common.InitServiceLogger(); err != nil { - return err - } - pods, err := h.k8sClient.CoreV1().Pods(h.Config.Namespace).List(metav1.ListOptions{LabelSelector: LABEL_SERVICE_JOB_ID}) if err != nil { return err diff --git a/go.mod b/go.mod index 0c5d3ce3f7..6ce8c1120c 100644 --- a/go.mod +++ b/go.mod @@ -193,7 +193,6 @@ require ( google.golang.org/genproto v0.0.0-20190817000702-55e96fffbd48 // indirect google.golang.org/grpc v1.23.0 gopkg.in/AlecAivazis/survey.v1 v1.7.1 - gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/bsm/sarama-cluster.v2 v2.1.4 gopkg.in/go-playground/assert.v1 v1.2.1 // indirect diff --git a/go.sum b/go.sum index 6d50f6a612..7ebd74366f 100644 --- a/go.sum +++ b/go.sum @@ -702,8 +702,6 @@ google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/AlecAivazis/survey.v1 v1.7.1 h1:mzQIVyOPSXJaQWi1m6AFCjrCEPIwQBSOn48Ri8ZpzAg= gopkg.in/AlecAivazis/survey.v1 v1.7.1/go.mod h1:2Ehl7OqkBl3Xb8VmC4oFW2bItAhnUfzIjrOzwRxCrOU= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0 h1:Xg23ydYYJLmb9AK3XdcEpplHZd1MpN3X2ZeeMoBClmY= -gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0/go.mod h1:CeDeqW4tj9FrgZXF/dQCWZrBdcZWWBenhJtxLH4On2g= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/ui/src/app/views/workflow/run/node/pipeline/step/step.log.component.ts b/ui/src/app/views/workflow/run/node/pipeline/step/step.log.component.ts index ffae0bee26..e37d42d53e 100644 --- a/ui/src/app/views/workflow/run/node/pipeline/step/step.log.component.ts +++ b/ui/src/app/views/workflow/run/node/pipeline/step/step.log.component.ts @@ -255,7 +255,6 @@ export class WorkflowStepLogComponent implements OnInit, OnDestroy { this.limitTo = this.splittedLogs.length - 40; this.splittedLogsToDisplay.splice(this.limitFrom, this.limitTo - this.limitFrom); } - console.log(this.splittedLogsToDisplay); this._cd.markForCheck(); } From 5ebf8fb7433e55129a65c9a54e87dbea36350a08 Mon Sep 17 00:00:00 2001 From: Guiheux Steven Date: Tue, 28 Apr 2020 17:21:31 +0200 Subject: [PATCH 11/19] Apply suggestions from code review code review Co-Authored-By: Yvonnick Esnault --- engine/cdn/cdn.go | 2 +- engine/cmd_start.go | 2 +- engine/hatchery/serve.go | 2 +- sdk/jws/jws.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/cdn/cdn.go b/engine/cdn/cdn.go index 53d1c4a09d..236b5108ec 100644 --- a/engine/cdn/cdn.go +++ b/engine/cdn/cdn.go @@ -98,7 +98,7 @@ func (s *Service) Serve(c context.Context) error { //Start the http server log.Info(ctx, "CDN> Starting HTTP Server on port %d", s.Cfg.HTTP.Port) if err := server.ListenAndServe(); err != nil { - log.Fatalf("CDN> Cannot start cds-cdn: %s", err) + log.Fatalf("CDN> Cannot start cds-cdn: %v", err) } return ctx.Err() } diff --git a/engine/cmd_start.go b/engine/cmd_start.go index fdd89761d1..9685292cb4 100644 --- a/engine/cmd_start.go +++ b/engine/cmd_start.go @@ -85,7 +85,7 @@ This component operates CDS CDN to handle storage Start all of this with a single command: - $ engine start [api] [hatchery:local] [hatchery:marathon] [hatchery:openstack] [hatchery:swarm] [hatchery:vsphere] [elasticsearch] [hooks] [vcs] [repositories] [cdn] [migrate] [ui] + $ engine start [api] [cdn] [hatchery:local] [hatchery:marathon] [hatchery:openstack] [hatchery:swarm] [hatchery:vsphere] [elasticsearch] [hooks] [vcs] [repositories] [migrate] [ui] All the services are using the same configuration file format. diff --git a/engine/hatchery/serve.go b/engine/hatchery/serve.go index 906d029023..ae29fd127a 100644 --- a/engine/hatchery/serve.go +++ b/engine/hatchery/serve.go @@ -193,7 +193,7 @@ func (c *Common) getPanicDumpListHandler() service.Handler { func (c *Common) InitServiceLogger() error { tcpServer := c.Common.ServiceInstance.LogServer var signer jose.Signer - if tcpServer.Addr != "" && tcpServer.Port != 0 { + if tcpServer.Addr != "" && tcpServer.Port > 0 { logger, err := log.New(fmt.Sprintf("%s:%d", tcpServer.Addr, tcpServer.Port)) if err != nil { return sdk.WithStack(err) diff --git a/sdk/jws/jws.go b/sdk/jws/jws.go index 06d515f1da..cd6baf7810 100644 --- a/sdk/jws/jws.go +++ b/sdk/jws/jws.go @@ -76,7 +76,7 @@ func NewSigner(privateKey *rsa.PrivateKey) (jose.Signer, error) { return jose.NewSigner(jose.SigningKey{Algorithm: jose.PS512, Key: privateKey}, nil) } -// NewSigner instantiate a signer using HMAC using SHA-512 with the given private key. +// NewHMacSigner instantiates a signer using HMAC using SHA-512 with the given private key. func NewHMacSigner(secret []byte) (jose.Signer, error) { sign, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS512, Key: secret}, nil) if err != nil { From cb714d0125575c38b34814af65084357af6f1506 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 29 Apr 2020 10:58:43 +0200 Subject: [PATCH 12/19] test: add unit test on api and hatcheries --- engine/api/workflow_queue_test.go | 132 ++++++++++++++++++ engine/cdn/cdn_log.go | 6 +- engine/hatchery/kubernetes/services_test.go | 100 +++++++++++++ engine/hatchery/swarm/helper_test.go | 2 + engine/hatchery/swarm/swarm_util_logs_test.go | 98 +++++++++++++ sdk/hatchery/service_log.go | 1 - sdk/jws/jws_test.go | 16 +++ 7 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 engine/hatchery/kubernetes/services_test.go create mode 100644 engine/hatchery/swarm/swarm_util_logs_test.go delete mode 100644 sdk/hatchery/service_log.go diff --git a/engine/api/workflow_queue_test.go b/engine/api/workflow_queue_test.go index e82560858f..0a99e04311 100644 --- a/engine/api/workflow_queue_test.go +++ b/engine/api/workflow_queue_test.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "github.com/ovh/cds/engine/cdn" "io/ioutil" "net/http" "net/http/httptest" @@ -418,6 +419,14 @@ func Test_postTakeWorkflowJobHandler(t *testing.T) { //Register the worker testRegisterWorker(t, api, router, &ctx) + // Add cdn config + api.Config.CDN = cdn.Configuration{ + TCP: sdk.TCPServer{ + Port: 8090, + Addr: "localhost", + }, + } + uri := router.GetRoute("POST", api.postTakeWorkflowJobHandler, vars) require.NotEmpty(t, uri) @@ -451,12 +460,18 @@ func Test_postTakeWorkflowJobHandler(t *testing.T) { } } + assert.Equal(t, "localhost:8090", pbji.GelfServiceAddr) + run, err := workflow.LoadNodeJobRun(context.TODO(), api.mustDB(), api.Cache, ctx.job.ID) require.NoError(t, err) assert.Equal(t, "Building", run.Status) assert.Equal(t, ctx.model.Name, run.Model) assert.Equal(t, ctx.worker.Name, run.WorkerName) assert.NotEmpty(t, run.HatcheryName) + + wkrDB, err := worker.LoadWorkerByIDWithDecryptKey(context.TODO(), api.mustDB(), ctx.worker.ID) + assert.NoError(t, err) + assert.Len(t, wkrDB.PrivateKey, 32) } func Test_postBookWorkflowJobHandler(t *testing.T) { @@ -890,6 +905,123 @@ func Test_postWorkflowJobStaticFilesHandler(t *testing.T) { require.Equal(t, http.StatusNotImplemented, rec.Code) } +func TestWorkerPrivateKey(t *testing.T) { + api, db, router, end := newTestAPI(t) + defer end() + + // Create user + u, pass := assets.InsertAdminUser(t, api.mustDB()) + consumer, _ := authentication.LoadConsumerByTypeAndUserID(context.TODO(), db, sdk.ConsumerLocal, u.ID, authentication.LoadConsumerOptions.WithAuthentifiedUser) + + // Create project + key := sdk.RandomString(10) + proj := assets.InsertTestProject(t, db, api.Cache, key, key) + + // add group + require.NoError(t, group.InsertLinkGroupUser(context.TODO(), api.mustDB(), &group.LinkGroupUser{ + GroupID: proj.ProjectGroups[0].Group.ID, + AuthentifiedUserID: u.ID, + Admin: true, + })) + u.Groups = append(u.Groups, proj.ProjectGroups[0].Group) + + // Create pipeline + pip := &sdk.Pipeline{ + ProjectID: proj.ID, + Name: sdk.RandomString(10), + } + assert.NoError(t, pipeline.InsertPipeline(db, pip)) + + s := sdk.Stage{ + PipelineID: pip.ID, + Name: "foo", + Enabled: true, + } + + assert.NoError(t, pipeline.InsertStage(db, &s)) + + // get script action + script := assets.GetBuiltinOrPluginActionByName(t, db, sdk.ScriptAction) + + j := sdk.Job{ + Enabled: true, + PipelineStageID: s.ID, + Action: sdk.Action{ + Name: "script", + Actions: []sdk.Action{ + assets.NewAction(script.ID, sdk.Parameter{Name: "script", Value: "echo lol"}), + }, + }, + } + assert.NoError(t, pipeline.InsertJob(db, &j, s.ID, pip)) + + var errPip error + pip, errPip = pipeline.LoadPipelineByID(context.TODO(), db, pip.ID, true) + assert.NoError(t, errPip) + + // Create application + app := sdk.Application{ + ProjectID: proj.ID, + Name: sdk.RandomString(10), + } + assert.NoError(t, application.Insert(db, *proj, &app)) + + // Create workflow + w := sdk.Workflow{ + Name: sdk.RandomString(10), + ProjectID: proj.ID, + ProjectKey: proj.Key, + WorkflowData: sdk.WorkflowData{ + Node: sdk.Node{ + Name: "node1", + Ref: "node1", + Type: sdk.NodeTypePipeline, + Context: &sdk.NodeContext{ + PipelineID: pip.ID, + ApplicationID: app.ID, + }, + }, + }, + } + + p, err := project.Load(db, proj.Key, project.LoadOptions.WithPipelines, project.LoadOptions.WithApplications) + assert.NoError(t, err) + assert.NoError(t, workflow.Insert(context.TODO(), db, api.Cache, *p, &w)) + + workflowDeepPipeline, err := workflow.LoadByID(context.TODO(), db, api.Cache, *p, w.ID, workflow.LoadOptions{DeepPipeline: true}) + assert.NoError(t, err) + + wrDB, errwr := workflow.CreateRun(db, workflowDeepPipeline, nil, u) + assert.NoError(t, errwr) + wrDB.Workflow = *workflowDeepPipeline + + _, errmr := workflow.StartWorkflowRun(context.Background(), db, api.Cache, *p, wrDB, + &sdk.WorkflowRunPostHandlerOption{ + Manual: &sdk.WorkflowNodeRunManual{Username: u.Username}, + }, + consumer, nil) + assert.NoError(t, errmr) + + ctx := testRunWorkflowCtx{ + user: u, + password: pass, + project: proj, + workflow: &w, + run: wrDB, + } + testRegisterWorker(t, api, router, &ctx) + ctx.worker.JobRunID = &wrDB.WorkflowNodeRuns[w.WorkflowData.Node.ID][0].Stages[0].RunJobs[0].ID + assert.NoError(t, worker.SetToBuilding(context.TODO(), db, ctx.worker.ID, *ctx.worker.JobRunID, []byte("mysecret"))) + + wkFromDB, err := worker.LoadWorkerByName(context.TODO(), db, ctx.worker.Name) + require.NoError(t, err) + require.NotEqual(t, "mysecret", string(wkFromDB.PrivateKey)) + + wkFromDB, err = worker.LoadWorkerByIDWithDecryptKey(context.TODO(), db, ctx.worker.ID) + require.NoError(t, err) + require.Equal(t, "mysecret", string(wkFromDB.PrivateKey)) +} + func TestPostVulnerabilityReportHandler(t *testing.T) { api, db, router, end := newTestAPI(t) defer end() diff --git a/engine/cdn/cdn_log.go b/engine/cdn/cdn_log.go index f01c0aebaf..27f3d04751 100644 --- a/engine/cdn/cdn_log.go +++ b/engine/cdn/cdn_log.go @@ -47,7 +47,9 @@ func (s *Service) RunTcpLogServer(ctx context.Context) { log.Error(ctx, "unable to accept connection: %v", err) return } - go s.handleConnection(ctx, conn) + sdk.GoRoutine(ctx, "cdn-logServer", func(ctx context.Context) { + s.handleConnection(ctx, conn) + }) } }() } @@ -75,7 +77,7 @@ func (s *Service) handleConnection(ctx context.Context, conn net.Conn) { func (s *Service) handleLogMessage(ctx context.Context, messageReceived []byte) error { m := hook.Message{} - if err := (&m).UnmarshalJSON(messageReceived); err != nil { + if err := m.UnmarshalJSON(messageReceived); err != nil { return sdk.WrapError(err, "unable to unmarshall gelf message: %s", string(messageReceived)) } diff --git a/engine/hatchery/kubernetes/services_test.go b/engine/hatchery/kubernetes/services_test.go new file mode 100644 index 0000000000..b26005efb3 --- /dev/null +++ b/engine/hatchery/kubernetes/services_test.go @@ -0,0 +1,100 @@ +package kubernetes + +import ( + "context" + "crypto/rand" + "crypto/rsa" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "net/http" + "strings" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/h2non/gock.v1" + + "github.com/ovh/cds/sdk" +) + +var loggerCall = 0 + +func Test_serviceLogs(t *testing.T) { + h := NewHatcheryKubernetesTest(t) + h.Common.ServiceInstance = &sdk.Service{ + LogServer: sdk.TCPServer{ + Addr: "tcphost", + Port: 8090, + }, + } + reader := rand.Reader + bitSize := 2048 + key, err := rsa.GenerateKey(reader, bitSize) + require.NoError(t, err) + h.Common.PrivateKey = key + require.NoError(t, h.InitServiceLogger()) + + podsList := v1.PodList{ + Items: []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + Namespace: "kyubi", + Labels: map[string]string{ + LABEL_SERVICE_JOB_ID: "666", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "service-666-blabla", + }, + }, + }, + }, + }, + } + gock.New("http://lolcat.kube").Get("/api/v1/namespaces/hachibi/pods").Reply(http.StatusOK).JSON(podsList) + + gock.New("http://lolcat.kube").AddMatcher(func(r *http.Request, rr *gock.Request) (bool, error) { + b, err := gock.MatchPath(r, rr) + assert.NoError(t, err) + if r.Method == http.MethodGet && strings.HasPrefix(r.URL.String(), "http://lolcat.kube/api/v1/namespaces/hachibi/pods/pod-name/log?container=service-666-blabla") { + if b { + return true, nil + } + return false, nil + } + return true, nil + }).Reply(http.StatusOK).Body(strings.NewReader("Je suis le log")) + + h.ServiceLogger = GetMockLogger() + + loggerCall = 0 + assert.NoError(t, h.getServicesLogs(context.TODO())) + + for _, p := range gock.Pending() { + t.Logf("%+v", p.Request().URLStruct.String()) + } + require.True(t, gock.IsDone()) + require.Equal(t, 1, loggerCall) +} + +func GetMockLogger() *logrus.Logger { + log := logrus.New() + log.AddHook(&HookMock{}) + return log +} + +type HookMock struct{} + +func (h *HookMock) Levels() []logrus.Level { + return []logrus.Level{ + logrus.InfoLevel, + } +} +func (h *HookMock) Fire(e *logrus.Entry) error { + loggerCall++ + return nil +} diff --git a/engine/hatchery/swarm/helper_test.go b/engine/hatchery/swarm/helper_test.go index d14ee8360f..c8c80f9c09 100644 --- a/engine/hatchery/swarm/helper_test.go +++ b/engine/hatchery/swarm/helper_test.go @@ -1,6 +1,7 @@ package swarm import ( + "github.com/ovh/cds/engine/hatchery" "github.com/ovh/cds/sdk/cdsclient" "github.com/stretchr/testify/require" "gopkg.in/h2non/gock.v1" @@ -37,6 +38,7 @@ func testSwarmHatchery(t *testing.T) *HatcherySwarm { Config: HatcheryConfiguration{ DisableDockerOptsOnRequirements: false, }, + Common: hatchery.Common{}, } h.dockerClients["default"] = &dockerClient{Client: *c, MaxContainers: 2, name: "default"} diff --git a/engine/hatchery/swarm/swarm_util_logs_test.go b/engine/hatchery/swarm/swarm_util_logs_test.go new file mode 100644 index 0000000000..14ff6e232e --- /dev/null +++ b/engine/hatchery/swarm/swarm_util_logs_test.go @@ -0,0 +1,98 @@ +package swarm + +import ( + "crypto/rand" + "crypto/rsa" + "net/http" + "strings" + "testing" + + "github.com/docker/docker/api/types" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/h2non/gock.v1" + + "github.com/ovh/cds/sdk" +) + +var loggerCall = 0 + +func Test_serviceLogs(t *testing.T) { + h := InitTestHatcherySwarm(t) + h.Common.ServiceInstance = &sdk.Service{ + LogServer: sdk.TCPServer{ + Addr: "tcphost", + Port: 8090, + }, + } + reader := rand.Reader + bitSize := 2048 + key, err := rsa.GenerateKey(reader, bitSize) + require.NoError(t, err) + h.Common.PrivateKey = key + require.NoError(t, h.InitServiceLogger()) + + containers := []types.Container{ + { + ID: "swarmy-model1-w1", + Names: []string{"swarmy-model1-w1"}, + Labels: map[string]string{ + "hatchery": "swarmy", + "worker_name": "swarmy-model1-w1", + }, + }, + { + ID: "service-1", + Names: []string{"swarmy-model1-w1"}, + Labels: map[string]string{ + "hatchery": "swarmy", + "worker_name": "swarmy-model1-w1", + "service_job_id": "666", + "service_id": "1", + }, + }, + } + + gock.New("https://lolcat.host").Get("/v6.66/containers/json").Reply(http.StatusOK).JSON(containers) + gock.New("https://lolcat.host").AddMatcher(func(r *http.Request, rr *gock.Request) (bool, error) { + b, err := gock.MatchPath(r, rr) + assert.NoError(t, err) + if r.Method == http.MethodGet && strings.HasPrefix(r.URL.String(), "https://lolcat.host/v6.66/containers/service-1/logs") { + if b { + return true, nil + } + return false, nil + } + return true, nil + }).Reply(http.StatusOK).Body(strings.NewReader("Je suis le log")) + + h.ServiceLogger = GetMockLogger() + + loggerCall = 0 + assert.NoError(t, h.getServicesLogs()) + + for _, p := range gock.Pending() { + t.Logf("%+v", p.Request().URLStruct.String()) + } + require.True(t, gock.IsDone()) + require.Equal(t, 1, loggerCall) +} + +func GetMockLogger() *logrus.Logger { + log := logrus.New() + log.AddHook(&HookMock{}) + return log +} + +type HookMock struct{} + +func (h *HookMock) Levels() []logrus.Level { + return []logrus.Level{ + logrus.InfoLevel, + } +} +func (h *HookMock) Fire(e *logrus.Entry) error { + loggerCall++ + return nil +} diff --git a/sdk/hatchery/service_log.go b/sdk/hatchery/service_log.go deleted file mode 100644 index 7c06ac419e..0000000000 --- a/sdk/hatchery/service_log.go +++ /dev/null @@ -1 +0,0 @@ -package hatchery diff --git a/sdk/jws/jws_test.go b/sdk/jws/jws_test.go index 576daa4a52..c690d18615 100644 --- a/sdk/jws/jws_test.go +++ b/sdk/jws/jws_test.go @@ -13,3 +13,19 @@ func TestNewRandomRSAKey(t *testing.T) { require.NoError(t, err) t.Log(string(btes)) } + +func TestHMacSignAndVerify(t *testing.T) { + secret, err := NewRandomSymmetricKey(32) + require.NoError(t, err) + signer, err := NewHMacSigner(secret) + require.NoError(t, err) + + message := "coucou" + messageSigned, err := Sign(signer, message) + require.NoError(t, err) + require.NotEqual(t, message, messageSigned) + + var unsigned string + require.NoError(t, Verify(secret, messageSigned, &unsigned)) + require.Equal(t, message, unsigned) +} From 106225cf9d36a719987eed6ef35a78315cd4de34 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 29 Apr 2020 12:03:51 +0200 Subject: [PATCH 13/19] test: add TU on cdn --- engine/cdn/cdn_log.go | 3 + engine/cdn/cdn_log_test.go | 132 +++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 engine/cdn/cdn_log_test.go diff --git a/engine/cdn/cdn_log.go b/engine/cdn/cdn_log.go index 27f3d04751..8d58c52483 100644 --- a/engine/cdn/cdn_log.go +++ b/engine/cdn/cdn_log.go @@ -198,6 +198,9 @@ func (s *Service) handleServiceLog(ctx context.Context, hatcheryID int64, hatche WorkflowNodeRunID: nodeRunJob.WorkflowNodeRunID, Val: m.Full, } + if !strings.HasSuffix(logs.Val, "\n") { + logs.Val += "\n" + } if err := workflow.AddServiceLog(s.Db, nodeRunJob, &logs, s.Cfg.Log.ServiceMaxSize); err != nil { return err diff --git a/engine/cdn/cdn_log_test.go b/engine/cdn/cdn_log_test.go new file mode 100644 index 0000000000..37eda70a19 --- /dev/null +++ b/engine/cdn/cdn_log_test.go @@ -0,0 +1,132 @@ +package cdn + +import ( + "context" + "fmt" + "github.com/ovh/cds/engine/api/bootstrap" + "github.com/ovh/cds/engine/api/test" + "github.com/ovh/cds/engine/api/workflow" + "github.com/ovh/cds/sdk" + gocache "github.com/patrickmn/go-cache" + "testing" + "time" + + "github.com/ovh/cds/sdk/jws" + "github.com/ovh/cds/sdk/log" + "github.com/stretchr/testify/require" +) + +func TestWorkerLog(t *testing.T) { + // Init DB + db, cache, end := test.SetupPG(t, bootstrap.InitiliazeDB) + defer end() + + // Create worker private key + key, err := jws.NewRandomSymmetricKey(32) + require.NoError(t, err) + + // Create worker signer + sign, err := jws.NewHMacSigner(key) + require.NoError(t, err) + + // Create cdn service + s := Service{ + Db: db, + Cache: cache, + } + + // Create run job + jobRun := sdk.WorkflowNodeJobRun{ + Start: time.Now(), + WorkflowNodeRunID: 1, + Status: sdk.StatusBuilding, + } + dbj := new(workflow.JobRun) + require.NoError(t, dbj.ToJobRun(&jobRun)) + require.NoError(t, db.Insert(dbj)) + + signature := log.Signature{ + Worker: &log.SignatureWorker{ + WorkerID: "abcdef-123456", + StepOrder: 0, + }, + JobID: dbj.ID, + Timestamp: time.Now().UnixNano(), + } + logCache.Set(fmt.Sprintf("worker-%s", signature.Worker.WorkerID), sdk.Worker{ + JobRunID: &signature.JobID, + PrivateKey: key, + }, gocache.DefaultExpiration) + + signatureField, err := jws.Sign(sign, signature) + require.NoError(t, err) + + message := `{"level": 1, "version": "1", "short": "this", "_facility": "fa", "_file": "file", + "host": "host", "_line":1, "_pid": 1, "_prefix": "prefix", "full_message": "this is my message", "_Signature": "%s"}` + message = fmt.Sprintf(message, signatureField) + + require.NoError(t, s.handleLogMessage(context.TODO(), []byte(message))) + + logs, err := workflow.LoadLogs(s.Db, dbj.ID) + require.NoError(t, err) + require.Len(t, logs, 1) + require.Equal(t, "this is my message\n", logs[0].Val) +} + +func TestServiceLog(t *testing.T) { + // Init DB + db, cache, end := test.SetupPG(t, bootstrap.InitiliazeDB) + defer end() + + // Create hatchery private key + key, err := jws.NewRandomRSAKey() + require.NoError(t, err) + + // Create worker signer + sign, err := jws.NewSigner(key) + require.NoError(t, err) + + // Create cdn service + s := Service{ + Db: db, + Cache: cache, + } + + // Create run job + jobRun := sdk.WorkflowNodeJobRun{ + Start: time.Now(), + WorkflowNodeRunID: 1, + Status: sdk.StatusBuilding, + } + dbj := new(workflow.JobRun) + require.NoError(t, dbj.ToJobRun(&jobRun)) + require.NoError(t, db.Insert(dbj)) + + signature := log.Signature{ + Service: &log.SignatureService{ + WorkerName: "my-worker-name", + HatcheryID: 1, + HatcheryName: "my-hatchery-name", + RequirementID: 1, + RequirementName: "service-1", + }, + JobID: dbj.ID, + Timestamp: time.Now().UnixNano(), + } + + logCache.Set(fmt.Sprintf("hatchery-key-%d", signature.Service.HatcheryID), &key.PublicKey, gocache.DefaultExpiration) + logCache.Set(fmt.Sprintf("service-worker-%s", signature.Service.WorkerName), true, gocache.DefaultExpiration) + + signatureField, err := jws.Sign(sign, signature) + require.NoError(t, err) + + message := `{"level": 1, "version": "1", "short": "this", "_facility": "fa", "_file": "file", + "host": "host", "_line":1, "_pid": 1, "_prefix": "prefix", "full_message": "this is my service message", "_Signature": "%s"}` + message = fmt.Sprintf(message, signatureField) + + require.NoError(t, s.handleLogMessage(context.TODO(), []byte(message))) + + logs, err := workflow.LoadServiceLog(db, dbj.ID, signature.Service.RequirementName) + require.NoError(t, err) + require.Equal(t, "this is my service message\n", logs.Val) +} From 0f509d46e2239faf6df0e4ffd91737fe4574e127 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 29 Apr 2020 15:52:38 +0200 Subject: [PATCH 14/19] fix: add retry when loosing connection --- sdk/log/hook/tcp.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/log/hook/tcp.go b/sdk/log/hook/tcp.go index bb41bc993d..a389860528 100644 --- a/sdk/log/hook/tcp.go +++ b/sdk/log/hook/tcp.go @@ -93,7 +93,10 @@ func (w *TCPWriter) Write(p []byte) (int, error) { } if err := w.WriteMessage(&m); err != nil { - return 0, err + fmt.Fprintln(os.Stderr, "[gelf] Try 1 retry: ", err) + if err := w.WriteMessage(&m); err != nil { + return 0, err + } } return len(p), nil From f0186a6e2830d12bd60bb09ca7c9e498a215b948 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 29 Apr 2020 16:56:04 +0200 Subject: [PATCH 15/19] try debug --- tests/04_sc_workflow_run_simple_plugin.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/04_sc_workflow_run_simple_plugin.yml b/tests/04_sc_workflow_run_simple_plugin.yml index 895a48832e..badbdbc77c 100644 --- a/tests/04_sc_workflow_run_simple_plugin.yml +++ b/tests/04_sc_workflow_run_simple_plugin.yml @@ -52,4 +52,5 @@ testcases: - name: check workflow log steps: - script: {{.cdsctl}} -f {{.cdsctl.config}} workflow logs download 04SCWORKFLOWRUNSIMPLEPLUGIN 04SCWorkflowRunSimplePlugin-WORKFLOW 1 --pattern Build + - script: cat *.log - script: grep "Hello, I'm the simple plugin" *.log From 9dd17e2dcb5129da0bb641474c4b4eee8892190c Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Wed, 29 Apr 2020 17:26:45 +0200 Subject: [PATCH 16/19] fix: no conf --- engine/api/workflow_queue.go | 4 +++- tests/04_sc_workflow_run_simple_plugin.yml | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/engine/api/workflow_queue.go b/engine/api/workflow_queue.go index 86d2962727..fa9e1e8430 100644 --- a/engine/api/workflow_queue.go +++ b/engine/api/workflow_queue.go @@ -100,7 +100,9 @@ func (api *API) postTakeWorkflowJobHandler() service.Handler { return sdk.WrapError(err, "cannot takeJob nodeJobRunID:%d", id) } - pbji.GelfServiceAddr = fmt.Sprintf("%s:%d", api.Config.CDN.TCP.Addr, api.Config.CDN.TCP.Port) + if api.Config.CDN.TCP.Addr != "" && api.Config.CDN.TCP.Port > 0 { + pbji.GelfServiceAddr = fmt.Sprintf("%s:%d", api.Config.CDN.TCP.Addr, api.Config.CDN.TCP.Port) + } workflow.ResyncNodeRunsWithCommits(ctx, api.mustDB(), api.Cache, *p, report) go WorkflowSendEvent(context.Background(), api.mustDB(), api.Cache, *p, report) diff --git a/tests/04_sc_workflow_run_simple_plugin.yml b/tests/04_sc_workflow_run_simple_plugin.yml index badbdbc77c..895a48832e 100644 --- a/tests/04_sc_workflow_run_simple_plugin.yml +++ b/tests/04_sc_workflow_run_simple_plugin.yml @@ -52,5 +52,4 @@ testcases: - name: check workflow log steps: - script: {{.cdsctl}} -f {{.cdsctl.config}} workflow logs download 04SCWORKFLOWRUNSIMPLEPLUGIN 04SCWorkflowRunSimplePlugin-WORKFLOW 1 --pattern Build - - script: cat *.log - script: grep "Hello, I'm the simple plugin" *.log From e29d6a5e927796db34417945f32c669426c15daa Mon Sep 17 00:00:00 2001 From: Guiheux Steven Date: Thu, 30 Apr 2020 10:47:33 +0200 Subject: [PATCH 17/19] Apply suggestions from code review apply suggestion Co-Authored-By: Richard LT --- engine/api/worker/dao.go | 8 ++++---- sdk/log.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/engine/api/worker/dao.go b/engine/api/worker/dao.go index 17f2a14140..73070bc128 100644 --- a/engine/api/worker/dao.go +++ b/engine/api/worker/dao.go @@ -140,7 +140,7 @@ func SetToBuilding(ctx context.Context, db gorp.SqlExecutor, workerID string, jo if err := gorpmapping.UpdateAndSign(ctx, db, dbData); err != nil { return err } - return err + return nil } // LoadWorkerByIDWithDecryptKey load worker with decrypted private key @@ -152,14 +152,14 @@ func LoadWorkerByIDWithDecryptKey(ctx context.Context, db gorp.SqlExecutor, work return sdk.Worker{}, err } if !found { - return sdk.Worker{}, sdk.ErrNotFound + return sdk.Worker{}, sdk.WithStack(sdk.ErrNotFound) } isValid, err := gorpmapping.CheckSignature(work, work.Signature) if err != nil { return sdk.Worker{}, err } if !isValid { - return sdk.Worker{}, sdk.ErrInvalidData + return sdk.Worker{}, sdk.WithStack(sdk.ErrInvalidData) } return work.Worker, err } @@ -173,7 +173,7 @@ func LoadWorkerByName(ctx context.Context, db gorp.SqlExecutor, workerName strin return sdk.Worker{}, err } if !found { - return sdk.Worker{}, sdk.ErrNotFound + return sdk.Worker{}, sdk.WithStack(sdk.ErrNotFound) } return work.Worker, err } diff --git a/sdk/log.go b/sdk/log.go index 38802ea91d..72d22e7965 100644 --- a/sdk/log.go +++ b/sdk/log.go @@ -40,5 +40,5 @@ type ServiceLog struct { ServiceRequirementID int64 `json:"requirement_id" db:"-"` ServiceRequirementName string `json:"requirement_service_name" db:"requirement_service_name"` Val string `json:"val,omitempty" db:"value"` - WorkerName string `json:"worker_name"` + WorkerName string `json:"worker_name" db:"-"` } From 5317a138728cb633dd3cd1fceda30221eb4feec1 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Thu, 30 Apr 2020 12:05:22 +0200 Subject: [PATCH 18/19] fix: core review --- engine/api/worker/dao.go | 62 +++++++++++++++++++++++++++++++++------- engine/cdn/cdn_log.go | 4 +-- sdk/worker.go | 2 +- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/engine/api/worker/dao.go b/engine/api/worker/dao.go index 73070bc128..d151c8918e 100644 --- a/engine/api/worker/dao.go +++ b/engine/api/worker/dao.go @@ -50,6 +50,13 @@ func LoadByConsumerID(ctx context.Context, db gorp.SqlExecutor, id string) (*sdk if !found { return nil, sdk.WithStack(sdk.ErrNotFound) } + isValid, err := gorpmapping.CheckSignature(w, w.Signature) + if err != nil { + return nil, err + } + if !isValid { + return nil, sdk.WithStack(sdk.ErrInvalidData) + } return &w.Worker, nil } @@ -63,6 +70,13 @@ func LoadByID(ctx context.Context, db gorp.SqlExecutor, id string) (*sdk.Worker, if !found { return nil, sdk.WithStack(sdk.ErrNotFound) } + isValid, err := gorpmapping.CheckSignature(w, w.Signature) + if err != nil { + return nil, err + } + if !isValid { + return nil, sdk.WithStack(sdk.ErrInvalidData) + } return &w.Worker, nil } @@ -74,6 +88,13 @@ func LoadAll(ctx context.Context, db gorp.SqlExecutor) ([]sdk.Worker, error) { } workers := make([]sdk.Worker, len(wks)) for i := range wks { + isValid, err := gorpmapping.CheckSignature(wks[i], wks[i].Signature) + if err != nil { + return nil, err + } + if !isValid { + return nil, sdk.WithStack(sdk.ErrInvalidData) + } workers[i] = wks[i].Worker } return workers, nil @@ -87,6 +108,13 @@ func LoadByHatcheryID(ctx context.Context, db gorp.SqlExecutor, hatcheryID int64 } workers := make([]sdk.Worker, len(wks)) for i := range wks { + isValid, err := gorpmapping.CheckSignature(wks[i], wks[i].Signature) + if err != nil { + return nil, err + } + if !isValid { + return nil, sdk.WithStack(sdk.ErrInvalidData) + } workers[i] = wks[i].Worker } return workers, nil @@ -104,6 +132,13 @@ func LoadDeadWorkers(ctx context.Context, db gorp.SqlExecutor, timeout float64, } workers := make([]sdk.Worker, len(wks)) for i := range wks { + isValid, err := gorpmapping.CheckSignature(wks[i], wks[i].Signature) + if err != nil { + return nil, err + } + if !isValid { + return nil, sdk.WithStack(sdk.ErrInvalidData) + } workers[i] = wks[i].Worker } return workers, nil @@ -144,36 +179,43 @@ func SetToBuilding(ctx context.Context, db gorp.SqlExecutor, workerID string, jo } // LoadWorkerByIDWithDecryptKey load worker with decrypted private key -func LoadWorkerByIDWithDecryptKey(ctx context.Context, db gorp.SqlExecutor, workerID string) (sdk.Worker, error) { +func LoadWorkerByIDWithDecryptKey(ctx context.Context, db gorp.SqlExecutor, workerID string) (*sdk.Worker, error) { var work dbWorker query := gorpmapping.NewQuery(`SELECT * FROM worker WHERE id = $1`).Args(workerID) found, err := gorpmapping.Get(ctx, db, query, &work, gorpmapping.GetOptions.WithDecryption) if err != nil { - return sdk.Worker{}, err + return nil, err } if !found { - return sdk.Worker{}, sdk.WithStack(sdk.ErrNotFound) + return nil, sdk.WithStack(sdk.ErrNotFound) } isValid, err := gorpmapping.CheckSignature(work, work.Signature) if err != nil { - return sdk.Worker{}, err + return nil, err } if !isValid { - return sdk.Worker{}, sdk.WithStack(sdk.ErrInvalidData) + return nil, sdk.WithStack(sdk.ErrInvalidData) } - return work.Worker, err + return &work.Worker, err } // LoadWorkerByName load worker by name -func LoadWorkerByName(ctx context.Context, db gorp.SqlExecutor, workerName string) (sdk.Worker, error) { +func LoadWorkerByName(ctx context.Context, db gorp.SqlExecutor, workerName string) (*sdk.Worker, error) { var work dbWorker query := gorpmapping.NewQuery(`SELECT * FROM worker WHERE name = $1`).Args(workerName) found, err := gorpmapping.Get(ctx, db, query, &work) if err != nil { - return sdk.Worker{}, err + return nil, err } if !found { - return sdk.Worker{}, sdk.WithStack(sdk.ErrNotFound) + return nil, sdk.WithStack(sdk.ErrNotFound) + } + isValid, err := gorpmapping.CheckSignature(work, work.Signature) + if err != nil { + return nil, err + } + if !isValid { + return nil, sdk.WithStack(sdk.ErrInvalidData) } - return work.Worker, err + return &work.Worker, err } diff --git a/engine/cdn/cdn_log.go b/engine/cdn/cdn_log.go index 8d58c52483..4c6c305c14 100644 --- a/engine/cdn/cdn_log.go +++ b/engine/cdn/cdn_log.go @@ -213,8 +213,8 @@ func (s *Service) getWorker(ctx context.Context, workerID string) (sdk.Worker, e if err != nil { return sdk.Worker{}, err } - logCache.Set(fmt.Sprintf("worker-%s", w.ID), w, gocache.DefaultExpiration) - return w, nil + logCache.Set(fmt.Sprintf("worker-%s", w.ID), *w, gocache.DefaultExpiration) + return *w, nil } func (s *Service) getHatchery(ctx context.Context, hatcheryID int64, hatcheryName string) (*rsa.PublicKey, error) { diff --git a/sdk/worker.go b/sdk/worker.go index 7342a07ba2..bfb04b2499 100644 --- a/sdk/worker.go +++ b/sdk/worker.go @@ -20,7 +20,7 @@ type Worker struct { Version string `json:"version" cli:"version" db:"version"` OS string `json:"os" cli:"os" db:"os"` Arch string `json:"arch" cli:"arch" db:"arch"` - PrivateKey []byte `json:"-" cli:"-" db:"cypher_private_key" gorpmapping:"encrypted,ID,Name"` + PrivateKey []byte `json:"-" cli:"-" db:"cypher_private_key" gorpmapping:"encrypted,ID,Name,JobRunID"` } // WorkerRegistrationForm represents the arguments needed to register a worker From 2036d0d5c89b5b45fbc71d02301fe40368ef0583 Mon Sep 17 00:00:00 2001 From: Steven Guiheux Date: Thu, 7 May 2020 15:19:58 +0200 Subject: [PATCH 19/19] rename sql file --- engine/sql/{200_worker.sql => 201_worker.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename engine/sql/{200_worker.sql => 201_worker.sql} (100%) diff --git a/engine/sql/200_worker.sql b/engine/sql/201_worker.sql similarity index 100% rename from engine/sql/200_worker.sql rename to engine/sql/201_worker.sql