diff --git a/cli/cdsctl/monitoring.go b/cli/cdsctl/monitoring.go index 6c0124c444..a394e38556 100644 --- a/cli/cdsctl/monitoring.go +++ b/cli/cdsctl/monitoring.go @@ -413,8 +413,6 @@ func (ui *Termui) updateQueue(baseURL string) string { ui.msg = fmt.Sprintf("[%s](bg-red)", errpb.Error()) return "" } - elapsed = time.Since(start) - msg += fmt.Sprintf(" | [queue pb %s](fg-cyan,bg-default)", sdk.Round(elapsed, time.Millisecond).String()) var maxQueued time.Duration diff --git a/engine/main.go b/engine/main.go index 12ff98d971..b8fa4a7688 100644 --- a/engine/main.go +++ b/engine/main.go @@ -135,61 +135,159 @@ var configNewCmd = &cobra.Command{ Use: "new", Short: "CDS configuration file assistant", Long: ` -Comming soon...`, +# generate the whole configuration file + $ engine config new > conf.toml + +# you can compose your file configuration +# this will generate a file configuration containing +# api and hatchery:local µService + $ engine config new api hatchery:local + +# For advanced usage, Debug and Tracing section can be generated as: + $ engine config new debug tracing [µService(s)...] + +# All options + $ engine config new [debug] [tracing] [api] [hatchery:local] [hatchery:marathon] [hatchery:openstack] [hatchery:swarm] [hatchery:vsphere] [elasticsearch] [hooks] [vcs] [repositories] [migrate] + +`, Run: func(cmd *cobra.Command, args []string) { + for _, a := range args { + if strings.HasPrefix(a, "hatchery:") { + if conf.Hatchery == nil { + conf.Hatchery = &HatcheryConfiguration{} + break + } + } + } + + for _, a := range args { + switch a { + case "debug": + conf.Debug = &DebugConfiguration{} + case "tracing": + conf.Tracing = &observability.Configuration{} + case "api": + conf.API = &api.Configuration{} + case "migrate": + conf.DatabaseMigrate = &migrateservice.Configuration{} + case "hatchery:local": + conf.Hatchery.Local = &local.HatcheryConfiguration{} + case "hatchery:kubernetes": + conf.Hatchery.Kubernetes = &kubernetes.HatcheryConfiguration{} + case "hatchery:marathon": + conf.Hatchery.Marathon = &marathon.HatcheryConfiguration{} + case "hatchery:openstack": + conf.Hatchery.Openstack = &openstack.HatcheryConfiguration{} + case "hatchery:swarm": + conf.Hatchery.Swarm = &swarm.HatcheryConfiguration{} + case "hatchery:vsphere": + conf.Hatchery.VSphere = &vsphere.HatcheryConfiguration{} + case "hooks": + conf.Hooks = &hooks.Configuration{} + case "vcs": + conf.VCS = &vcs.Configuration{} + case "repositories": + conf.Repositories = &repositories.Configuration{} + case "elasticsearch": + conf.ElasticSearch = &elasticsearch.Configuration{} + default: + fmt.Printf("Error: service '%s' unknown\n", a) + os.Exit(1) + } + } + + if len(args) == 0 { + conf.Debug = &DebugConfiguration{} + conf.Tracing = &observability.Configuration{} + conf.API = &api.Configuration{} + conf.DatabaseMigrate = &migrateservice.Configuration{} + conf.Hatchery = &HatcheryConfiguration{} + conf.Hatchery.Local = &local.HatcheryConfiguration{} + conf.Hatchery.Kubernetes = &kubernetes.HatcheryConfiguration{} + conf.Hatchery.Marathon = &marathon.HatcheryConfiguration{} + conf.Hatchery.Openstack = &openstack.HatcheryConfiguration{} + conf.Hatchery.Swarm = &swarm.HatcheryConfiguration{} + conf.Hatchery.VSphere = &vsphere.HatcheryConfiguration{} + conf.Hooks = &hooks.Configuration{} + conf.VCS = &vcs.Configuration{} + conf.Repositories = &repositories.Configuration{} + conf.ElasticSearch = &elasticsearch.Configuration{} + } + defaults.SetDefaults(conf) - conf.API.Auth.SharedInfraToken = sdk.RandomString(128) - conf.API.Secrets.Key = sdk.RandomString(32) - conf.API.Providers = append(conf.API.Providers, api.ProviderConfiguration{ - Name: "sample-provider", - Token: sdk.RandomString(32), - }) - conf.API.Services = append(conf.API.Services, api.ServiceConfiguration{ - Name: "sample-service", - URL: "https://ovh.github.io", - Port: "443", - Path: "/cds", - HealthPath: "/cds", - HealthPort: "443", - HealthURL: "https://ovh.github.io", - Type: "doc", - }) - conf.Hatchery.Local.API.Token = conf.API.Auth.SharedInfraToken - conf.Hatchery.Openstack.API.Token = conf.API.Auth.SharedInfraToken - conf.Hatchery.VSphere.API.Token = conf.API.Auth.SharedInfraToken - conf.Hatchery.Swarm.API.Token = conf.API.Auth.SharedInfraToken - conf.Hatchery.Swarm.DockerEngines = map[string]swarm.DockerEngineConfiguration{ - "sample-docker-engine": { - Host: "///var/run/docker.sock", - }, - } - conf.Hatchery.Marathon.API.Token = conf.API.Auth.SharedInfraToken - conf.Hooks.API.Token = conf.API.Auth.SharedInfraToken - conf.Repositories.API.Token = conf.API.Auth.SharedInfraToken - conf.DatabaseMigrate.API.Token = conf.API.Auth.SharedInfraToken - conf.VCS.API.Token = conf.API.Auth.SharedInfraToken - conf.VCS.Servers = map[string]vcs.ServerConfiguration{} - conf.VCS.Servers["Github"] = vcs.ServerConfiguration{ - URL: "https://github.com", - Github: &vcs.GithubServerConfiguration{ - ClientID: "xxxx", - ClientSecret: "xxxx", - }, - } - conf.VCS.Servers["Bitbucket"] = vcs.ServerConfiguration{ - URL: "https://mybitbucket.com", - Bitbucket: &vcs.BitbucketServerConfiguration{ - ConsumerKey: "xxx", - PrivateKey: "xxx", - }, - } - conf.VCS.Servers["Gitlab"] = vcs.ServerConfiguration{ - URL: "https://gitlab.com", - Gitlab: &vcs.GitlabServerConfiguration{ - AppID: "xxxx", - Secret: "xxxx", - }, + var sharedInfraToken = sdk.RandomString(128) + + if conf.API != nil { + conf.API.Auth.SharedInfraToken = sharedInfraToken + conf.API.Secrets.Key = sdk.RandomString(32) + conf.API.Providers = append(conf.API.Providers, api.ProviderConfiguration{ + Name: "sample-provider", + Token: sdk.RandomString(32), + }) + conf.API.Services = append(conf.API.Services, api.ServiceConfiguration{ + Name: "sample-service", + URL: "https://ovh.github.io", + Port: "443", + Path: "/cds", + HealthPath: "/cds", + HealthPort: "443", + HealthURL: "https://ovh.github.io", + Type: "doc", + }) + } else { + sharedInfraToken = "enter sharedInfraToken from section [api.auth] here" + } + + if conf.Hatchery != nil { + conf.Hatchery.Local.API.Token = sharedInfraToken + conf.Hatchery.Openstack.API.Token = sharedInfraToken + conf.Hatchery.VSphere.API.Token = sharedInfraToken + conf.Hatchery.Swarm.API.Token = sharedInfraToken + conf.Hatchery.Swarm.DockerEngines = map[string]swarm.DockerEngineConfiguration{ + "sample-docker-engine": { + Host: "///var/run/docker.sock", + }, + } + conf.Hatchery.Marathon.API.Token = sharedInfraToken + } + + if conf.Hooks != nil { + conf.Hooks.API.Token = sharedInfraToken + } + + if conf.Repositories != nil { + conf.Repositories.API.Token = sharedInfraToken + } + + if conf.DatabaseMigrate != nil { + conf.DatabaseMigrate.API.Token = sharedInfraToken + } + + if conf.VCS != nil { + conf.VCS.API.Token = sharedInfraToken + conf.VCS.Servers = map[string]vcs.ServerConfiguration{} + conf.VCS.Servers["Github"] = vcs.ServerConfiguration{ + URL: "https://github.com", + Github: &vcs.GithubServerConfiguration{ + ClientID: "xxxx", + ClientSecret: "xxxx", + }, + } + conf.VCS.Servers["Bitbucket"] = vcs.ServerConfiguration{ + URL: "https://mybitbucket.com", + Bitbucket: &vcs.BitbucketServerConfiguration{ + ConsumerKey: "xxx", + PrivateKey: "xxx", + }, + } + conf.VCS.Servers["Gitlab"] = vcs.ServerConfiguration{ + URL: "https://gitlab.com", + Gitlab: &vcs.GitlabServerConfiguration{ + AppID: "xxxx", + Secret: "xxxx", + }, + } } if !configNewAsEnvFlag { @@ -229,44 +327,82 @@ var configCheckCmd = &cobra.Command{ config() var hasError bool - if conf.API.URL.API != "" { - if err := api.New().CheckConfiguration(conf.API); err != nil { - fmt.Println(err) + if conf.API != nil && conf.API.URL.API != "" { + fmt.Printf("checking api configuration...\n") + if err := api.New().CheckConfiguration(*conf.API); err != nil { + fmt.Printf("api Configuration: %v\n", err) + hasError = true + } + } + + if conf.DatabaseMigrate != nil && conf.DatabaseMigrate.API.HTTP.URL != "" { + fmt.Printf("checking migrate configuration...\n") + if err := api.New().CheckConfiguration(*conf.DatabaseMigrate); err != nil { + fmt.Printf("migrate Configuration: %v\n", err) + hasError = true + } + } + + if conf.Hatchery != nil && conf.Hatchery.Local != nil && conf.Hatchery.Local.API.HTTP.URL != "" { + fmt.Printf("checking hatchery:local configuration...\n") + if err := local.New().CheckConfiguration(*conf.Hatchery.Local); err != nil { + fmt.Printf("hatchery:local Configuration: %v\n", err) + hasError = true + } + } + + if conf.Hatchery != nil && conf.Hatchery.Marathon != nil && conf.Hatchery.Marathon.API.HTTP.URL != "" { + fmt.Printf("checking hatchery:marathon configuration...\n") + if err := marathon.New().CheckConfiguration(*conf.Hatchery.Marathon); err != nil { + fmt.Printf("hatchery:marathon Configuration: %v\n", err) hasError = true } } - if conf.DatabaseMigrate.API.HTTP.URL != "" { - if err := api.New().CheckConfiguration(conf.DatabaseMigrate); err != nil { - fmt.Println(err) + if conf.Hatchery != nil && conf.Hatchery.Openstack != nil && conf.Hatchery.Openstack.API.HTTP.URL != "" { + fmt.Printf("checking hatchery:openstack configuration...\n") + if err := openstack.New().CheckConfiguration(*conf.Hatchery.Openstack); err != nil { + fmt.Printf("hatchery:openstack Configuration: %v\n", err) hasError = true } } - if conf.Hatchery.Local.API.HTTP.URL != "" { - if err := local.New().CheckConfiguration(conf.Hatchery.Local); err != nil { - fmt.Println(err) + if conf.Hatchery != nil && conf.Hatchery.Kubernetes != nil && conf.Hatchery.Kubernetes.API.HTTP.URL != "" { + fmt.Printf("checking hatchery:kubernetes configuration...\n") + if err := kubernetes.New().CheckConfiguration(*conf.Hatchery.Kubernetes); err != nil { + fmt.Printf("hatchery:kubernetes Configuration: %v\n", err) hasError = true } } - if conf.Hatchery.Marathon.API.HTTP.URL != "" { - if err := marathon.New().CheckConfiguration(conf.Hatchery.Marathon); err != nil { - fmt.Println(err) + if conf.Hatchery != nil && conf.Hatchery.Swarm != nil && conf.Hatchery.Swarm.API.HTTP.URL != "" { + fmt.Printf("checking hatchery:swarm configuration...\n") + if err := swarm.New().CheckConfiguration(*conf.Hatchery.Swarm); err != nil { + fmt.Printf("hatchery:swarm Configuration: %v\n", err) hasError = true } } - if conf.Hatchery.Openstack.API.HTTP.URL != "" { - if err := openstack.New().CheckConfiguration(conf.Hatchery.Openstack); err != nil { - fmt.Println(err) + if conf.Hatchery != nil && conf.Hatchery.VSphere != nil && conf.Hatchery.VSphere.API.HTTP.URL != "" { + fmt.Printf("checking hatchery:vsphere configuration...\n") + if err := vsphere.New().CheckConfiguration(*conf.Hatchery.VSphere); err != nil { + fmt.Printf("hatchery:vsphere Configuration: %v\n", err) hasError = true } } - if conf.Hatchery.Swarm.API.HTTP.URL != "" { - if err := swarm.New().CheckConfiguration(conf.Hatchery.Swarm); err != nil { - fmt.Println(err) + if conf.VCS != nil && conf.VCS.API.HTTP.URL != "" { + fmt.Printf("checking vcs configuration...\n") + if err := vcs.New().CheckConfiguration(*conf.VCS); err != nil { + fmt.Printf("vcs Configuration: %v\n", err) + hasError = true + } + } + + if conf.Hooks != nil && conf.Hooks.API.HTTP.URL != "" { + fmt.Printf("checking hooks configuration...\n") + if err := hooks.New().CheckConfiguration(*conf.Hooks); err != nil { + fmt.Printf("hooks Configuration: %v\n", err) hasError = true } } @@ -309,7 +445,7 @@ This component operates CDS VCS connectivity Start all of this with a single command: - $ engine start [api] [hatchery:local] [hatchery:marathon] [hatchery:openstack] [hatchery:swarm] [hatchery:vsphere] [hooks] [vcs] [repositories] + $ engine start [api] [hatchery:local] [hatchery:marathon] [hatchery:openstack] [hatchery:swarm] [hatchery:vsphere] [elasticsearch] [hooks] [vcs] [repositories] [migrate] All the services are using the same configuration file format. @@ -365,45 +501,101 @@ See $ engine config command for more details. } services := []serviceConf{} + if conf.Debug == nil { + conf.Debug = &DebugConfiguration{} + } + + if conf.Tracing == nil { + conf.Tracing = &observability.Configuration{} + } + names := []string{} for _, a := range args { fmt.Printf("Starting service %s\n", a) switch a { case "api": - services = append(services, serviceConf{arg: a, service: api.New(), cfg: conf.API}) + if conf.API == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: api.New(), cfg: *conf.API}) names = append(names, conf.API.Name) case "migrate": - services = append(services, serviceConf{arg: a, service: migrateservice.New(), cfg: conf.DatabaseMigrate}) + if conf.DatabaseMigrate == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: migrateservice.New(), cfg: *conf.DatabaseMigrate}) names = append(names, conf.DatabaseMigrate.Name) case "hatchery:local": - services = append(services, serviceConf{arg: a, service: local.New(), cfg: conf.Hatchery.Local}) + if conf.Hatchery.Local == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: local.New(), cfg: *conf.Hatchery.Local}) names = append(names, conf.Hatchery.Local.Name) case "hatchery:kubernetes": - services = append(services, serviceConf{arg: a, service: kubernetes.New(), cfg: conf.Hatchery.Kubernetes}) + if conf.Hatchery.Kubernetes == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: kubernetes.New(), cfg: *conf.Hatchery.Kubernetes}) names = append(names, conf.Hatchery.Kubernetes.Name) case "hatchery:marathon": - services = append(services, serviceConf{arg: a, service: marathon.New(), cfg: conf.Hatchery.Marathon}) + if conf.Hatchery.Marathon == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: marathon.New(), cfg: *conf.Hatchery.Marathon}) names = append(names, conf.Hatchery.Marathon.Name) case "hatchery:openstack": - services = append(services, serviceConf{arg: a, service: openstack.New(), cfg: conf.Hatchery.Openstack}) + if conf.Hatchery.Openstack == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: openstack.New(), cfg: *conf.Hatchery.Openstack}) names = append(names, conf.Hatchery.Openstack.Name) case "hatchery:swarm": - services = append(services, serviceConf{arg: a, service: swarm.New(), cfg: conf.Hatchery.Swarm}) + if conf.Hatchery.Swarm == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: swarm.New(), cfg: *conf.Hatchery.Swarm}) names = append(names, conf.Hatchery.Swarm.Name) case "hatchery:vsphere": - services = append(services, serviceConf{arg: a, service: vsphere.New(), cfg: conf.Hatchery.VSphere}) + if conf.Hatchery.VSphere == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: vsphere.New(), cfg: *conf.Hatchery.VSphere}) names = append(names, conf.Hatchery.VSphere.Name) case "hooks": - services = append(services, serviceConf{arg: a, service: hooks.New(), cfg: conf.Hooks}) + if conf.Hooks == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: hooks.New(), cfg: *conf.Hooks}) names = append(names, conf.Hooks.Name) case "vcs": - services = append(services, serviceConf{arg: a, service: vcs.New(), cfg: conf.VCS}) + if conf.VCS == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: vcs.New(), cfg: *conf.VCS}) names = append(names, conf.VCS.Name) case "repositories": - services = append(services, serviceConf{arg: a, service: repositories.New(), cfg: conf.Repositories}) + if conf.Repositories == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: repositories.New(), cfg: *conf.Repositories}) names = append(names, conf.Repositories.Name) case "elasticsearch": - services = append(services, serviceConf{arg: a, service: elasticsearch.New(), cfg: conf.ElasticSearch}) + if conf.ElasticSearch == nil { + fmt.Printf("Error: missing configuration for service '%s'\n", a) + os.Exit(1) + } + services = append(services, serviceConf{arg: a, service: elasticsearch.New(), cfg: *conf.ElasticSearch}) names = append(names, conf.ElasticSearch.Name) default: fmt.Printf("Error: service '%s' unknown\n", a) @@ -437,7 +629,7 @@ See $ engine config command for more details. } // Initialiaze tracing - if err := observability.Init(conf.Tracing, "cds-"+s.arg); err != nil { + if err := observability.Init(*conf.Tracing, "cds-"+s.arg); err != nil { sdk.Exit("Unable to start tracing exporter: %v", err) } } diff --git a/engine/types.go b/engine/types.go index 0c6ac87c6e..4ae8bd5d12 100644 --- a/engine/types.go +++ b/engine/types.go @@ -34,25 +34,31 @@ type Configuration struct { ExtraValue string `toml:"extraValue" comment:"value for extraKey field. For many keys: valueaaa,valuebbb"` } `toml:"graylog"` } `toml:"log" comment:"#####################\n CDS Logs Settings \n####################"` - Debug struct { - Enable bool `toml:"enable" default:"false" comment:"allow debugging with gops"` - RemoteDebugURL string `toml:"remoteDebugURL" comment:"start a gops agent on specified URL. Ex: localhost:9999"` - } `toml:"debug" comment:"#####################\n Debug with gops \n####################"` - API api.Configuration `toml:"api" comment:"#####################\n API Configuration \n####################"` - Hatchery struct { - Local local.HatcheryConfiguration `toml:"local" comment:"Hatchery Local."` - Kubernetes kubernetes.HatcheryConfiguration `toml:"kubernetes" comment:"Hatchery Kubernetes."` - Marathon marathon.HatcheryConfiguration `toml:"marathon" comment:"Hatchery Marathon."` - Openstack openstack.HatcheryConfiguration `toml:"openstack" comment:"Hatchery OpenStack. Doc: https://ovh.github.io/cds/advanced/advanced.hatcheries.openstack/"` - Swarm swarm.HatcheryConfiguration `toml:"swarm" comment:"Hatchery Swarm. Doc: https://ovh.github.io/cds/advanced/advanced.hatcheries.swarm/"` - VSphere vsphere.HatcheryConfiguration `toml:"vsphere" comment:"Hatchery VShpere. Doc: https://ovh.github.io/cds/advanced/advanced.hatcheries.vsphere/"` - } `toml:"hatchery"` - Hooks hooks.Configuration `toml:"hooks" comment:"######################\n CDS Hooks Settings \n######################"` - VCS vcs.Configuration `toml:"vcs" comment:"######################\n CDS VCS Settings \n######################"` - Repositories repositories.Configuration `toml:"repositories" comment:"######################\n CDS Repositories Settings \n######################"` - ElasticSearch elasticsearch.Configuration `toml:"elasticsearch" comment:"######################\n CDS ElasticSearch Settings \n This is use for CDS timeline and is optional\n######################"` - DatabaseMigrate migrateservice.Configuration `toml:"databaseMigrate" comment:"######################\n CDS DB Migrate Service Settings \n######################"` - Tracing observability.Configuration `toml:"tracing" comment:"###########################\n CDS Tracing Settings \n##########################"` + Debug *DebugConfiguration `toml:"debug" comment:"#####################\n Debug with gops \n####################"` + API *api.Configuration `toml:"api" comment:"#####################\n API Configuration \n####################"` + Hatchery *HatcheryConfiguration `toml:"hatchery"` + Hooks *hooks.Configuration `toml:"hooks" comment:"######################\n CDS Hooks Settings \n######################"` + VCS *vcs.Configuration `toml:"vcs" comment:"######################\n CDS VCS Settings \n######################"` + Repositories *repositories.Configuration `toml:"repositories" comment:"######################\n CDS Repositories Settings \n######################"` + ElasticSearch *elasticsearch.Configuration `toml:"elasticsearch" comment:"######################\n CDS ElasticSearch Settings \n This is use for CDS timeline and is optional\n######################"` + DatabaseMigrate *migrateservice.Configuration `toml:"databaseMigrate" comment:"######################\n CDS DB Migrate Service Settings \n######################"` + Tracing *observability.Configuration `toml:"tracing" comment:"###########################\n CDS Tracing Settings \n##########################"` +} + +// DebugConfiguration contains debug configuration +type DebugConfiguration struct { + Enable bool `toml:"enable" default:"false" comment:"allow debugging with gops"` + RemoteDebugURL string `toml:"remoteDebugURL" comment:"start a gops agent on specified URL. Ex: localhost:9999"` +} + +// HatcheryConfiguration contains subsection of Hatchery configuration +type HatcheryConfiguration struct { + Local *local.HatcheryConfiguration `toml:"local" comment:"Hatchery Local."` + Kubernetes *kubernetes.HatcheryConfiguration `toml:"kubernetes" comment:"Hatchery Kubernetes."` + Marathon *marathon.HatcheryConfiguration `toml:"marathon" comment:"Hatchery Marathon."` + Openstack *openstack.HatcheryConfiguration `toml:"openstack" comment:"Hatchery OpenStack. Doc: https://ovh.github.io/cds/advanced/advanced.hatcheries.openstack/"` + Swarm *swarm.HatcheryConfiguration `toml:"swarm" comment:"Hatchery Swarm. Doc: https://ovh.github.io/cds/advanced/advanced.hatcheries.swarm/"` + VSphere *vsphere.HatcheryConfiguration `toml:"vsphere" comment:"Hatchery VShpere. Doc: https://ovh.github.io/cds/advanced/advanced.hatcheries.vsphere/"` } // AsEnvVariables returns the object attributes as env variables. It used for configuration structs