diff --git a/docker-compose.yml b/docker-compose.yml index c1a9aa04a6..2288582ca2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,68 +57,68 @@ services: if [ ! -f /app/conf/conf.toml ]; then /app/cds-engine-linux-amd64 config new api cdn ui hooks repositories vcs elasticsearch hatchery:local hatchery:swarm > /app/conf/conf.toml; else - echo '/!\ Config file already exists'; + echo '/!\\ Config file already exists'; fi; chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/cdn-buffer /app/cdn-storage; /app/cds-engine-linux-amd64 config new > /app/conf/conf.toml; mkdir -p /app/artifacts /app/repositories /app/hatchery-local; chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/hatchery-local /app/cdn-buffer /app/cdn-storage; - /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml \ - api.artifact.local.baseDirectory=/app/artifacts \ - api.cache.redis.host=cds-cache:6379 \ - api.cache.redis.password=cds \ - api.database.host=cds-db \ - api.download.directory=/app \ - api.log.level=info \ - cdn.cache.redis.host=cds-cache:6379 \ - cdn.cache.redis.password=cds \ - cdn.storageUnits.buffers.redis.redis.host=cds-cache:6379 \ - cdn.storageUnits.buffers.redis.redis.password=cds \ - cdn.storageUnits.buffers.local-buffer.local.path=/app/cdn-buffer \ - cdn.storageUnits.storages.local.local.path=/app/cdn-storage \ - cdn.database.host=cds-db \ - cdn.publicTCP=${HOSTNAME}:8090 \ - cdn.api.http.url=http://cds-api:8081 \ - cdn.URL=http://cds-cdn:8089 \ - hatchery.local.commonConfiguration.url=http://cds-hatchery-local:8086 \ - hatchery.local.commonConfiguration.api.http.url=http://cds-api:8081 \ - hatchery.local.basedir=/app/hatchery-local \ - hatchery.swarm.commonConfiguration.url=http://cds-hatchery-swarm:8086 \ - hatchery.swarm.commonConfiguration.api.http.url=http://${HOSTNAME}:8081 \ - hatchery.swarm.ratioService=50 \ - hatchery.swarm.dockerEngines.sample-docker-engine.host=tcp://dockerhost:2375 \ - hatchery.swarm.dockerEngines.sample-docker-engine.maxContainers=4 \ - hooks.url=http://cds-hooks:8083 \ - hooks.api.http.url=http://cds-api:8081 \ - hooks.cache.redis.host=cds-cache:6379 \ - hooks.cache.redis.password=cds \ - vcs.URL=http://cds-vcs:8084 \ - vcs.api.http.url=http://cds-api:8081 \ - vcs.cache.redis.host=cds-cache:6379 \ - vcs.cache.redis.password=cds \ - repositories.URL=http://cds-repositories:8085 \ - repositories.basedir=/app/repositories \ - repositories.api.http.url=http://cds-api:8081 \ - repositories.cache.redis.host=cds-cache:6379 \ - repositories.cache.redis.password=cds \ - elasticsearch.URL=http://cds-elasticsearch:8088 \ - elasticsearch.api.http.url=http://cds-api:8081 \ - elasticsearch.name=cds-elasticsearch \ - elasticsearch.elasticsearch.indexEvents=cds-index-events \ - elasticsearch.elasticsearch.indexMetrics=cds-index-metrics \ - elasticsearch.elasticsearch.url=http://elasticsearch:9200 \ - ui.enableServiceProxy=true \ - ui.api.http.url=http://cds-api:8081 \ - ui.hooksURL=http://cds-hooks:8083 \ - ui.cdnURL=http://cds-cdn:8089; - echo ''; - echo '/!\ Please run the export command BELOW, this will be useful to create admin user'; - echo ''; - TOKEN=`/app/cds-engine-linux-amd64 config init-token --config /app/conf/conf.toml`; - echo 'export INIT_TOKEN='$$TOKEN; - echo ''; - echo '/!\ Please run the export command ABOVE, this will be useful to create admin user'; - echo '';" + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.artifact.local.baseDirectory=/app/artifacts; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.cache.redis.host=cds-cache:6379; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.cache.redis.password=cds; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.database.host=cds-db; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.download.directory=/app; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.log.level=info; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.api.http.url=http://cds-api:8081; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.cache.redis.host=cds-cache:6379; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.cache.redis.password=cds; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.database.host=cds-db; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.publicTCP=${HOSTNAME}:8090; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.buffers.local-buffer.local.path=/app/cdn-buffer; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.buffers.redis.redis.host=cds-cache:6379; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.buffers.redis.redis.password=cds; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.storages.local.local.path=/app/cdn-storage; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.URL=http://cds-cdn:8089; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.api.http.url=http://cds-api:8081; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.elasticsearch.indexEvents=cds-index-events; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.elasticsearch.indexMetrics=cds-index-metrics; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.elasticsearch.url=http://elasticsearch:9200; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.name=cds-elasticsearch; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.URL=http://cds-elasticsearch:8088; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.local.basedir=/app/hatchery-local; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.local.commonConfiguration.api.http.url=http://cds-api:8081; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.local.commonConfiguration.url=http://cds-hatchery-local:8086; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.commonConfiguration.api.http.url=http://cds-api:8081; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.commonConfiguration.url=http://cds-hatchery-swarm:8086; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.commonConfiguration.provision.workerApiHttp.url=http://$HOSTNAME:8081; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.dockerEngines.sample-docker-engine.host=tcp://dockerhost:2375; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.dockerEngines.sample-docker-engine.maxContainers=4; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.ratioService=50; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.api.http.url=http://cds-api:8081; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.cache.redis.host=cds-cache:6379; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.cache.redis.password=cds; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.url=http://cds-hooks:8083; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.api.http.url=http://cds-api:8081; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.basedir=/app/repositories; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.cache.redis.host=cds-cache:6379; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.cache.redis.password=cds; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.URL=http://cds-repositories:8085; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.api.http.url=http://cds-api:8081; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.cdnURL=http://cds-cdn:8089; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.enableServiceProxy=true; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.hooksURL=http://cds-hooks:8083; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.api.http.url=http://cds-api:8081; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.cache.redis.host=cds-cache:6379; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.cache.redis.password=cds; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.URL=http://cds-vcs:8084; + echo ''; + echo '/!\\ Please run the export command BELOW, this will be useful to create admin user'; + echo ''; + TOKEN=`/app/cds-engine-linux-amd64 config init-token --config /app/conf/conf.toml`; + echo 'export INIT_TOKEN='$$TOKEN; + echo ''; + echo '/!\\ Please run the export command ABOVE, this will be useful to create admin user'; + echo '';" user: root volumes: - cds-conf-volume:/app/conf @@ -272,8 +272,7 @@ services: image: ${CDS_DOCKER_IMAGE:-ovhcom/cds-engine:latest} command: > sh -c " - /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml \ - ${CDS_EDIT_CONFIG}"; + /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ${CDS_EDIT_CONFIG}"; volumes: - cds-conf-volume:/app/conf diff --git a/docs/content/hosting/cds_services.md b/docs/content/hosting/cds_services.md index 115661bd01..aa665d5603 100644 --- a/docs/content/hosting/cds_services.md +++ b/docs/content/hosting/cds_services.md @@ -18,7 +18,7 @@ A CDS µService is a process that can be launch by the `engine` binary. - **api**: this is the main µService. - Each `api` must shared the same PostgreSQL and redis databases. All other CDS µServices have to communicate with `api`. - The database user does not need to have the admin rights to create / alter tables. -- **cdn**: this is the main µService. +- **cdn**: the `cdn` service is dedicated to receive and store CDS's logs and artifacts. - Each `cdn` must shared the same PostgreSQL and redis databases. - The database user does not need to have the admin rights to create / alter tables. - **ui**: the `ui` service serves the CDS UI static files. diff --git a/docs/content/hosting/ready-to-run/docker-compose/docker-compose.md b/docs/content/hosting/ready-to-run/docker-compose/docker-compose.md index f3a619024e..6dcdd8f0af 100644 --- a/docs/content/hosting/ready-to-run/docker-compose/docker-compose.md +++ b/docs/content/hosting/ready-to-run/docker-compose/docker-compose.md @@ -7,6 +7,15 @@ card: ## Run with Docker-Compose +{{% notice warning %}} + +This tutorial allows you to start a CDS locally quickly. + +* Do not use this docker-compose installation to run it in production without modification. +* For production purpose, please read [CDS Services]({{< relref "../../cds_services" >}}) + +{{% /notice %}} + The [docker-compose.yml](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/docker-compose.yml) contains: - cds-db service with a PostgreSQL @@ -27,7 +36,7 @@ Docker compose is very convenient to launch CDS for testing it. But this is not ## How to run ```bash -$ mkdir /tmp/cdstest && cd /tmp/cdstest && mkdir -p tools/smtpmock +$ mkdir /tmp/cdstest && cd /tmp/cdstest $ curl https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/docker-compose.yml -o docker-compose.yml $ export HOSTNAME=$(hostname) $ export CDS_DOCKER_IMAGE=ovhcom/cds-engine:{{< param "version" "latest" >}} @@ -57,9 +66,11 @@ $ TOKEN_CMD=$(docker logs $(docker-compose ps -q cds-prepare) | grep INIT_TOKEN) # if you have this error: "command too long: export INIT_TOKEN=....", # you can manually execute the command "export INIT_TOKEN=...." -# create user -$ curl 'http://localhost:8081/download/cdsctl/linux/amd64?variant=nokeychain' -o cdsctl +# download cdsctl +# on linux: $ curl 'http://localhost:8081/download/cdsctl/linux/amd64?variant=nokeychain' -o cdsctl # on OSX: $ curl 'http://localhost:8081/download/cdsctl/darwin/amd64?variant=nokeychain' -o cdsctl + +# create user $ chmod +x cdsctl $ ./cdsctl signup --api-url http://localhost:8081 --email admin@localhost.local --username admin --fullname admin # enter a strong password @@ -101,6 +112,15 @@ Workflow MyFirstWorkflow #1 has been launched http://localhost:8080/project/DEMO/workflow/MyFirstWorkflow/run/1 ``` +{{% notice warning %}} + +Important: the service is exposed without **https**. + +* Do not use this docker-compose installation to run it in production without modification. +* Safari don't manage secured cookie on localhost, please use another browser to avoid authentication error on your browser + +{{% /notice %}} + - Login on WebUI Open a browser on http://localhost:8080/account/signup, then login with the user `admin`, @@ -120,6 +140,30 @@ The build pipeline contains two stages, with only one job in each stage ![Build Pipeline](/images/ready_to_run_docker_compose_build_pipeline.png) +## If the job does not start + +The worker model used in this tutorial is a docker image golang. CDS will run jobs only if the worker model is *registered*. + +So, if the first job of the workflow stay in *Waiting* status, you can check if you worker model is well registered. +Go on http://localhost:8080/settings/worker-model/shared.infra/go-official-1.17 , you must have the flag `need registration: false`. + +If the flag is `true`, you can check the swarm hatchery logs: + +```bash +$ docker-compose logs -f cds-hatchery-swarm +``` + +When a worker (so, a container) is starting, this container communicates with the api with the url `http://$HOSTNAME:8081` to download the worker binary. +This URL is using your `$HOSTNAME`, but perhaps that this can't be used on your docker installation from a container. If the container can't communicate to the api with this url, you can update it: + +```bash +# you can replace $HOSTNAME by your IP on local network. Example: 192.168.xxx.xxx +$ export CDS_EDIT_CONFIG="hatchery.swarm.commonConfiguration.provision.workerApiHttp.url=http://192.168.xxx.xxx:8081 " +$ docker-compose up cds-edit-config +$ docker-compose restart cds-hatchery-swarm +``` + + ## Setup connection with a VCS ```bash diff --git a/engine/service/types.go b/engine/service/types.go index 868e80b54a..a0550e3e53 100644 --- a/engine/service/types.go +++ b/engine/service/types.go @@ -55,7 +55,7 @@ type HatcheryCommonConfiguration struct { Region string `toml:"region" default:"" comment:"region of this hatchery - optional. With a free text as 'myregion', user can set a prerequisite 'region' with value 'myregion' on CDS Job" json:"region"` IgnoreJobWithNoRegion bool `toml:"ignoreJobWithNoRegion" default:"false" comment:"Ignore job without a region prerequisite if ignoreJobWithNoRegion=true" json:"ignoreJobWithNoRegion"` WorkerAPIHTTP struct { - URL string `toml:"url" default:"http://localhost:8081" commented:"true" comment:"CDS API URL for worker, let empty or commented to use the same URL that is used by the Hatchery" json:"url"` + URL string `toml:"url" default:"" commented:"false" comment:"CDS API URL for worker, let empty or commented to use the same URL that is used by the Hatchery. Example: http://localhost:8081" json:"url"` Insecure bool `toml:"insecure" default:"false" commented:"true" comment:"sslInsecureSkipVerify, set to true if you use a self-signed SSL on CDS API" json:"insecure"` } `toml:"workerApiHttp" json:"workerApiHttp"` WorkerBasedir string `toml:"workerBasedir" commented:"true" comment:"Worker Basedir" json:"workerBasedir"`