From 8d175a02173f21e2c94c3ce2e7d124a988a3bfd1 Mon Sep 17 00:00:00 2001
From: Yvonnick Esnault <yvonnick.esnault@corp.ovh.com>
Date: Sat, 18 Dec 2021 17:27:51 +0100
Subject: [PATCH] fix: docker-compose upgrade (#6043)

* fix: docker-compose upgrade

Signed-off-by: Yvonnick Esnault <yvonnick@esnau.lt>
---
 docker-compose.yml                            | 117 +++++++++---------
 docs/content/hosting/cds_services.md          |   2 +-
 .../docker-compose/docker-compose.md          |  50 +++++++-
 engine/service/types.go                       |   2 +-
 4 files changed, 107 insertions(+), 64 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index c1a9aa04a6..2288582ca2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -57,68 +57,68 @@ services:
         if [ ! -f /app/conf/conf.toml ]; then
           /app/cds-engine-linux-amd64 config new api cdn ui hooks repositories vcs elasticsearch hatchery:local hatchery:swarm > /app/conf/conf.toml;
         else
-          echo '/!\ Config file already exists';
+          echo '/!\\ Config file already exists';
         fi;
         chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/cdn-buffer /app/cdn-storage;
         /app/cds-engine-linux-amd64 config new > /app/conf/conf.toml;
         mkdir -p /app/artifacts /app/repositories /app/hatchery-local;
         chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/hatchery-local /app/cdn-buffer /app/cdn-storage;
-        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml \
-            api.artifact.local.baseDirectory=/app/artifacts \
-            api.cache.redis.host=cds-cache:6379 \
-            api.cache.redis.password=cds \
-            api.database.host=cds-db \
-            api.download.directory=/app \
-            api.log.level=info \
-            cdn.cache.redis.host=cds-cache:6379 \
-            cdn.cache.redis.password=cds \
-            cdn.storageUnits.buffers.redis.redis.host=cds-cache:6379 \
-            cdn.storageUnits.buffers.redis.redis.password=cds \
-            cdn.storageUnits.buffers.local-buffer.local.path=/app/cdn-buffer \
-            cdn.storageUnits.storages.local.local.path=/app/cdn-storage \
-            cdn.database.host=cds-db \
-            cdn.publicTCP=${HOSTNAME}:8090 \
-            cdn.api.http.url=http://cds-api:8081 \
-            cdn.URL=http://cds-cdn:8089 \
-            hatchery.local.commonConfiguration.url=http://cds-hatchery-local:8086 \
-            hatchery.local.commonConfiguration.api.http.url=http://cds-api:8081 \
-            hatchery.local.basedir=/app/hatchery-local \
-            hatchery.swarm.commonConfiguration.url=http://cds-hatchery-swarm:8086 \
-            hatchery.swarm.commonConfiguration.api.http.url=http://${HOSTNAME}:8081 \
-            hatchery.swarm.ratioService=50 \
-            hatchery.swarm.dockerEngines.sample-docker-engine.host=tcp://dockerhost:2375 \
-            hatchery.swarm.dockerEngines.sample-docker-engine.maxContainers=4 \
-            hooks.url=http://cds-hooks:8083 \
-            hooks.api.http.url=http://cds-api:8081 \
-            hooks.cache.redis.host=cds-cache:6379 \
-            hooks.cache.redis.password=cds \
-            vcs.URL=http://cds-vcs:8084 \
-            vcs.api.http.url=http://cds-api:8081 \
-            vcs.cache.redis.host=cds-cache:6379 \
-            vcs.cache.redis.password=cds \
-            repositories.URL=http://cds-repositories:8085 \
-            repositories.basedir=/app/repositories \
-            repositories.api.http.url=http://cds-api:8081 \
-            repositories.cache.redis.host=cds-cache:6379 \
-            repositories.cache.redis.password=cds \
-            elasticsearch.URL=http://cds-elasticsearch:8088 \
-            elasticsearch.api.http.url=http://cds-api:8081 \
-            elasticsearch.name=cds-elasticsearch \
-            elasticsearch.elasticsearch.indexEvents=cds-index-events \
-            elasticsearch.elasticsearch.indexMetrics=cds-index-metrics \
-            elasticsearch.elasticsearch.url=http://elasticsearch:9200 \
-            ui.enableServiceProxy=true \
-            ui.api.http.url=http://cds-api:8081 \
-            ui.hooksURL=http://cds-hooks:8083 \
-            ui.cdnURL=http://cds-cdn:8089;
-            echo '';
-            echo '/!\ Please run the export command BELOW, this will be useful to create admin user';
-            echo '';
-            TOKEN=`/app/cds-engine-linux-amd64 config init-token --config /app/conf/conf.toml`;
-            echo 'export INIT_TOKEN='$$TOKEN;
-            echo '';
-            echo '/!\ Please run the export command ABOVE, this will be useful to create admin user';
-            echo '';"
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.artifact.local.baseDirectory=/app/artifacts;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.cache.redis.host=cds-cache:6379;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.cache.redis.password=cds;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.database.host=cds-db;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.download.directory=/app;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.log.level=info;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.api.http.url=http://cds-api:8081;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.cache.redis.host=cds-cache:6379;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.cache.redis.password=cds;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.database.host=cds-db;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.publicTCP=${HOSTNAME}:8090;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.buffers.local-buffer.local.path=/app/cdn-buffer;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.buffers.redis.redis.host=cds-cache:6379;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.buffers.redis.redis.password=cds;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.storages.local.local.path=/app/cdn-storage;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.URL=http://cds-cdn:8089;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.api.http.url=http://cds-api:8081;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.elasticsearch.indexEvents=cds-index-events;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.elasticsearch.indexMetrics=cds-index-metrics;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.elasticsearch.url=http://elasticsearch:9200;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.name=cds-elasticsearch;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.URL=http://cds-elasticsearch:8088;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.local.basedir=/app/hatchery-local;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.local.commonConfiguration.api.http.url=http://cds-api:8081;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.local.commonConfiguration.url=http://cds-hatchery-local:8086;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.commonConfiguration.api.http.url=http://cds-api:8081;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.commonConfiguration.url=http://cds-hatchery-swarm:8086;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.commonConfiguration.provision.workerApiHttp.url=http://$HOSTNAME:8081;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.dockerEngines.sample-docker-engine.host=tcp://dockerhost:2375;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.dockerEngines.sample-docker-engine.maxContainers=4;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.ratioService=50;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.api.http.url=http://cds-api:8081;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.cache.redis.host=cds-cache:6379;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.cache.redis.password=cds;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.url=http://cds-hooks:8083;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.api.http.url=http://cds-api:8081;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.basedir=/app/repositories;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.cache.redis.host=cds-cache:6379;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.cache.redis.password=cds;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.URL=http://cds-repositories:8085;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.api.http.url=http://cds-api:8081;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.cdnURL=http://cds-cdn:8089;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.enableServiceProxy=true;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.hooksURL=http://cds-hooks:8083;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.api.http.url=http://cds-api:8081;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.cache.redis.host=cds-cache:6379;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.cache.redis.password=cds;
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.URL=http://cds-vcs:8084;
+        echo '';
+        echo '/!\\ Please run the export command BELOW, this will be useful to create admin user';
+        echo '';
+        TOKEN=`/app/cds-engine-linux-amd64 config init-token --config /app/conf/conf.toml`;
+        echo 'export INIT_TOKEN='$$TOKEN;
+        echo '';
+        echo '/!\\ Please run the export command ABOVE, this will be useful to create admin user';
+        echo '';"
     user: root
     volumes:
       - cds-conf-volume:/app/conf
@@ -272,8 +272,7 @@ services:
     image: ${CDS_DOCKER_IMAGE:-ovhcom/cds-engine:latest}
     command: >
         sh -c "
-        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml \
-            ${CDS_EDIT_CONFIG}";
+        /app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ${CDS_EDIT_CONFIG}";
     volumes:
       - cds-conf-volume:/app/conf
 
diff --git a/docs/content/hosting/cds_services.md b/docs/content/hosting/cds_services.md
index 115661bd01..aa665d5603 100644
--- a/docs/content/hosting/cds_services.md
+++ b/docs/content/hosting/cds_services.md
@@ -18,7 +18,7 @@ A CDS µService is a process that can be launch by the `engine` binary.
 - **api**: this is the main µService. 
   - Each `api` must shared the same PostgreSQL and redis databases. All other CDS µServices have to communicate with `api`.
   - The database user does not need to have the admin rights to create / alter tables.
-- **cdn**: this is the main µService. 
+- **cdn**: the `cdn` service is dedicated to receive and store CDS's logs and artifacts. 
   - Each `cdn` must shared the same PostgreSQL and redis databases.
   - The database user does not need to have the admin rights to create / alter tables.
 - **ui**: the `ui` service serves the CDS UI static files.
diff --git a/docs/content/hosting/ready-to-run/docker-compose/docker-compose.md b/docs/content/hosting/ready-to-run/docker-compose/docker-compose.md
index f3a619024e..6dcdd8f0af 100644
--- a/docs/content/hosting/ready-to-run/docker-compose/docker-compose.md
+++ b/docs/content/hosting/ready-to-run/docker-compose/docker-compose.md
@@ -7,6 +7,15 @@ card:
 
 ## Run with Docker-Compose
 
+{{% notice warning %}}
+
+This tutorial allows you to start a CDS locally quickly.
+
+* Do not use this docker-compose installation to run it in production without modification.
+* For production purpose, please read [CDS Services]({{< relref "../../cds_services" >}})
+
+{{% /notice %}}
+
 The [docker-compose.yml](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/docker-compose.yml) contains:
 
 - cds-db service with a PostgreSQL
@@ -27,7 +36,7 @@ Docker compose is very convenient to launch CDS for testing it. But this is not
 ## How to run
 
 ```bash
-$ mkdir /tmp/cdstest && cd /tmp/cdstest && mkdir -p tools/smtpmock
+$ mkdir /tmp/cdstest && cd /tmp/cdstest
 $ curl https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/docker-compose.yml -o docker-compose.yml
 $ export HOSTNAME=$(hostname)
 $ export CDS_DOCKER_IMAGE=ovhcom/cds-engine:{{< param "version" "latest" >}}
@@ -57,9 +66,11 @@ $ TOKEN_CMD=$(docker logs $(docker-compose ps -q cds-prepare) | grep INIT_TOKEN)
 # if you have this error:  "command too long: export INIT_TOKEN=....",
 # you can manually execute the command "export INIT_TOKEN=...."
 
-# create user
-$ curl 'http://localhost:8081/download/cdsctl/linux/amd64?variant=nokeychain' -o cdsctl
+# download cdsctl
+# on linux: $ curl 'http://localhost:8081/download/cdsctl/linux/amd64?variant=nokeychain' -o cdsctl
 # on OSX: $ curl 'http://localhost:8081/download/cdsctl/darwin/amd64?variant=nokeychain' -o cdsctl
+
+# create user
 $ chmod +x cdsctl
 $ ./cdsctl signup --api-url http://localhost:8081 --email admin@localhost.local --username admin --fullname admin
 # enter a strong password
@@ -101,6 +112,15 @@ Workflow MyFirstWorkflow #1 has been launched
 http://localhost:8080/project/DEMO/workflow/MyFirstWorkflow/run/1
 ```
 
+{{% notice warning %}}
+
+Important: the service is exposed without **https**.
+
+* Do not use this docker-compose installation to run it in production without modification.
+* Safari don't manage secured cookie on localhost, please use another browser to avoid authentication error on your browser
+
+{{% /notice %}}
+
 - Login on WebUI
 
 Open a browser on http://localhost:8080/account/signup, then login with the user `admin`,
@@ -120,6 +140,30 @@ The build pipeline contains two stages, with only one job in each stage
 
 ![Build Pipeline](/images/ready_to_run_docker_compose_build_pipeline.png)
 
+## If the job does not start
+
+The worker model used in this tutorial is a docker image golang. CDS will run jobs only if the worker model is *registered*.
+
+So, if the first job of the workflow stay in *Waiting* status, you can check if you worker model is well registered.
+Go on http://localhost:8080/settings/worker-model/shared.infra/go-official-1.17 , you must have the flag `need registration: false`.
+
+If the flag is `true`, you can check the swarm hatchery logs:
+
+```bash
+$ docker-compose logs -f cds-hatchery-swarm
+```
+
+When a worker (so, a container) is starting, this container communicates with the api with the url `http://$HOSTNAME:8081` to download the worker binary.
+This URL is using your `$HOSTNAME`, but perhaps that this can't be used on your docker installation from a container. If the container can't communicate to the api with this url, you can update it:
+
+```bash
+# you can replace $HOSTNAME by your IP on local network. Example: 192.168.xxx.xxx
+$ export CDS_EDIT_CONFIG="hatchery.swarm.commonConfiguration.provision.workerApiHttp.url=http://192.168.xxx.xxx:8081 "
+$ docker-compose up cds-edit-config
+$ docker-compose restart cds-hatchery-swarm
+```
+
+
 ## Setup connection with a VCS
 
 ```bash
diff --git a/engine/service/types.go b/engine/service/types.go
index 868e80b54a..a0550e3e53 100644
--- a/engine/service/types.go
+++ b/engine/service/types.go
@@ -55,7 +55,7 @@ type HatcheryCommonConfiguration struct {
 		Region                    string   `toml:"region" default:"" comment:"region of this hatchery - optional. With a free text as 'myregion', user can set a prerequisite 'region' with value 'myregion' on CDS Job" json:"region"`
 		IgnoreJobWithNoRegion     bool     `toml:"ignoreJobWithNoRegion" default:"false" comment:"Ignore job without a region prerequisite if ignoreJobWithNoRegion=true" json:"ignoreJobWithNoRegion"`
 		WorkerAPIHTTP             struct {
-			URL      string `toml:"url" default:"http://localhost:8081" commented:"true" comment:"CDS API URL for worker, let empty or commented to use the same URL that is used by the Hatchery" json:"url"`
+			URL      string `toml:"url" default:"" commented:"false" comment:"CDS API URL for worker, let empty or commented to use the same URL that is used by the Hatchery. Example: http://localhost:8081" json:"url"`
 			Insecure bool   `toml:"insecure" default:"false" commented:"true" comment:"sslInsecureSkipVerify, set to true if you use a self-signed SSL on CDS API" json:"insecure"`
 		} `toml:"workerApiHttp" json:"workerApiHttp"`
 		WorkerBasedir     string `toml:"workerBasedir" commented:"true" comment:"Worker Basedir" json:"workerBasedir"`