Skip to content

Commit

Permalink
fix: docker-compose upgrade (#6043)
Browse files Browse the repository at this point in the history
* fix: docker-compose upgrade

Signed-off-by: Yvonnick Esnault <[email protected]>
  • Loading branch information
yesnault authored Dec 18, 2021
1 parent 139ea40 commit 8d175a0
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 64 deletions.
117 changes: 58 additions & 59 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,68 +57,68 @@ services:
if [ ! -f /app/conf/conf.toml ]; then
/app/cds-engine-linux-amd64 config new api cdn ui hooks repositories vcs elasticsearch hatchery:local hatchery:swarm > /app/conf/conf.toml;
else
echo '/!\ Config file already exists';
echo '/!\\ Config file already exists';
fi;
chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/cdn-buffer /app/cdn-storage;
/app/cds-engine-linux-amd64 config new > /app/conf/conf.toml;
mkdir -p /app/artifacts /app/repositories /app/hatchery-local;
chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/hatchery-local /app/cdn-buffer /app/cdn-storage;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml \
api.artifact.local.baseDirectory=/app/artifacts \
api.cache.redis.host=cds-cache:6379 \
api.cache.redis.password=cds \
api.database.host=cds-db \
api.download.directory=/app \
api.log.level=info \
cdn.cache.redis.host=cds-cache:6379 \
cdn.cache.redis.password=cds \
cdn.storageUnits.buffers.redis.redis.host=cds-cache:6379 \
cdn.storageUnits.buffers.redis.redis.password=cds \
cdn.storageUnits.buffers.local-buffer.local.path=/app/cdn-buffer \
cdn.storageUnits.storages.local.local.path=/app/cdn-storage \
cdn.database.host=cds-db \
cdn.publicTCP=${HOSTNAME}:8090 \
cdn.api.http.url=http://cds-api:8081 \
cdn.URL=http://cds-cdn:8089 \
hatchery.local.commonConfiguration.url=http://cds-hatchery-local:8086 \
hatchery.local.commonConfiguration.api.http.url=http://cds-api:8081 \
hatchery.local.basedir=/app/hatchery-local \
hatchery.swarm.commonConfiguration.url=http://cds-hatchery-swarm:8086 \
hatchery.swarm.commonConfiguration.api.http.url=http://${HOSTNAME}:8081 \
hatchery.swarm.ratioService=50 \
hatchery.swarm.dockerEngines.sample-docker-engine.host=tcp://dockerhost:2375 \
hatchery.swarm.dockerEngines.sample-docker-engine.maxContainers=4 \
hooks.url=http://cds-hooks:8083 \
hooks.api.http.url=http://cds-api:8081 \
hooks.cache.redis.host=cds-cache:6379 \
hooks.cache.redis.password=cds \
vcs.URL=http://cds-vcs:8084 \
vcs.api.http.url=http://cds-api:8081 \
vcs.cache.redis.host=cds-cache:6379 \
vcs.cache.redis.password=cds \
repositories.URL=http://cds-repositories:8085 \
repositories.basedir=/app/repositories \
repositories.api.http.url=http://cds-api:8081 \
repositories.cache.redis.host=cds-cache:6379 \
repositories.cache.redis.password=cds \
elasticsearch.URL=http://cds-elasticsearch:8088 \
elasticsearch.api.http.url=http://cds-api:8081 \
elasticsearch.name=cds-elasticsearch \
elasticsearch.elasticsearch.indexEvents=cds-index-events \
elasticsearch.elasticsearch.indexMetrics=cds-index-metrics \
elasticsearch.elasticsearch.url=http://elasticsearch:9200 \
ui.enableServiceProxy=true \
ui.api.http.url=http://cds-api:8081 \
ui.hooksURL=http://cds-hooks:8083 \
ui.cdnURL=http://cds-cdn:8089;
echo '';
echo '/!\ Please run the export command BELOW, this will be useful to create admin user';
echo '';
TOKEN=`/app/cds-engine-linux-amd64 config init-token --config /app/conf/conf.toml`;
echo 'export INIT_TOKEN='$$TOKEN;
echo '';
echo '/!\ Please run the export command ABOVE, this will be useful to create admin user';
echo '';"
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.artifact.local.baseDirectory=/app/artifacts;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.cache.redis.host=cds-cache:6379;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.cache.redis.password=cds;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.database.host=cds-db;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.download.directory=/app;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml api.log.level=info;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.api.http.url=http://cds-api:8081;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.cache.redis.host=cds-cache:6379;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.cache.redis.password=cds;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.database.host=cds-db;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.publicTCP=${HOSTNAME}:8090;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.buffers.local-buffer.local.path=/app/cdn-buffer;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.buffers.redis.redis.host=cds-cache:6379;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.buffers.redis.redis.password=cds;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.storageUnits.storages.local.local.path=/app/cdn-storage;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml cdn.URL=http://cds-cdn:8089;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.api.http.url=http://cds-api:8081;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.elasticsearch.indexEvents=cds-index-events;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.elasticsearch.indexMetrics=cds-index-metrics;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.elasticsearch.url=http://elasticsearch:9200;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.name=cds-elasticsearch;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml elasticsearch.URL=http://cds-elasticsearch:8088;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.local.basedir=/app/hatchery-local;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.local.commonConfiguration.api.http.url=http://cds-api:8081;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.local.commonConfiguration.url=http://cds-hatchery-local:8086;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.commonConfiguration.api.http.url=http://cds-api:8081;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.commonConfiguration.url=http://cds-hatchery-swarm:8086;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.commonConfiguration.provision.workerApiHttp.url=http://$HOSTNAME:8081;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.dockerEngines.sample-docker-engine.host=tcp://dockerhost:2375;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.dockerEngines.sample-docker-engine.maxContainers=4;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hatchery.swarm.ratioService=50;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.api.http.url=http://cds-api:8081;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.cache.redis.host=cds-cache:6379;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.cache.redis.password=cds;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml hooks.url=http://cds-hooks:8083;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.api.http.url=http://cds-api:8081;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.basedir=/app/repositories;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.cache.redis.host=cds-cache:6379;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.cache.redis.password=cds;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml repositories.URL=http://cds-repositories:8085;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.api.http.url=http://cds-api:8081;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.cdnURL=http://cds-cdn:8089;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.enableServiceProxy=true;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ui.hooksURL=http://cds-hooks:8083;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.api.http.url=http://cds-api:8081;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.cache.redis.host=cds-cache:6379;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.cache.redis.password=cds;
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml vcs.URL=http://cds-vcs:8084;
echo '';
echo '/!\\ Please run the export command BELOW, this will be useful to create admin user';
echo '';
TOKEN=`/app/cds-engine-linux-amd64 config init-token --config /app/conf/conf.toml`;
echo 'export INIT_TOKEN='$$TOKEN;
echo '';
echo '/!\\ Please run the export command ABOVE, this will be useful to create admin user';
echo '';"
user: root
volumes:
- cds-conf-volume:/app/conf
Expand Down Expand Up @@ -272,8 +272,7 @@ services:
image: ${CDS_DOCKER_IMAGE:-ovhcom/cds-engine:latest}
command: >
sh -c "
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml \
${CDS_EDIT_CONFIG}";
/app/cds-engine-linux-amd64 config edit /app/conf/conf.toml --output /app/conf/conf.toml ${CDS_EDIT_CONFIG}";
volumes:
- cds-conf-volume:/app/conf

Expand Down
2 changes: 1 addition & 1 deletion docs/content/hosting/cds_services.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ A CDS µService is a process that can be launch by the `engine` binary.
- **api**: this is the main µService.
- Each `api` must shared the same PostgreSQL and redis databases. All other CDS µServices have to communicate with `api`.
- The database user does not need to have the admin rights to create / alter tables.
- **cdn**: this is the main µService.
- **cdn**: the `cdn` service is dedicated to receive and store CDS's logs and artifacts.
- Each `cdn` must shared the same PostgreSQL and redis databases.
- The database user does not need to have the admin rights to create / alter tables.
- **ui**: the `ui` service serves the CDS UI static files.
Expand Down
50 changes: 47 additions & 3 deletions docs/content/hosting/ready-to-run/docker-compose/docker-compose.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ card:

## Run with Docker-Compose

{{% notice warning %}}

This tutorial allows you to start a CDS locally quickly.

* Do not use this docker-compose installation to run it in production without modification.
* For production purpose, please read [CDS Services]({{< relref "../../cds_services" >}})

{{% /notice %}}

The [docker-compose.yml](https://github.com/ovh/cds/blob/{{< param "version" "master" >}}/docker-compose.yml) contains:

- cds-db service with a PostgreSQL
Expand All @@ -27,7 +36,7 @@ Docker compose is very convenient to launch CDS for testing it. But this is not
## How to run

```bash
$ mkdir /tmp/cdstest && cd /tmp/cdstest && mkdir -p tools/smtpmock
$ mkdir /tmp/cdstest && cd /tmp/cdstest
$ curl https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/docker-compose.yml -o docker-compose.yml
$ export HOSTNAME=$(hostname)
$ export CDS_DOCKER_IMAGE=ovhcom/cds-engine:{{< param "version" "latest" >}}
Expand Down Expand Up @@ -57,9 +66,11 @@ $ TOKEN_CMD=$(docker logs $(docker-compose ps -q cds-prepare) | grep INIT_TOKEN)
# if you have this error: "command too long: export INIT_TOKEN=....",
# you can manually execute the command "export INIT_TOKEN=...."

# create user
$ curl 'http://localhost:8081/download/cdsctl/linux/amd64?variant=nokeychain' -o cdsctl
# download cdsctl
# on linux: $ curl 'http://localhost:8081/download/cdsctl/linux/amd64?variant=nokeychain' -o cdsctl
# on OSX: $ curl 'http://localhost:8081/download/cdsctl/darwin/amd64?variant=nokeychain' -o cdsctl

# create user
$ chmod +x cdsctl
$ ./cdsctl signup --api-url http://localhost:8081 --email [email protected] --username admin --fullname admin
# enter a strong password
Expand Down Expand Up @@ -101,6 +112,15 @@ Workflow MyFirstWorkflow #1 has been launched
http://localhost:8080/project/DEMO/workflow/MyFirstWorkflow/run/1
```

{{% notice warning %}}

Important: the service is exposed without **https**.

* Do not use this docker-compose installation to run it in production without modification.
* Safari don't manage secured cookie on localhost, please use another browser to avoid authentication error on your browser

{{% /notice %}}

- Login on WebUI

Open a browser on http://localhost:8080/account/signup, then login with the user `admin`,
Expand All @@ -120,6 +140,30 @@ The build pipeline contains two stages, with only one job in each stage

![Build Pipeline](/images/ready_to_run_docker_compose_build_pipeline.png)

## If the job does not start

The worker model used in this tutorial is a docker image golang. CDS will run jobs only if the worker model is *registered*.

So, if the first job of the workflow stay in *Waiting* status, you can check if you worker model is well registered.
Go on http://localhost:8080/settings/worker-model/shared.infra/go-official-1.17 , you must have the flag `need registration: false`.

If the flag is `true`, you can check the swarm hatchery logs:

```bash
$ docker-compose logs -f cds-hatchery-swarm
```

When a worker (so, a container) is starting, this container communicates with the api with the url `http://$HOSTNAME:8081` to download the worker binary.
This URL is using your `$HOSTNAME`, but perhaps that this can't be used on your docker installation from a container. If the container can't communicate to the api with this url, you can update it:

```bash
# you can replace $HOSTNAME by your IP on local network. Example: 192.168.xxx.xxx
$ export CDS_EDIT_CONFIG="hatchery.swarm.commonConfiguration.provision.workerApiHttp.url=http://192.168.xxx.xxx:8081 "
$ docker-compose up cds-edit-config
$ docker-compose restart cds-hatchery-swarm
```


## Setup connection with a VCS

```bash
Expand Down
2 changes: 1 addition & 1 deletion engine/service/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ type HatcheryCommonConfiguration struct {
Region string `toml:"region" default:"" comment:"region of this hatchery - optional. With a free text as 'myregion', user can set a prerequisite 'region' with value 'myregion' on CDS Job" json:"region"`
IgnoreJobWithNoRegion bool `toml:"ignoreJobWithNoRegion" default:"false" comment:"Ignore job without a region prerequisite if ignoreJobWithNoRegion=true" json:"ignoreJobWithNoRegion"`
WorkerAPIHTTP struct {
URL string `toml:"url" default:"http://localhost:8081" commented:"true" comment:"CDS API URL for worker, let empty or commented to use the same URL that is used by the Hatchery" json:"url"`
URL string `toml:"url" default:"" commented:"false" comment:"CDS API URL for worker, let empty or commented to use the same URL that is used by the Hatchery. Example: http://localhost:8081" json:"url"`
Insecure bool `toml:"insecure" default:"false" commented:"true" comment:"sslInsecureSkipVerify, set to true if you use a self-signed SSL on CDS API" json:"insecure"`
} `toml:"workerApiHttp" json:"workerApiHttp"`
WorkerBasedir string `toml:"workerBasedir" commented:"true" comment:"Worker Basedir" json:"workerBasedir"`
Expand Down

0 comments on commit 8d175a0

Please sign in to comment.