doc: migrate 0.50.0 (#6039)

* doc: migrate 0.50.0 Signed-off-by: Yvonnick Esnault <[email protected]>
ovh · Dec 16, 2021 · a5bdc40 · a5bdc40
1 parent afc5141
commit a5bdc40
Show file tree

Hide file tree

Showing 22 changed files with 353 additions and 105 deletions.
diff --git a/contrib/worker-models/README.md b/contrib/worker-models/README.md
@@ -8,9 +8,9 @@ See https://ovh.github.io/cds/docs/concepts/worker-model/
 
 ```bash
 # import a worker model
-cdsctl worker model import ./go-official-1.13.yml
+cdsctl worker model import ./go-official-1.17.yml
 
 # or with a remote file
-cdsctl worker model import https://raw.githubusercontent.com/ovh/cds/master/contrib/worker-models/go-official-1.13.yml
+cdsctl worker model import https://raw.githubusercontent.com/ovh/cds/master/contrib/worker-models/go-official-1.17.yml
 
 ```
diff --git a/contrib/worker-models/go-official-1.13.yml → contrib/worker-models/go-official-1.17.yml b/contrib/worker-models/go-official-1.13.yml → contrib/worker-models/go-official-1.17.yml
@@ -1,6 +1,6 @@
-name: go-official-1.13
+name: go-official-1.17
 group: shared.infra
-image: golang:1.13
+image: golang:1.17
 description: official from https://hub.docker.com/_/golang/
 type: docker
 pattern_name: basic_unix
diff --git a/contrib/workflow-templates/demo-usage-service-postgresql/README.md b/contrib/workflow-templates/demo-usage-service-postgresql/README.md
@@ -17,7 +17,7 @@ If you want to import it, you have to be CDS Administrator on your CDS Instance.
 
 This template uses a pre-requisite binary `apt-get`, you need a [worker model](https://ovh.github.io/cds/docs/concepts/worker-model/) with this capability on you CDS Instance.
 
-``` bash
+```sh
 # import from github
 cdsctl template push https://raw.githubusercontent.com/ovh/cds/master/contrib/workflow-templates/demo-usage-service-postgresql/demo-usage-service-postgresql.yml
 ```
diff --git a/contrib/workflow-templates/demo-usage-service-postgresql/workflow.yml b/contrib/workflow-templates/demo-usage-service-postgresql/workflow.yml
@@ -1,3 +1,5 @@
 name: [[.name]]
-version: v1.0
-pipeline: demo-service-postgresql
+version: v2.0
+workflow:
+  demo-service-postgresql:
+    pipeline: demo-service-postgresql
diff --git a/contrib/workflow-templates/demo-workflow-hello-world/README.md b/contrib/workflow-templates/demo-workflow-hello-world/README.md
@@ -18,7 +18,7 @@ This template is linked to group: `shared.infra`
 
 If you want to import it, you have to be CDS Administrator on your CDS Instance.
 
-``` bash
+```sh
 # import from github
 cdsctl template push https://raw.githubusercontent.com/ovh/cds/master/contrib/workflow-templates/demo-workflow-hello-world/demo-workflow-hello-world.yml
 ```
diff --git a/contrib/workflow-templates/demo-workflow-hello-world/workflow.yml b/contrib/workflow-templates/demo-workflow-hello-world/workflow.yml
@@ -1,5 +1,5 @@
 name: [[.name]]
-version: v1.0
+version: v2.0
 workflow:
   build:
     pipeline: build-[[.id]]
@@ -27,10 +27,10 @@ workflow:
     - success
     depends_on:
     - fork
-  deploy-prod:   
+  deploy-prod:
     pipeline: deploy-[[.id]]
     application: [[.name]]
-    environment: [[.name]]-prod 
+    environment: [[.name]]-prod
     when:
     - manual
     - success

diff --git a/contrib/workflow-templates/demo-workflow-skeleton/README.md b/contrib/workflow-templates/demo-workflow-skeleton/README.md
@@ -14,7 +14,7 @@ This template is linked to group: `shared.infra`
 
 If you want to import it, you have to be CDS Administrator on your CDS Instance.
 
-``` bash
+```sh
 # import from github
 cdsctl template push https://raw.githubusercontent.com/ovh/cds/master/contrib/workflow-templates/demo-workflow-skeleton/demo-workflow-skeleton.yml
 ```
diff --git a/contrib/workflow-templates/demo-workflow-skeleton/workflow.yml b/contrib/workflow-templates/demo-workflow-skeleton/workflow.yml
@@ -1,5 +1,5 @@
 name: [[.name]]
-version: v1.0
+version: v2.0
 workflow:
   build:
     pipeline: build-[[.id]]
@@ -27,10 +27,10 @@ workflow:
     - success
     depends_on:
     - fork
-  deploy-prod:   
+  deploy-prod:
     pipeline: deploy-[[.id]]
     application: [[.name]]
-    environment: [[.name]]-prod 
+    environment: [[.name]]-prod
     when:
     - manual
     - success

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -11,19 +11,22 @@ services:
       POSTGRES_PASSWORD: cds
       POSTGRES_USER: cds
       POSTGRES_DB: cds
+    volumes:
+      - cds-postgres-volume:/var/lib/postgresql/data
 
   cds-db-init:
     image: postgres:14.0
     command: >
-        sh -exc "
-        PGPASSWORD=cds psql -h cds-db -p 5432 -U cds -d cds -c \"CREATE SCHEMA IF NOT EXISTS cdn AUTHORIZATION cds;\";"
+        sh -exc "PGPASSWORD=cds psql -h cds-db -p 5432 -U cds -d cds -c \"CREATE SCHEMA IF NOT EXISTS cdn AUTHORIZATION cds;\";"
     links:
        - cds-db
 
   cds-cache:
     image: redis:alpine
-    command: redis-server --requirepass cds
+    command: redis-server --requirepass cds --save 60 1
     restart: always
+    volumes:
+      - cds-redis-volume:/data
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:6.7.2
@@ -50,7 +53,13 @@ services:
   cds-prepare:
     image: ${CDS_DOCKER_IMAGE:-ovhcom/cds-engine:latest}
     command: >
-        sh -c "
+        bash -c "
+        if [ ! -f /app/conf/conf.toml ]; then
+          /app/cds-engine-linux-amd64 config new api cdn ui hooks repositories vcs elasticsearch hatchery:local hatchery:swarm > /app/conf/conf.toml;
+        else
+          echo '/!\ Config file already exists';
+        fi;
+        chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/cdn-buffer /app/cdn-storage;
         /app/cds-engine-linux-amd64 config new > /app/conf/conf.toml;
         mkdir -p /app/artifacts /app/repositories /app/hatchery-local;
         chown -R cds:cds /app/conf /app/artifacts /app/repositories /app/hatchery-local /app/cdn-buffer /app/cdn-storage;
@@ -60,12 +69,7 @@ services:
             api.cache.redis.password=cds \
             api.database.host=cds-db \
             api.download.directory=/app \
-            api.url.api=http://${HOSTNAME}:8081 \
-            api.url.ui=http://${HOSTNAME}:8080 \
             api.log.level=info \
-            api.smtp.disable=false \
-            api.smtp.port=2023 \
-            api.smtp.host=smtpmocksrv \
             cdn.cache.redis.host=cds-cache:6379 \
             cdn.cache.redis.password=cds \
             cdn.storageUnits.buffers.redis.redis.host=cds-cache:6379 \
@@ -103,7 +107,6 @@ services:
             elasticsearch.elasticsearch.indexEvents=cds-index-events \
             elasticsearch.elasticsearch.indexMetrics=cds-index-metrics \
             elasticsearch.elasticsearch.url=http://elasticsearch:9200 \
-            ui.url=http://${HOSTNAME}:8080 \
             ui.enableServiceProxy=true \
             ui.api.http.url=http://cds-api:8081 \
             ui.hooksURL=http://cds-hooks:8083 \
@@ -282,13 +285,6 @@ services:
     ports:
       - "127.0.0.1:2375:2375"
 
-  smtpmocksrv:
-    build:
-      context: ./tools/smtpmock
-    ports:
-      - "2023:2023"
-      - "2024:2024"
-
   kibana:
     image: docker.elastic.co/kibana/kibana-oss:6.7.2
     ports:
@@ -297,10 +293,12 @@ services:
        - elasticsearch
 
 volumes:
+  cds-postgres-volume:
+  cds-redis-volume:
+  cds-elasticsearch-volume:
+  cds-conf-volume:
   cds-artifacts-volume:
   cds-repositories-volume:
   cds-hatchery-local-volume:
-  cds-conf-volume:
-  cds-elasticsearch-volume:
   cds-cdn-buffer-volume:
   cds-cdn-storage-volume:
diff --git a/docs/content/docs/components/cdn.md b/docs/content/docs/components/cdn.md
@@ -4,36 +4,138 @@ weight: 3
 ---
 
 ## What's CDN
-CDN is a service dedicated to receive and store CDS's logs. In a near future, it will also be able to manage artifacts and cache used by your jobs. 
+
+CDN is a service dedicated to receive and store CDS's logs and artifacts. 
 
 CDN stores the list of all known log or artifact items in a Postgres database and communicates with storage backends to store the contents of those items.
 These backends are call units and there are two types of units in CDN:
 
 * Buffer unit: To store logs and artifacts of incoming jobs, these units are designed to be fast for read / write operations, but with limited capacity.
 
-* Storage Unit: to store complete job's logs and artifact.
+* Storage Unit: to store complete job's logs and artifacts.
 
-When logs or file are received by CDN, it will first store these items in its buffer. Then, when the item is fully received, it will be moved to one of the configured storage units.
+When logs or file are received by CDN from a cds worker, it will first store these items in its buffer. Then, when the item is fully received, it will be moved to one of the configured storage units.
 If the CDN service is configured with multiple storage units, each unit periodically checks for missing items and synchronizes these items from other units.
 
+CDS UI and CLI communicate with CDN to get entire logs, or stream them.
+
+## Supported units
+* Buffer (type: log): Redis.
+* Buffer (type: file): Local, NFS
+* Storage: Local, Swift, S3, Webdav
 
 ## Configuration
-Like any other CDS service, CDN requires to be authenticated with a consumer. The required scopes are Service, Worker and RunExecution.
+Like any other CDS service, CDN requires to be authenticated with a consumer. The required scopes are `Service`, `Worker` and `RunExecution`.
+
+You can generate a configuration file with the `engine` binary:
+
+```sh
+$ engine config new cdn > cds-configuration.toml
+```
 
 You must have at least one storage unit, one file buffer and one log buffer to be able to run CDN.
 
-## Supported units
-* Buffer (type: log): Redis.
-* Buffer (type: file): Local.
-* Storage: Local, Swift, S3, Webdav, CDS (cds unit is used for migration purpose and will be removed in future release).
+### CDN artifact configuration
 
+#### Storage Unit Buffer
 
-## Use case
+You must have a `storageUnits.buffers` , one for the type `log`, another for the type `file`.
 
-Workers and hatcheries communicate with CDN, sending step logs and service log.
+Type `log`:
 
-![CDN_RECEIVE](/images/cdn_logs_receive.png?width=600px)
+```toml
+      [cdn.storageUnits.buffers.redis]
+        bufferType = "log"
 
-CDS UI and CLI communicate with CDN to get entire logs, or stream them.
+        [cdn.storageUnits.buffers.redis.redis]
+          host = "aaa@instance0,instance1,instance2"
+          password = "your-password"
+```
+
+Type `file`:
+
+```toml
+      [cdn.storageUnits.buffers.local-buffer]
+
+        # it can be 'log' to receive logs or 'file' to receive artifacts
+        bufferType = "file"
+
+        [cdn.storageUnits.buffers.local-buffer.local]
+          path = "/var/lib/cds-engine/cdn-buffer"
+```
+
+To multi-instanciate the cdn service, you can use a NFS for the bufferType file, example:
+
+```toml
+      [cdn.storageUnits.buffers.buffer-nfs]
+        bufferType = "file"
+
+        [cdn.storageUnits.buffers.buffer-nfs.nfs]
+          host = "w.x.y.z"
+          targetPartition = "/zpool-partition/cdn"
+          userID = 0
+          groupID = 0
+
+          [[cdn.storageUnits.buffers.buffer-nfs.nfs.encryption]]
+            Cipher = "aes-gcm"
+            Identifier = "nfs-buffer-id"
+            ## enter a key here, 32 lenght
+            Key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+            Sealed = false   
+```
+
+#### Storage Units Storage
+
+The storage unit 'storage' store the artifacts. 
+You can use `Local`, `Swift`, `S3`, `Webdav`
+
+Example of storage unit `local`:
+
+```toml
+    [cdn.storageUnits.storages]
+
+      [cdn.storageUnits.storages.local]
+
+        # flag to disabled backend synchronization
+        disableSync = false
+
+        # global bandwith shared by the sync processes (in Mb)
+        syncBandwidth = 128
+
+        # number of parallel sync processes
+        syncParallel = 2
+
+        [cdn.storageUnits.storages.local.local]
+          path = "/tmp/cds/local-storage"
+
+          [[cdn.storageUnits.storages.local.local.encryption]]
+            Cipher = "aes-gcm"
+            Identifier = "cdn-storage-local"
+            LocatorSalt = "xxxxxxxxx"
+            SecretValue = "xxxxxxxxxxxxxxxxx"
+            Timestamp = 0
+```
+
+Example of storage unit `swift`:
+```
+    [cdn.storageUnits.storages]
+
+      [cdn.storageUnits.storages.swift]
+        syncParallel = 6
+        syncBandwidth = 1000
+
+        [cdn.storageUnits.storages.swift.swift]
+          address = "https://xxx.yyy.zzz/v3"
+          username = "foo"
+          password = "your-password-here"
+          tenant = "your-tenant-here"
+          domain = "Default"
+          region = "XXX"
+          containerPrefix = "prod"
 
-![CDN_GET](/images/cdn_logs_get.png?width=600px)
+          [[cdn.storageUnits.storages.swift.swift.encryption]]
+            Cipher = "aes-gcm"
+            Identifier = "swift-backend-id"
+            LocatorSalt = "XXXXXXXX"
+            SecretValue = "XXXXXXXXXXXXXXXX"
+```
diff --git a/docs/content/docs/concepts/files/worker_model-syntax.md b/docs/content/docs/concepts/files/worker_model-syntax.md
@@ -8,9 +8,9 @@ card:
 Example:
 
 ```yml
-name: go-official-1.13
+name: go-official-1.17
 group: shared.infra
-image: golang:1.13
+image: golang:1.17
 description: official from https://hub.docker.com/_/golang/
 type: docker
 pattern_name: basic_unix
@@ -19,13 +19,13 @@ pattern_name: basic_unix
 Import a worker model:
 
 ```bash
-cdsctl worker model import ./go-official-1.13.yml
+cdsctl worker model import ./go-official-1.17.yml
 ```
 
 or with a remote file:
 
 ```bash
-cdsctl worker model import https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/contrib/worker-models/go-official-1.13.yml
+cdsctl worker model import https://raw.githubusercontent.com/ovh/cds/{{< param "version" "master" >}}/contrib/worker-models/go-official-1.17.yml
 ```
 
 {{< note >}}