Add telemetry support to docker-compose deployments

tryretool · Mar 25, 2024 · d51b8c2 · d51b8c2
1 parent 7b59d4e
commit d51b8c2
Show file tree

Hide file tree

Showing 7 changed files with 129 additions and 20 deletions.
diff --git a/docker-compose-with-temporal.yml b/docker-compose-with-temporal.yml
@@ -10,6 +10,7 @@ services:
     environment:
       - DEPLOYMENT_TEMPLATE_TYPE=docker-compose
       - SERVICE_TYPE=MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR
+      - RTEL_SERVICE_NAME=main-backend
       - DBCONNECTOR_POSTGRES_POOL_MAX_SIZE=100
       - DBCONNECTOR_QUERY_TIMEOUT_MS=120000
       - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000
@@ -48,6 +49,7 @@ services:
     environment:
       - DEPLOYMENT_TEMPLATE_TYPE=docker-compose
       - SERVICE_TYPE=JOBS_RUNNER
+      - RTEL_SERVICE_NAME=jobs-runner
     networks:
       - backend-network
     depends_on:
@@ -69,6 +71,7 @@ services:
     environment:
       - DEPLOYMENT_TEMPLATE_TYPE=docker-compose
       - SERVICE_TYPE=WORKFLOW_TEMPORAL_WORKER
+      - RTEL_SERVICE_NAME=workflow-worker
       - DISABLE_DATABASE_MIGRATIONS=true
       - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000
       - WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST=temporal
@@ -88,6 +91,7 @@ services:
     environment:
       - DEPLOYMENT_TEMPLATE_TYPE=docker-compose
       - SERVICE_TYPE=WORKFLOW_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR
+      - RTEL_SERVICE_NAME=workflow-backend
       - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000
       - WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_HOST=temporal
       - WORKFLOW_TEMPORAL_CLUSTER_FRONTEND_PORT=7233
@@ -119,6 +123,7 @@ services:
     env_file: ./docker.env
     environment:
       - DEPLOYMENT_TEMPLATE_TYPE=docker-compose
+      - RTEL_SERVICE_NAME=code-executor
       - NODE_OPTIONS=--max_old_space_size=1024
     networks:
       - code-executor-network
@@ -130,6 +135,47 @@ services:
     privileged: true
     restart: on-failure
 
+  telemetry:
+    image: "telemetry:ra-dev-0" # TODO: change to public image
+    restart: on-failure
+    env_file: docker.env
+    ports:
+      - '9125:9125/udp'
+    command:
+      - bash
+      - -c
+      - |
+        if [[ "$${RTEL_ENABLED:-false}" == "false" ]]; then
+          echo 1>&2 "RTEL_ENABLED is not true, going to sleep."
+          sleep inf
+        else
+          exec retool-telemetry
+        fi
+    environment:
+      RTEL_DEPLOYMENT_MODE: 'docker-compose'
+      DEPLOYMENT_TEMPLATE_TYPE: 'docker-compose'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /:/rootfs:ro
+      - /sys:/sys:ro
+
+      # Uncomment this to use custom [vector](https://vector.dev) config, i.e.
+      # to add extra telemetry sinks to your own destination.
+      - ${PWD}/vector-custom.yaml:/etc/vector-custom/vector-custom.yaml:ro
+
+      # Uncomment this to use custom
+      # [grafana-agent](https://grafana.com/docs/agent/latest/flow/) river
+      # config, i.e. to add extra telemetry sources to your own destination.
+      # - ${PWD}/grafana-agent-custom.river:/etc/grafana-agent-custom/grafana-agent-custom.river:ro
+    devices:
+      - /dev/kmsg:/dev/kmsg
+    cap_add:
+      - SYS_ADMIN
+    networks:
+      - backend-network
+      - code-executor-network
+
   # Retool's storage database. See these docs to migrate to an externally hosted database: https://docs.retool.com/docs/configuring-retools-storage-database
   postgres:
     image: "postgres:11.13"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -6,6 +6,7 @@ services:
       dockerfile: Dockerfile
     env_file: ./docker.env
     environment:
+      - RTEL_SERVICE_NAME=main-backend
       - DEPLOYMENT_TEMPLATE_TYPE=docker-compose
       - SERVICE_TYPE=MAIN_BACKEND,DB_CONNECTOR_SERVICE,DB_SSH_CONNECTOR_SERVICE
       - DBCONNECTOR_POSTGRES_POOL_MAX_SIZE=100
@@ -52,6 +53,7 @@ services:
     environment:
       - DEPLOYMENT_TEMPLATE_TYPE=docker-compose
       - SERVICE_TYPE=JOBS_RUNNER
+      - RTEL_SERVICE_NAME=jobs-runner
     networks:
       - backend-network
     depends_on:
@@ -71,6 +73,7 @@ services:
     environment:
       - DEPLOYMENT_TEMPLATE_TYPE=docker-compose
       - SERVICE_TYPE=WORKFLOW_TEMPORAL_WORKER
+      - RTEL_SERVICE_NAME=workflow-worker
       - NODE_OPTIONS=--max_old_space_size=1024
       - DISABLE_DATABASE_MIGRATIONS=true
       - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000
@@ -97,6 +100,7 @@ services:
     environment:
       - DEPLOYMENT_TEMPLATE_TYPE=docker-compose
       - SERVICE_TYPE=WORKFLOW_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR
+      - RTEL_SERVICE_NAME=workflow-backend
       - WORKFLOW_BACKEND_HOST=http://workflows-backend:3000
       - CODE_EXECUTOR_INGRESS_DOMAIN=http://code-executor:3004
       - DBCONNECTOR_POSTGRES_POOL_MAX_SIZE=100
@@ -120,6 +124,48 @@ services:
       - ./retool:/usr/local/retool-git-repo
       - ${BOOTSTRAP_SOURCE:-./retool}:/usr/local/retool-repo
 
+  telemetry:
+    # TODO: update to `latest` when released as stable
+    image: "tryretool/telemetry:3.39.0-edge"
+    restart: on-failure
+    env_file: docker.env
+    ports:
+      - '9125:9125/udp'
+    command:
+      - bash
+      - -c
+      - |
+        if [[ "$${RTEL_ENABLED:-false}" == "false" ]]; then
+          echo 1>&2 "RTEL_ENABLED is not true, going to sleep."
+          sleep inf
+        else
+          exec retool-telemetry
+        fi
+    environment:
+      RTEL_DEPLOYMENT_MODE: 'docker-compose'
+      DEPLOYMENT_TEMPLATE_TYPE: 'docker-compose'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /:/rootfs:ro
+      - /sys:/sys:ro
+
+      # Uncomment this to use custom [vector](https://vector.dev) config, i.e.
+      # to add extra telemetry sinks to your own destination.
+      - ${PWD}/vector-custom.yaml:/etc/vector-custom/vector-custom.yaml:ro
+
+      # Uncomment this to use custom
+      # [grafana-agent](https://grafana.com/docs/agent/latest/flow/) river
+      # config, i.e. to add extra telemetry sources to your own destination.
+      # - ${PWD}/grafana-agent-custom.river:/etc/grafana-agent-custom/grafana-agent-custom.river:ro
+    devices:
+      - /dev/kmsg:/dev/kmsg
+    cap_add:
+      - SYS_ADMIN
+    networks:
+      - backend-network
+      - code-executor-network
+
   code-executor:
     build:
       context: ./
@@ -129,6 +175,7 @@ services:
     environment:
       - DEPLOYMENT_TEMPLATE_TYPE=docker-compose
       - NODE_OPTIONS=--max_old_space_size=1024
+      - RTEL_SERVICE_NAME=code-executor
     networks:
       - code-executor-network
     # code-executor uses nsjail to sandbox code execution. nsjail requires

diff --git a/docker.env.template b/docker.env.template
@@ -28,3 +28,9 @@ POSTGRES_PASSWORD=randomstring
 ## If you wish for Retool to be hosted on a server with a public IP address, then you can use these configs to run the nginx container
 # HOSTNAME=https://retool.company.com
 # DOMAINS=http://localhost -> http://api:3000
+
+## If you wish to enable telemetry collection and forwarding to Retool for help troubleshooting, uncomment these.
+# RTEL_ENABLED=true
+# RTEL_SEND_TO_RETOOL=true
+# STATSD_HOST=telemetry
+# STATSD_PORT="9125"
diff --git a/docker_setup b/docker_setup
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+if [ -f ./docker.env ]; then
+  echo "Found existing docker.env file..."
+  echo "exiting to avoid overwriting existing the configuration file..."
+  exit 0
+fi
+touch docker.env
+
+if [ -f ./retooldb.env ]; then
+  echo "Found existing retooldb.env file..."
+  echo "exiting to avoid overwriting existing the configuration file..."
+  exit 0
+fi
+touch retooldb.env
+
 postgresPassword=$(cat /dev/urandom | base64 | head -c 64)
 retooldbPostgresPassword=$(cat /dev/urandom | base64 | head -c 64)
 jwtSecret=$(cat /dev/urandom | base64 | head -c 256)
@@ -21,51 +35,37 @@ if [ -z "$hostname" ]; then
   hostname=$publicIpAddress
 fi
 
-if [ -f ./docker.env ]; then
-  echo "Found existing docker.env file..."
-  echo "exiting to avoid overwriting existing the configuration file..."
-  exit 0
-fi
-touch docker.env
-
-if [ -f ./retooldb.env ]; then
-  echo "Found existing retooldb.env file..."
-  echo "exiting to avoid overwriting existing the configuration file..."
-  exit 0
-fi
-touch retooldb.env
-
 echo '## For a complete list of all environment variables, see docs.retool.com/docs/environment-variables' >> docker.env
 echo '' >> docker.env
 
 echo '## Set node environment to production' >> docker.env
 echo 'NODE_ENV=production' >> docker.env
 echo '' >> docker.env
 echo '## Set the JWT secret for the API server' >> docker.env
-echo "JWT_SECRET=${jwtSecret}" >> docker.env
+echo "JWT_SECRET=\"${jwtSecret}\"" >> docker.env
 echo '' >> docker.env
 
 echo '## Set and generate postgres credentials' >> docker.env
 echo 'POSTGRES_DB=hammerhead_production' >> docker.env
 echo 'POSTGRES_USER=retool_internal_user' >> docker.env
 echo 'POSTGRES_HOST=postgres' >> docker.env
 echo 'POSTGRES_PORT=5432' >> docker.env
-echo "POSTGRES_PASSWORD=${postgresPassword}" >> docker.env
+echo "POSTGRES_PASSWORD=\"${postgresPassword}\"" >> docker.env
 echo '' >> docker.env
 
 echo '## Set and generate retooldb postgres credentials' >> docker.env
 echo 'RETOOLDB_POSTGRES_DB=postgres' >> docker.env
 echo 'RETOOLDB_POSTGRES_USER=root' >> docker.env
 echo 'RETOOLDB_POSTGRES_HOST=retooldb-postgres' >> docker.env
 echo 'RETOOLDB_POSTGRES_PORT=5432' >> docker.env
-echo "RETOOLDB_POSTGRES_PASSWORD=${retooldbPostgresPassword}" >> docker.env
+echo "RETOOLDB_POSTGRES_PASSWORD=\"${retooldbPostgresPassword}\"" >> docker.env
 echo '' >> docker.env
 
 echo '## Set and generate retooldb postgres credentials' >> retooldb.env
 echo 'POSTGRES_HOST=retooldb-postgres' >> retooldb.env
 echo 'POSTGRES_DB=postgres' >> retooldb.env
 echo 'POSTGRES_USER=root' >> retooldb.env
-echo "POSTGRES_PASSWORD=${retooldbPostgresPassword}" >> retooldb.env
+echo "POSTGRES_PASSWORD=\"${retooldbPostgresPassword}\"" >> retooldb.env
 echo 'POSTGRES_PORT=5432' >> retooldb.env
 echo '' >> retooldb.env
 
@@ -80,7 +80,7 @@ echo '' >> docker.env
 
 echo '## Set key to encrypt and decrypt database passwords, etc.' >> docker.env
 echo '## This random string value should be stored privately, and should not be changed over the liftetime of the deployment' >> docker.env
-echo "ENCRYPTION_KEY=${encryptionKey}" >> docker.env
+echo "ENCRYPTION_KEY=\"${encryptionKey}\"" >> docker.env
 echo '' >> docker.env
 
 echo "## Google SSO configuration" >> docker.env

diff --git a/get-docker-compose.sh b/get-docker-compose.sh
@@ -4,5 +4,5 @@ if command -v docker-compose &> /dev/null ; then
     exit 0
 fi
 
-sudo -E curl -L https://github.com/docker/compose/releases/download/1.29.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
+sudo -E curl -L https://github.com/docker/compose/releases/download/v2.24.7/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
 sudo chmod +x /usr/local/bin/docker-compose
diff --git a/grafana-agent-custom.river b/grafana-agent-custom.river
@@ -0,0 +1,5 @@
+// Use this file for custom
+// [grafana-agent](https://grafana.com/docs/agent/latest/flow/) river config, 
+// i.e. to add extra telemetry sources.
+//
+// Note: must enable the corresponding telemetry volume mount in your compose file.
diff --git a/vector-custom.yaml b/vector-custom.yaml
@@ -0,0 +1,5 @@
+# Use this file for custom [vector](https://vector.dev) config, i.e. to add
+# extra telemetry sinks to your own destination.
+#
+# Note: must enable the corresponding telemetry volume mount in your compose
+# file.