feat(database monitor): handles multi-region and connection pooling (#…

…3685)

.circleci/config.yml

@@ -1049,7 +1049,6 @@ jobs:
   docker-build-monitor-container:
     <<: *build-job
     environment:
-      FOLDER: utils
       SPECKLE_SERVER_PACKAGE: monitor-deployment
 
   docker-build-docker-compose-ingress:
@@ -1119,7 +1118,6 @@ jobs:
   docker-publish-monitor-container:
     <<: *publish-job
     environment:
-      FOLDER: utils
       SPECKLE_SERVER_PACKAGE: monitor-deployment
 
   docker-publish-docker-compose-ingress:

.gitignore

@@ -74,7 +74,9 @@ redis-data/
 .tshy-build
 obj/
 bin/
-
+!packages/monitor-deployment/bin
+!packages/preview-service/bin
+!packages/server/bin
 
 # Server
 multiregion.json

packages/fileimport-service/Dockerfile

@@ -20,7 +20,7 @@ RUN chmod +x /usr/bin/tini
 RUN apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y \
      --no-install-recommends \
-     curl=8.5.0-2ubuntu10.5 \
+     curl=8.5.0-2ubuntu10.6 \
     && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y \
      --no-install-recommends \

packages/monitor-deployment/.env.example

@@ -0,0 +1,16 @@
+PG_CONNECTION_STRING='postgres://speckle:[email protected]/speckle'
+POSTGRES_MAX_CONNECTIONS='2'
+PROMETHEUS_METRICS_PORT='9092'
+LOG_LEVEL='info'
+LOG_PRETTY='true'
+METRICS_COLLECTION_PERIOD_SECONDS='120'
+FF_WORKSPACES_MULTI_REGION_ENABLED='false'
+
+# Enable this if you want to use a custom CA certificate for the connection to the database
+# You also have to save it in a file and set NODE_EXTRA_CA_CERTS when running `yarn start` or `yarn dev``
+# POSTGRES_CA_CERTIFICATE='-----BEGIN CERTIFICATE-----
+# XXXX
+# -----END CERTIFICATE-----'
+
+# Used if the database name to be queried is different from the path in the connection string, which is the case for connection pools
+# POSTGRES_DATABASE='speckle'

packages/monitor-deployment/Dockerfile

@@ -0,0 +1,51 @@
+# NOTE: Docker context must be the git root directory, to include the shared directory
+ARG NODE_ENV=production
+
+FROM node:18-bookworm-slim@sha256:408f8cbbb7b33a5bb94bdb8862795a94d2b64c2d516856824fd86c4a5594a443 AS build-stage
+
+WORKDIR /speckle-server
+
+# Download tini
+ARG TINI_VERSION=v0.19.0
+ENV TINI_VERSION=${TINI_VERSION}
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini ./tini
+RUN chmod +x ./tini
+
+ARG NODE_ENV
+ENV NODE_ENV=${NODE_ENV}
+
+WORKDIR /speckle-server
+
+COPY .yarnrc.yml .
+COPY .yarn ./.yarn
+COPY package.json yarn.lock ./
+
+# Only copy in the relevant package.json files for the dependencies
+COPY packages/shared/package.json ./packages/shared/
+COPY packages/monitor-deployment/package.json ./packages/monitor-deployment/
+
+RUN yarn workspaces focus -A && yarn install
+
+# Only copy in the relevant source files for the dependencies
+COPY packages/shared ./packages/shared/
+COPY packages/monitor-deployment ./packages/monitor-deployment/
+
+RUN yarn workspaces foreach -W run build
+
+WORKDIR /speckle-server/packages/monitor-deployment
+RUN yarn workspaces focus --production
+
+FROM gcr.io/distroless/nodejs18-debian12:nonroot@sha256:afdea027580f7afcaf1f316b2b3806690c297cb3ce6ddc5cf6a15804dc1c790f AS production-stage
+
+ARG NODE_ENV
+ENV NODE_ENV=${NODE_ENV}
+
+WORKDIR /speckle-server
+COPY --from=build-stage /speckle-server/tini /usr/bin/tini
+COPY --from=build-stage /speckle-server/packages/shared ./packages/shared
+COPY --from=build-stage /speckle-server/packages/monitor-deployment ./packages/monitor-deployment
+COPY --from=build-stage /speckle-server/node_modules ./node_modules
+
+WORKDIR /speckle-server/packages/monitor-deployment
+
+ENTRYPOINT [ "tini", "--", "/nodejs/bin/node", "--loader=./dist/src/aliasLoader.js", "bin/www.js" ]

packages/monitor-deployment/README.md

@@ -0,0 +1,23 @@
+# Database Monitor
+
+Responsible for querying all databases and generating metrics.
+
+Metrics are available at `/metrics` endpoint and are in Prometheus format.
+
+## Development
+
+```bash
+yarn dev
+```
+
+## Databases with self-signed certificates
+
+Add the self-signed CA certificate to a file at `packages/monitor-deployment/ca-certificate.crt`
+
+Run `NODE_EXTRA_CA_CERTS=./ca-certificate.crt yarn dev` or `NODE_EXTRA_CA_CERTS=./ca-certificate.crt yarn start`
+
+## Production
+
+```bash
+yarn start
+```

packages/monitor-deployment/bin/www.js

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import '../dist/src/bin.js'

packages/monitor-deployment/eslint.config.mjs

@@ -0,0 +1,61 @@
+import tseslint from 'typescript-eslint'
+import {
+  baseConfigs,
+  getESMDirname,
+  globals,
+  prettierConfig
+} from '../../eslint.config.mjs'
+
+const configs = [
+  ...baseConfigs,
+  {
+    ignores: ['dist', 'public', 'docs']
+  },
+  {
+    files: ['**/*.js'],
+    ignores: ['**/*.mjs'],
+    languageOptions: {
+      sourceType: 'module',
+      globals: {
+        ...globals.node
+      }
+    }
+  },
+  {
+    files: ['bin/www'],
+    languageOptions: {
+      sourceType: 'module',
+      globals: {
+        ...globals.node
+      }
+    }
+  },
+  ...tseslint.configs.recommendedTypeChecked.map((c) => ({
+    ...c,
+    files: [...(c.files || []), '**/*.ts', '**/*.d.ts']
+  })),
+  {
+    files: ['**/*.ts', '**/*.d.ts'],
+    languageOptions: {
+      parserOptions: {
+        tsconfigRootDir: getESMDirname(import.meta.url),
+        project: './tsconfig.json'
+      }
+    },
+    rules: {
+      '@typescript-eslint/no-explicit-any': 'error',
+      '@typescript-eslint/no-unsafe-return': 'error'
+    }
+  },
+  {
+    files: ['**/*.spec.{js,ts}'],
+    languageOptions: {
+      globals: {
+        ...globals.node
+      }
+    }
+  },
+  prettierConfig
+]
+
+export default configs

packages/monitor-deployment/multiregion.example.json

@@ -0,0 +1,34 @@
+{
+  "main": {
+    "postgres": {
+      "connectionUri": "postgresql://speckle:[email protected]:5432/speckle",
+      "privateConnectionUri": "postgresql://speckle:speckle@postgres:5432/speckle",
+      "databaseName": "speckle"
+    },
+    "blobStorage": {
+      "accessKey": "minioadmin",
+      "secretKey": "minioadmin",
+      "bucket": "speckle-server",
+      "createBucketIfNotExists": true,
+      "endpoint": "http://127.0.0.1:9000",
+      "s3Region": "us-east-1"
+    }
+  },
+  "regions": {
+    "region1": {
+      "postgres": {
+        "connectionUri": "postgresql://speckle:[email protected]:5401/speckle",
+        "privateConnectionUri": "postgresql://speckle:speckle@postgres-region1:5432/speckle",
+        "databaseName": "speckle"
+      },
+      "blobStorage": {
+        "accessKey": "minioadmin",
+        "secretKey": "minioadmin",
+        "bucket": "speckle-server",
+        "createBucketIfNotExists": true,
+        "endpoint": "http://127.0.0.1:9020",
+        "s3Region": "us-east-1"
+      }
+    }
+  }
+}

packages/monitor-deployment/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@speckle/monitor-deployment",
+  "private": true,
+  "version": "2.5.4",
+  "description": "Query connected databases and generate metrics.",
+  "main": "bin/www",
+  "homepage": "https://speckle.systems",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/specklesystems/speckle-server.git",
+    "directory": "packages/monitor-deployment"
+  },
+  "type": "module",
+  "engines": {
+    "node": "^18.19.0"
+  },
+  "scripts": {
+    "build:tsc:watch": "tsc -p ./tsconfig.build.json --watch",
+    "run:watch": "NODE_ENV=development LOG_PRETTY=true LOG_LEVEL=debug nodemon --exec \"yarn start\" --trace-deprecation --watch ./bin/www.js --watch ./dist",
+    "dev": "concurrently \"npm:build:tsc:watch\" \"npm:run:watch\"",
+    "dev:headed": "yarn dev",
+    "build:tsc": "rimraf ./dist/src && tsc -p ./tsconfig.build.json",
+    "build": "yarn build:tsc",
+    "lint": "yarn lint:tsc && yarn lint:eslint",
+    "lint:ci": "yarn lint:tsc",
+    "lint:tsc": "tsc --noEmit",
+    "lint:eslint": "eslint .",
+    "start": "node --loader=./dist/src/aliasLoader.js ./bin/www.js",
+    "test": "NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true vitest run --sequence.shuffle"
+  },
+  "dependencies": {
+    "@speckle/shared": "workspace:^",
+    "crypto": "^1.0.1",
+    "dotenv": "^16.4.5",
+    "esm-module-alias": "^2.2.0",
+    "express": "^4.19.2",
+    "http-errors": "~1.6.3",
+    "knex": "^2.4.1",
+    "lodash": "^4.17.21",
+    "lodash-es": "^4.17.21",
+    "pg": "^8.7.3",
+    "pino": "^8.7.0",
+    "pino-http": "^8.2.1",
+    "pino-pretty": "^9.1.1",
+    "prom-client": "^14.0.1",
+    "znv": "^0.4.0",
+    "zod": "^3.24.1"
+  },
+  "devDependencies": {
+    "@types/express": "^4.17.13",
+    "@types/http-errors": "^2.0.4",
+    "@types/lodash-es": "^4.17.6",
+    "@types/node": "^18.19.38",
+    "@vitest/coverage-istanbul": "^1.6.0",
+    "concurrently": "^8.2.2",
+    "crypto-random-string": "^5.0.0",
+    "eslint": "^9.4.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-vitest": "^0.5.4",
+    "nodemon": "^2.0.20",
+    "prettier": "^2.5.1",
+    "rimraf": "^5.0.7",
+    "typescript": "^4.6.4",
+    "typescript-eslint": "^7.12.0",
+    "vitest": "^1.6.0"
+  }
+}

packages/monitor-deployment/src/aliasLoader.ts

@@ -0,0 +1,6 @@
+import generateAliasesResolver from 'esm-module-alias'
+import { srcRoot } from './root.js'
+
+export const resolve = generateAliasesResolver({
+  '@': srcRoot
+})

packages/monitor-deployment/src/bin.ts

@@ -0,0 +1,9 @@
+import '@/bootstrap.js' // This has side-effects and has to be imported first
+
+import { startServer } from '@/server/server.js'
+
+const start = () => {
+  startServer()
+}
+
+start()

packages/monitor-deployment/src/bootstrap.ts

@@ -0,0 +1,2 @@
+import dotenv from 'dotenv'
+dotenv.config()