fix: metrics computed async (#1085)

web3-storage · Mar 9, 2022 · 99b52e5 · 99b52e5
1 parent 22f4170
commit 99b52e5
Show file tree

Hide file tree

Showing 12 changed files with 308 additions and 349 deletions.
diff --git a/.github/workflows/cron-metrics.yml b/.github/workflows/cron-metrics.yml
@@ -0,0 +1,30 @@
+name: Cron Metrics
+
+on:
+  schedule:
+    - cron: '*/20 * * * *'
+  workflow_dispatch:
+
+jobs:
+  update:
+    name: Calculate metrics
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        env: ['staging', 'production']
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+      - uses: bahmutov/npm-install@v1
+      - name: Run job
+        env:
+          DEBUG: '*'
+          ENV: ${{ matrix.env }}
+          STAGING_PG_CONNECTION: ${{ secrets.STAGING_PG_CONNECTION }}
+          STAGING_RO_PG_CONNECTION: ${{ secrets.STAGING_PG_CONNECTION }} # no replica for staging
+          PROD_PG_CONNECTION: ${{ secrets.PROD_PG_CONNECTION }}
+          PROD_RO_PG_CONNECTION: ${{ secrets.PROD_RO_PG_CONNECTION }}
+        run: yarn --cwd packages/cron start:metrics
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/api/src/metrics.js b/packages/api/src/metrics.js
@@ -32,15 +32,15 @@ export async function metricsGet (request, env, ctx) {
     Promise.all(
       UPLOAD_TYPES.map(async (t) => ({
         type: t,
-        total: await env.db.getMetricsValue(t)
+        total: await env.db.getMetricsValue(`uploads_${t.toLowerCase()}_total`)
       }))
     ),
     env.db.getMetricsValue('content_bytes_total'),
     env.db.getMetricsValue('pins_total'),
     Promise.all(
       PIN_STATUSES.map(async (s) => ({
         status: s,
-        total: await env.db.getMetricsValue(s)
+        total: await env.db.getMetricsValue(`pins_${s.toLowerCase()}_total`)
       }))
     ),
     env.db.getMetricsValue('pin_requests_total')

diff --git a/packages/cron/package.json b/packages/cron/package.json
@@ -7,7 +7,7 @@
   "type": "module",
   "scripts": {
     "start": "run-s start:*",
-    "start:metrics": "node src/bin/metrics.js",
+    "start:metrics": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/metrics.js",
     "start:pins": "node src/bin/pins.js",
     "start:dagcargo:sizes": "NODE_TLS_REJECT_UNAUTHORIZED=0 node src/bin/dagcargo-sizes.js",
     "test": "npm-run-all -p -r mock:cluster mock:pgrest test:e2e",
@@ -27,6 +27,7 @@
     "multiformats": "^9.6.2",
     "node-fetch": "^2.6.1",
     "p-retry": "^4.6.1",
+    "p-settle": "^5.0.0",
     "pg": "^8.7.1",
     "piggybacker": "^2.0.0"
   },

diff --git a/packages/cron/src/bin/metrics.js b/packages/cron/src/bin/metrics.js
@@ -2,13 +2,18 @@
 
 import { updateMetrics } from '../jobs/metrics.js'
 import { envConfig } from '../lib/env.js'
-import { getDBClient } from '../lib/utils.js'
+import { getPgPool } from '../lib/utils.js'
 
 async function main () {
-  const env = process.env.ENV || 'dev'
-  const db = getDBClient(process.env)
+  const rwPg = getPgPool(process.env, 'rw')
+  const roPg = getPgPool(process.env, 'ro')
 
-  await updateMetrics({ env, db })
+  try {
+    await updateMetrics({ rwPg, roPg })
+  } finally {
+    await rwPg.end()
+    await roPg.end()
+  }
 }
 
 envConfig()