feat(api-cardano-db-hasura)!: separate DB and Hasura management tasks…

… to new process - introduce `db-manager` dir - adds `HasuraManagementClient`, moves Hasura setup steps from `HasuraClient` - adds `run` entrypoint and associated config - adds Docker build target and Docker Compose service - makes DB.init `onDbInit` optional.
cardano-foundation · Jan 30, 2023 · 8acb58f · 8acb58f
1 parent 494b2e2
commit 8acb58f
Show file tree

Hide file tree

Showing 15 changed files with 325 additions and 126 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -57,7 +57,7 @@ ENV \
   POSTGRES_USER_FILE=/run/secrets/postgres_user
 WORKDIR /src
 
-FROM ubuntu-nodejs as server
+FROM ubuntu-nodejs as db-manager
 ARG NETWORK=mainnet
 ARG METADATA_SERVER_URI="https://tokens.cardano.org"
 RUN curl --proto '=https' --tlsv1.2 -sSf -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - &&\
@@ -66,14 +66,9 @@ RUN curl --proto '=https' --tlsv1.2 -sSf -L https://www.postgresql.org/media/key
   ca-certificates
 COPY --from=downloader /usr/local/bin/hasura /usr/local/bin/hasura
 ENV \
-  CARDANO_NODE_CONFIG_PATH=/config/cardano-node/config.json \
   HASURA_CLI_PATH=/usr/local/bin/hasura \
-  HASURA_GRAPHQL_ENABLE_TELEMETRY=false \
   HASURA_URI="http://hasura:8080" \
   LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" \
-  METADATA_SERVER_URI=${METADATA_SERVER_URI} \
-  NETWORK=${NETWORK} \
-  OGMIOS_HOST="cardano-node-ogmios" \
   POSTGRES_DB_FILE=/run/secrets/postgres_db \
   POSTGRES_HOST=postgres \
   POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password \
@@ -83,6 +78,29 @@ COPY --from=cardano-graphql-builder /app/packages/api-cardano-db-hasura/dist /ap
 COPY --from=cardano-graphql-builder /app/packages/api-cardano-db-hasura/hasura/project /app/packages/api-cardano-db-hasura/hasura/project
 COPY --from=cardano-graphql-builder /app/packages/api-cardano-db-hasura/package.json /app/packages/api-cardano-db-hasura/package.json
 COPY --from=cardano-graphql-builder /app/packages/api-cardano-db-hasura/schema.graphql /app/packages/api-cardano-db-hasura/schema.graphql
+COPY --from=cardano-graphql-builder /app/packages/util/dist /app/packages/util/dist
+COPY --from=cardano-graphql-builder /app/packages/util/package.json /app/packages/util/package.json
+COPY --from=cardano-graphql-production-deps /app/node_modules /app/node_modules
+COPY --from=cardano-graphql-production-deps /app/packages/api-cardano-db-hasura/node_modules /app/packages/api-cardano-db-hasura/node_modules
+WORKDIR /app/packages/api-cardano-db-hasura/dist
+CMD ["node", "db-manager/run.js"]
+
+FROM ubuntu-nodejs as server
+ARG NETWORK=mainnet
+ARG METADATA_SERVER_URI="https://tokens.cardano.org"
+COPY --from=downloader /usr/local/bin/hasura /usr/local/bin/hasura
+ENV \
+  CARDANO_NODE_CONFIG_PATH=/config/cardano-node/config.json \
+  HASURA_GRAPHQL_ENABLE_TELEMETRY=false \
+  HASURA_URI="http://hasura:8080" \
+  LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" \
+  METADATA_SERVER_URI=${METADATA_SERVER_URI} \
+  NETWORK=${NETWORK} \
+  OGMIOS_HOST="cardano-node-ogmios"
+COPY --from=cardano-graphql-builder /app/packages/api-cardano-db-hasura/dist /app/packages/api-cardano-db-hasura/dist
+COPY --from=cardano-graphql-builder /app/packages/api-cardano-db-hasura/hasura/project /app/packages/api-cardano-db-hasura/hasura/project
+COPY --from=cardano-graphql-builder /app/packages/api-cardano-db-hasura/package.json /app/packages/api-cardano-db-hasura/package.json
+COPY --from=cardano-graphql-builder /app/packages/api-cardano-db-hasura/schema.graphql /app/packages/api-cardano-db-hasura/schema.graphql
 COPY --from=cardano-graphql-builder /app/packages/server/dist /app/packages/server/dist
 COPY --from=cardano-graphql-builder /app/packages/server/package.json /app/packages/server/package.json
 COPY --from=cardano-graphql-builder /app/packages/util/dist /app/packages/util/dist

diff --git a/README.md b/README.md
@@ -114,7 +114,8 @@ your use-case.
 Get the most recent weekly snapshot link [here](https://update-cardano-mainnet.iohk.io/cardano-db-sync/index.html#11/), and set it as `RESTORE_SNAPSHOT` below, or omit if you wish to sync from genesis.
 ``` console
 export NETWORK=mainnet &&\
-docker pull inputoutput/cardano-graphql:7.0.2-${NETWORK} &&\
+docker pull inputoutput/cardano-graphql-server:7.0.2-${NETWORK} &&\
+docker pull inputoutput/cardano-graphql-background:7.0.2-${NETWORK} &&\
 docker pull inputoutput/cardano-graphql-hasura:7.0.2 &&\
 docker pull cardanosolutions/cardano-node-ogmios:v5.5.8_1.35.5-${NETWORK} &&\
 RESTORE_SNAPSHOT=https://update-cardano-mainnet.iohk.io/cardano-db-sync/13/db-sync-snapshot-schema-13-block-8291499-x86_64.tgz \
@@ -128,7 +129,8 @@ docker compose logs -f
 
 ``` console
 export NETWORK=preprod &&\
-docker pull inputoutput/cardano-graphql:7.0.2-${NETWORK} &&\
+docker pull inputoutput/cardano-graphql-server:7.0.2-${NETWORK} &&\
+docker pull inputoutput/cardano-graphql-background:7.0.2-${NETWORK} &&\
 docker pull inputoutput/cardano-graphql-hasura:7.0.2 &&\
 docker pull cardanosolutions/cardano-node-ogmios:v5.5.8_1.35.5-${NETWORK} &&\
 API_PORT=3101 \
@@ -146,7 +148,8 @@ docker compose -p ${NETWORK} logs -f
 
 ``` console
 export NETWORK=preview &&\
-docker pull inputoutput/cardano-graphql:7.0.2-${NETWORK} &&\
+docker pull inputoutput/cardano-graphql-server:7.0.2-${NETWORK} &&\
+docker pull inputoutput/cardano-graphql-background:7.0.2-${NETWORK} &&\
 docker pull inputoutput/cardano-graphql-hasura:7.0.2 &&\
 docker pull cardanosolutions/cardano-node-ogmios:v5.5.8_1.35.5-${NETWORK} &&\
 API_PORT=3102 \

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -98,7 +98,29 @@ services:
         max-size: "200k"
         max-file: "10"
 
-  cardano-graphql:
+  db-manager:
+    build:
+      cache_from: [ inputoutput/cardano-graphql-db-manager:latest ]
+      context: .
+      target: db-manager
+    image: inputoutput/cardano-graphql-db-manager:${CARDANO_GRAPHQL_VERSION:-7.0.2}-${NETWORK:-mainnet}
+    depends_on:
+      - "hasura"
+      - "postgres"
+    environment:
+      - LOGGER_MIN_SEVERITY=${LOGGER_MIN_SEVERITY:-info}
+    restart: on-failure
+    secrets:
+      - postgres_db
+      - postgres_password
+      - postgres_user
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "200k"
+        max-file: "10"
+
+  http-server:
     build:
       args:
         - NETWORK=${NETWORK:-mainnet}
@@ -116,10 +138,6 @@ services:
     ports:
       - ${API_PORT:-3100}:3100
     restart: on-failure
-    secrets:
-      - postgres_db
-      - postgres_password
-      - postgres_user
     logging:
       driver: "json-file"
       options:

diff --git a/package.json b/package.json
@@ -21,30 +21,35 @@
     "mainnet:up": "yarn mainnet:stack && docker compose -p mainnet logs -f",
     "mainnet:down": "docker compose -p mainnet down --remove-orphans",
     "mainnet:server": "CONTEXT=$PWD NETWORK=mainnet . ./scripts/export_env.sh && yarn workspace @cardano-graphql/server start",
+    "mainnet:db-manager": "CONTEXT=$PWD NETWORK=mainnet . ./scripts/export_env.sh && yarn workspace @cardano-graphql/api-cardano-db-hasura db-manager",
     "preprod:stack": "CONTEXT=$PWD NETWORK=preprod . ./scripts/export_env.sh && docker compose -p preprod up --build -d",
     "preprod:dev": "CONTEXT=$PWD NETWORK=preprod . ./scripts/export_env.sh && yarn service-dependencies -p preprod up --build",
     "preprod:up": "yarn preprod:stack && docker compose -p preprod logs -f",
     "preprod:down": "docker compose -p preprod down --remove-orphans",
     "preprod:server": "CONTEXT=$PWD NETWORK=preprod . ./scripts/export_env.sh && yarn workspace @cardano-graphql/server start",
+    "preprod:db-manager": "CONTEXT=$PWD NETWORK=preprod . ./scripts/export_env.sh && yarn workspace @cardano-graphql/api-cardano-db-hasura db-manager",
     "preview:stack": "CONTEXT=$PWD NETWORK=preview . ./scripts/export_env.sh && docker compose -p preview up --build -d",
     "preview:dev": "CONTEXT=$PWD NETWORK=preview . ./scripts/export_env.sh && yarn service-dependencies -p preview up --build",
     "preview:up": "yarn preview:stack && docker compose -p preview logs -f",
     "preview:down": "docker compose -p preview down --remove-orphans",
     "preview:server": "CONTEXT=$PWD NETWORK=preview . ./scripts/export_env.sh && yarn workspace @cardano-graphql/server start",
+    "preview:db-manager": "CONTEXT=$PWD NETWORK=preview . ./scripts/export_env.sh && yarn workspace @cardano-graphql/api-cardano-db-hasura db-manager",
     "testnet:stack": "CONTEXT=$PWD NETWORK=testnet . ./scripts/export_env.sh && docker compose -p testnet up --build -d",
     "testnet:dev": "CONTEXT=$PWD NETWORK=testnet . ./scripts/export_env.sh && yarn service-dependencies -p testnet up --build",
     "testnet:up": "yarn testnet:stack && docker compose -p testnet logs -f",
     "testnet:down": "docker compose -p testnet down --remove-orphans",
     "testnet:server": "CONTEXT=$PWD NETWORK=testnet . ./scripts/export_env.sh && yarn workspace @cardano-graphql/server start",
+    "testnet:db-manager": "CONTEXT=$PWD NETWORK=testnet . ./scripts/export_env.sh && yarn workspace @cardano-graphql/api-cardano-db-hasura db-manager",
     "vasil-dev:stack": "CONTEXT=$PWD NETWORK=vasil-dev . ./scripts/export_env.sh && docker compose -p vasil-dev up --build -d",
     "vasil-dev:dev": "CONTEXT=$PWD NETWORK=vasil-dev . ./scripts/export_env.sh && yarn service-dependencies -p vasil-dev up --build",
     "vasil-dev:up": "yarn vasil-dev:stack && docker compose -p vasil-dev logs -f",
     "vasil-dev:down": "docker compose -p vasil-dev down --remove-orphans",
     "vasil-dev:server": "CONTEXT=$PWD NETWORK=vasil-dev . ./scripts/export_env.sh && yarn workspace @cardano-graphql/server start",
+    "vasil-dev:db-manager": "CONTEXT=$PWD NETWORK=vasil-dev . ./scripts/export_env.sh && yarn workspace @cardano-graphql/api-cardano-db-hasura db-manager",
     "lint": "yarn workspaces run lint",
     "loadtest:byron-staging": "artillery run test/loadtest/byron-staging-config.yml",
     "publish-packages": "yarn workspaces run publish",
-    "service-dependencies": "docker compose config --services | grep -v \"cardano-graphql\" | xargs docker compose",
+    "service-dependencies": "docker-compose config --services | grep -vE \"server|db-manager\" | xargs docker-compose",
     "test": "yarn workspaces run test"
   },
   "contributors": [

diff --git a/packages/api-cardano-db-hasura/package.json b/packages/api-cardano-db-hasura/package.json
@@ -10,6 +10,8 @@
     "cleanup": "shx rm -rf dist node_modules src/graphql_types.ts",
     "lint": "eslint --ignore-path ../../.eslintignore \"**/*.ts\"",
     "prepack": "yarn build",
+    "predb-manager": "yarn build",
+    "db-manager": "node dist/db-manager/run.js",
     "test": "yarn build && NODE_ENV=test jest -c ./test/jest.config.js"
   },
   "repository": {

diff --git a/packages/api-cardano-db-hasura/src/Config.ts b/packages/api-cardano-db-hasura/src/Config.ts
@@ -1,14 +1,12 @@
-
 export interface Config {
   cardanoNodeConfigPath: string,
-  db: {
-    database: string,
-    host: string,
-    password: string,
-    port: number
-    user: string,
+  db?: {
+    database?: string,
+    host?: string,
+    password?: string,
+    port?: number
+    user?: string,
   },
-  hasuraCliPath: string,
   hasuraUri: string,
   metadataServerUri: string,
   metadataUpdateInterval?: {

diff --git a/packages/api-cardano-db-hasura/src/HasuraClient.ts b/packages/api-cardano-db-hasura/src/HasuraClient.ts
@@ -1,12 +1,10 @@
 import { Schema } from '@cardano-ogmios/client'
-import { exec } from 'child_process'
 import util, { DataFetcher, ModuleState } from '@cardano-graphql/util'
 import fetch from 'cross-fetch'
 import { DocumentNode, GraphQLSchema, print } from 'graphql'
 import { GraphQLClient, gql } from 'graphql-request'
 import { introspectSchema, wrapSchema } from '@graphql-tools/wrap'
 import pRetry from 'p-retry'
-import path from 'path'
 import {
   AdaPots,
   Asset,
@@ -34,19 +32,16 @@ const withHexPrefix = (value: string) => `\\x${value !== undefined ? value : ''}
 
 export class HasuraClient {
   private client: GraphQLClient
-  private applyingSchemaAndMetadata: boolean
   public adaPotsToCalculateSupplyFetcher: DataFetcher<AdaPotsToCalculateSupply>
   private state: ModuleState
   public schema: GraphQLSchema
 
   constructor (
-    readonly hasuraCliPath: string,
     readonly hasuraUri: string,
     pollingInterval: number,
     private logger: Logger = dummyLogger
   ) {
     this.state = null
-    this.applyingSchemaAndMetadata = false
     this.adaPotsToCalculateSupplyFetcher = new DataFetcher<AdaPotsToCalculateSupply>(
       'AdaPotsToCalculateSupply',
       () => {
@@ -123,26 +118,10 @@ export class HasuraClient {
     }
   }
 
-  private async hasuraCli (command: string) {
-    return new Promise((resolve, reject) => {
-      exec(
-        `${this.hasuraCliPath} --skip-update-check --project ${path.resolve(__dirname, '..', 'hasura', 'project')} --endpoint ${this.hasuraUri} ${command}`,
-        (error, stdout) => {
-          if (error) {
-            reject(error)
-          }
-          this.logger.debug({ module: 'HasuraClient' }, stdout)
-          resolve()
-        }
-      )
-    })
-  }
-
   public async initialize () {
     if (this.state !== null) return
     this.state = 'initializing'
     this.logger.info({ module: 'HasuraClient' }, 'Initializing')
-    await this.applySchemaAndMetadata()
     await pRetry(async () => {
       this.schema = await this.buildHasuraSchema()
     }, {
@@ -154,27 +133,6 @@ export class HasuraClient {
       )
     })
     this.logger.debug({ module: 'HasuraClient' }, 'graphql-engine setup')
-    await pRetry(async () => {
-      const result = await this.client.request(
-        gql`query {
-            epochs (limit: 1, order_by: { number: desc }) {
-                number
-            }
-        }`
-      )
-      if (result.epochs.length === 0) {
-        this.logger.debug({ module: 'HasuraClient' }, epochInformationNotYetAvailable)
-        throw new Error(epochInformationNotYetAvailable)
-      }
-    }, {
-      factor: 1.05,
-      retries: 100,
-      onFailedAttempt: util.onFailedAttemptFor(
-        'Detecting DB sync state has reached minimum progress',
-        this.logger
-      )
-    })
-    this.logger.debug({ module: 'HasuraClient' }, 'DB sync state has reached minimum progress')
     await this.adaPotsToCalculateSupplyFetcher.initialize()
     this.state = 'initialized'
     this.logger.info({ module: 'HasuraClient' }, 'Initialized')
@@ -184,31 +142,6 @@ export class HasuraClient {
     await this.adaPotsToCalculateSupplyFetcher.shutdown()
   }
 
-  public async applySchemaAndMetadata (): Promise<void> {
-    if (this.applyingSchemaAndMetadata) return
-    this.applyingSchemaAndMetadata = true
-    await pRetry(async () => {
-      await this.hasuraCli('migrate apply --down all')
-      await this.hasuraCli('migrate apply --up all')
-    }, {
-      factor: 1.75,
-      retries: 9,
-      onFailedAttempt: util.onFailedAttemptFor(
-        'Applying PostgreSQL schema migrations',
-        this.logger
-      )
-    })
-    await pRetry(async () => {
-      await this.hasuraCli('metadata clear')
-      await this.hasuraCli('metadata apply')
-    }, {
-      factor: 1.75,
-      retries: 9,
-      onFailedAttempt: util.onFailedAttemptFor('Applying Hasura metadata', this.logger)
-    })
-    this.applyingSchemaAndMetadata = false
-  }
-
   public async buildHasuraSchema () {
     const executor = async ({ document, variables }: { document: DocumentNode, variables?: Object }) => {
       const query = print(document)

diff --git a/packages/api-cardano-db-hasura/src/Db.ts → ...pi-cardano-db-hasura/src/db-manager/Db.ts b/packages/api-cardano-db-hasura/src/Db.ts → ...pi-cardano-db-hasura/src/db-manager/Db.ts
@@ -23,7 +23,7 @@ export class Db {
     onDbInit,
     onDbSetup
   }: {
-    onDbInit: () => void,
+    onDbInit?: () => void,
     onDbSetup: () => void
   }): Promise<void> {
     if (this.state !== null) return