fix(api-cardano-db-hasura): mitigate unexpected Ogmios socket disconnect

Ogmios disconnects the WebSocket when attempting to submit some invalid payloads, and there's no recovery. This mitigation gracefully handles the scenario by shutting down the client and re-initializes it to re-establish the connection.
cardano-foundation · Sep 16, 2022 · 3d81bf5 · 3d81bf5
1 parent df82008
commit 3d81bf5
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 9 deletions.
diff --git a/packages/api-cardano-db-hasura/src/CardanoNodeClient.ts b/packages/api-cardano-db-hasura/src/CardanoNodeClient.ts
@@ -62,11 +62,15 @@ export class CardanoNodeClient {
     )
     await pRetry(async () => {
       await this.serverHealthFetcher.initialize()
+      const onClose = async () => {
+        await this.shutdown()
+        await this.initialize(ogmiosConnectionConfig)
+      }
       this.stateQueryClient = await createStateQueryClient(
-        await createInteractionContextWithLogger(ogmiosConnectionConfig, this.logger, MODULE_NAME)
+        await createInteractionContextWithLogger(ogmiosConnectionConfig, this.logger, MODULE_NAME, onClose)
       )
       this.txSubmissionClient = await createTxSubmissionClient(
-        await createInteractionContextWithLogger(ogmiosConnectionConfig, this.logger, MODULE_NAME)
+        await createInteractionContextWithLogger(ogmiosConnectionConfig, this.logger, MODULE_NAME, onClose)
       )
     }, {
       factor: 1.2,
@@ -86,6 +90,7 @@ export class CardanoNodeClient {
       this.stateQueryClient.shutdown,
       this.txSubmissionClient.shutdown
     ])
+    this.state = null
   }
 
   public async submitTransaction (transaction: string): Promise<Transaction['hash']> {

diff --git a/packages/api-cardano-db-hasura/src/ChainFollower.ts b/packages/api-cardano-db-hasura/src/ChainFollower.ts
@@ -38,7 +38,10 @@ export class ChainFollower {
     this.state = 'initializing'
     this.logger.info({ module: MODULE_NAME }, 'Initializing')
     await pRetry(async () => {
-      const context = await createInteractionContextWithLogger(ogmiosConfig, this.logger, MODULE_NAME)
+      const context = await createInteractionContextWithLogger(ogmiosConfig, this.logger, MODULE_NAME, async () => {
+        await this.shutdown()
+        await this.initialize(ogmiosConfig)
+      })
       this.chainSyncClient = await createChainSyncClient(
         context,
         {
@@ -122,7 +125,7 @@ export class ChainFollower {
     this.logger.info({ module: MODULE_NAME }, 'Shutting down')
     await this.chainSyncClient.shutdown()
     await this.queue.stop()
-    this.state = 'initialized'
+    this.state = null
     this.logger.info(
       { module: MODULE_NAME },
       'Shutdown complete')

diff --git a/packages/api-cardano-db-hasura/src/util.ts b/packages/api-cardano-db-hasura/src/util.ts
@@ -1,16 +1,17 @@
 import { ConnectionConfig, createInteractionContext } from '@cardano-ogmios/client'
 import { Logger } from 'ts-log'
 
-export const createInteractionContextWithLogger = (connection: ConnectionConfig, logger: Logger, module: string) =>
+export const createInteractionContextWithLogger = (connection: ConnectionConfig, logger: Logger, module: string, onClose?: () => Promise<void>) =>
   createInteractionContext(
     (error) => {
       logger.error({ module, error }, error.message)
     },
-    (code, reason) => {
-      if (code === 1006) {
-        logger.error({ module, code }, 'Connection was closed abnormally')
-      } else {
+    async (code, reason) => {
+      if (code === 1000) {
         logger.info({ module, code }, reason)
+      } else {
+        logger.error({ module, code }, 'Connection closed')
+        await onClose?.()
       }
     },
     {