Open Telemetry Integration (#569)

* Integrate the Armory app with Open Telemetry

* Add OTEL module factory class

* Add OTEL section in the README

Add docker/otel/up command.

* Add OTEL module in the Policy Engine and Vault

* Add stateful implementation of metric and trace service

* Correct OTEL auto instrument

* Refactor OTEL instrumentation to ensure it works correctly

Instrument Prisma.

* Remove unused tracing file

* Remove DiagLogLevel from the Auth

* Refactor tests to better track metrics

* Use forTest

* Fix @narval/open-telemetry in the package-lock.json

* Remove custom logger tracing

* Install OTEL winston transport

Lock OTEL packages version.

Makefile

@@ -100,8 +100,8 @@ docker/stop:
 docker/stack/up:
 	docker-compose up --detach
 
-docker/stack/stop:
-	docker-compose stop
+docker/otel/up:
+	docker-compose up postgres redis jaeger otel_collector --detach
 
 docker/stack/build:
 	docker buildx build \
@@ -123,4 +123,4 @@ git/tag/push:
 	git tag armory-$(TAG)
 	git tag policy-engine-$(TAG)
 	git tag vault-$(TAG)
-	git push origin armory-$(TAG) policy-engine-$(TAG) vault-$(TAG)
+	git push origin armory-$(TAG) policy-engine-$(TAG) vault-$(TAG)

README.md

@@ -60,7 +60,7 @@ Alternatively, you can run the entire development stack in Docker containers.
 make docker/stack/build
 
 make docker/stack/up
-make docker/stack/stop
+make docker/stop
 ```
 
 ## Testing
@@ -116,7 +116,7 @@ below to generate a project of your choice.
 
 ```bash
 # Generate an standard JavaScript library.
-npx nx g @nrwl/workspace:lib
+npx nx g @nrwl/js:lib 
 # Generate an NestJS library.
 npx nx g @nx/nest:library
 # Generate an NestJS application.
@@ -142,6 +142,46 @@ packages to NPM.
 You can find the publishable packages listed in the `release.projects` value in
 the `nx.json`.
 
+## OpenTelemetry
+
+The Armory Stack uses [OpenTelemetry](https://opentelemetry.io/docs/) (OTEL)
+for observability through traces and metrics.
+
+For details on how OTEL is instrumented, head over
+[@narval/open-telemetry](./packages/open-telemetry/).
+
+### Setup
+
+1. Start local dependencies:
+
+```bash
+make docker/otel/up
+```
+
+2. Set environment variables:
+
+```bash
+OTEL_SDK_DISABLED=false
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+```
+
+> [!NOTE]
+> OTEL is disabled by default in development. 
+
+3. Restart the application.
+
+4. Access Jaeger UI: http://localhost:16686
+
+### Naming Conventions
+
+- Traces: `operation.entity.action` (e.g., `policyStore.policies.update`)
+- Metrics: `system_entity_unit_state` (e.g., `policy_store_updates_total`) 
+
+See [Metric Naming](./packages/nestjs-shared/src/lib/module/open-telemetry/service/metric.service.ts)
+and [Trace/Span
+Name](./packages/nestjs-shared/src/lib/module/open-telemetry/service/trace.service.ts)
+adopted conventions.
+
 ## Troubleshooting
 
 ### DB URL in env variable fails when using `docker run`, but works when running outside docker

apps/armory/.env.default

@@ -12,7 +12,8 @@ ADMIN_API_KEY=171c50ec62122b8c08362dcf9dce9b016ed615cfc7b90d4bc3fe5b223d967fb2
 # APP db connection string
 APP_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/armory?schema=public
 
-# MIGRATOR db credentials. host/port/name should be the same, username&password may be different
+# Migrator db credentials.
+# The host/port/name should be the same, username&password may be different
 APP_DATABASE_USERNAME=postgres
 APP_DATABASE_PASSWORD=postgres
 APP_DATABASE_HOST=host.docker.internal
@@ -30,4 +31,9 @@ HISTORICAL_TRANSFER_FEED_PRIVATE_KEY=0xf5c8f17cc09215c5038f6b8d5e557c0d98d341236
 POLICY_ENGINE_URLS=http://localhost:3010
 POLICY_ENGINE_ADMIN_API_KEYS=engine-admin-api-key
 
-MANAGED_DATASTORE_BASE_URL=http://localhost:3005/v1/data
+MANAGED_DATASTORE_BASE_URL=http://localhost:3005/v1/data
+
+# OpenTelemetry configuration.
+OTEL_SDK_DISABLED=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+OTEL_LOGS_EXPORTER=otlp

apps/armory/src/armory.module.ts

@@ -1,5 +1,5 @@
 import { ConfigModule } from '@narval/config-module'
-import { HttpLoggerMiddleware, LoggerModule } from '@narval/nestjs-shared'
+import { HttpLoggerMiddleware, LoggerModule, OpenTelemetryModule } from '@narval/nestjs-shared'
 import { ClassSerializerInterceptor, MiddlewareConsumer, Module, NestModule } from '@nestjs/common'
 import { APP_INTERCEPTOR } from '@nestjs/core'
 import { AppModule } from './app/app.module'
@@ -18,7 +18,8 @@ const INFRASTRUCTURE_MODULES = [
     load: [load],
     isGlobal: true
   }),
-  QueueModule.forRoot()
+  QueueModule.forRoot(),
+  OpenTelemetryModule.forRoot()
 ]
 
 const DOMAIN_MODULES = [AppModule, OrchestrationModule, TransferTrackingModule, ManagedDataStoreModule, ClientModule]

apps/armory/src/client/__test__/e2e/client.spec.ts

@@ -1,5 +1,5 @@
 import { ConfigModule, ConfigService } from '@narval/config-module'
-import { LoggerModule, secret } from '@narval/nestjs-shared'
+import { LoggerModule, OpenTelemetryModule, secret } from '@narval/nestjs-shared'
 import { DataStoreConfiguration, HttpSource, PublicClient, Source, SourceType } from '@narval/policy-engine-shared'
 import { getPublicKey, privateKeyToJwk } from '@narval/signature'
 import { HttpStatus, INestApplication } from '@nestjs/common'
@@ -77,6 +77,7 @@ describe('Client', () => {
           load: [load],
           isGlobal: true
         }),
+        OpenTelemetryModule.forTest(),
         ClientModule
       ]
     }).compile()

apps/armory/src/main.ts

@@ -1,3 +1,11 @@
+import { instrumentOpenTelemetry } from '@narval/open-telemetry'
+
+// IMPORTANT: OpenTelemetry SDK must be registered before any other imports to
+// ensure proper instrumentation. The instrumentation packages patches Node.js
+// runtime - if NestFactory or other dependencies load first, they'll use the
+// unpatched runtime and won't be instrumented correctly.
+instrumentOpenTelemetry({ serviceName: 'auth' })
+
 import { ConfigService } from '@narval/config-module'
 import { LoggerService, withApiVersion, withCors, withLogger, withSwagger } from '@narval/nestjs-shared'
 import { ClassSerializerInterceptor, INestApplication, ValidationPipe } from '@nestjs/common'

apps/armory/src/managed-data-store/__test__/e2e/data-store.spec.ts

@@ -1,5 +1,5 @@
 import { ConfigModule } from '@narval/config-module'
-import { LoggerModule, secret } from '@narval/nestjs-shared'
+import { LoggerModule, OpenTelemetryModule, secret } from '@narval/nestjs-shared'
 import {
   Criterion,
   Entities,
@@ -134,6 +134,7 @@ describe('Data Store', () => {
           load: [load],
           isGlobal: true
         }),
+        OpenTelemetryModule.forTest(),
         ManagedDataStoreModule
       ]
     })

apps/armory/src/managed-data-store/core/service/entity-data-store.service.ts

@@ -1,5 +1,7 @@
+import { MetricService, TraceService } from '@narval/nestjs-shared'
 import { EntityStore } from '@narval/policy-engine-shared'
-import { HttpStatus, Injectable, NotFoundException } from '@nestjs/common'
+import { HttpStatus, Inject, Injectable, NotFoundException } from '@nestjs/common'
+import { Counter } from '@opentelemetry/api'
 import { ClientService } from '../../../client/core/service/client.service'
 import { ClusterService } from '../../../policy-engine/core/service/cluster.service'
 import { SetEntityStoreResponse } from '../../http/rest/dto/set-entity-store.dto'
@@ -8,21 +10,45 @@ import { SignatureService } from './signature.service'
 
 @Injectable()
 export class EntityDataStoreService extends SignatureService {
+  private getCounter: Counter
+  private setCounter: Counter
+
   constructor(
     private entityDataStoreRepository: EntityDataStoreRepository,
     private clientService: ClientService,
-    private clusterService: ClusterService
+    private clusterService: ClusterService,
+    @Inject(TraceService) private traceService: TraceService,
+    @Inject(MetricService) private metricService: MetricService
   ) {
     super()
+
+    this.getCounter = this.metricService.createCounter('entity_data_store_get_count')
+    this.setCounter = this.metricService.createCounter('entity_data_store_set_count')
   }
 
   async getEntities(clientId: string): Promise<EntityStore | null> {
+    this.getCounter.add(1, { clientId })
+
+    const span = this.traceService.startSpan(`${EntityDataStoreService.name}.getEntities`, {
+      attributes: { clientId }
+    })
+
     const entityStore = await this.entityDataStoreRepository.getLatestDataStore(clientId)
 
-    return entityStore ? EntityStore.parse(entityStore.data) : null
+    const response = entityStore ? EntityStore.parse(entityStore.data) : null
+
+    span.end()
+
+    return response
   }
 
   async setEntities(clientId: string, payload: EntityStore) {
+    this.setCounter.add(1, { clientId })
+
+    const span = this.traceService.startSpan(`${EntityDataStoreService.name}.setEntities`, {
+      attributes: { clientId }
+    })
+
     const client = await this.clientService.findById(clientId)
 
     if (!client) {
@@ -47,10 +73,14 @@ export class EntityDataStoreService extends SignatureService {
 
     const success = await this.clusterService.sync(clientId)
 
-    return SetEntityStoreResponse.parse({
+    const response = SetEntityStoreResponse.parse({
       latestSync: { success },
       entity: EntityStore.parse(data),
       version
     })
+
+    span.end()
+
+    return response
   }
 }

apps/armory/src/managed-data-store/core/service/policy-data-store.service.ts

@@ -1,27 +1,53 @@
+import { MetricService, TraceService } from '@narval/nestjs-shared'
 import { PolicyStore } from '@narval/policy-engine-shared'
-import { HttpStatus, Injectable, NotFoundException } from '@nestjs/common'
+import { HttpStatus, Inject, Injectable, NotFoundException } from '@nestjs/common'
+import { Counter } from '@opentelemetry/api'
 import { ClientService } from '../../../client/core/service/client.service'
 import { ClusterService } from '../../../policy-engine/core/service/cluster.service'
 import { PolicyDataStoreRepository } from '../../persistence/repository/policy-data-store.repository'
 import { SignatureService } from './signature.service'
 
 @Injectable()
 export class PolicyDataStoreService extends SignatureService {
+  private getCounter: Counter
+  private setCounter: Counter
+
   constructor(
     private policyDataStoreRepository: PolicyDataStoreRepository,
     private clientService: ClientService,
-    private clusterService: ClusterService
+    private clusterService: ClusterService,
+    @Inject(TraceService) private traceService: TraceService,
+    @Inject(MetricService) private metricService: MetricService
   ) {
     super()
+
+    this.getCounter = this.metricService.createCounter('policy_data_store_get_count')
+    this.setCounter = this.metricService.createCounter('policy_data_store_set_count')
   }
 
   async getPolicies(clientId: string): Promise<PolicyStore | null> {
+    this.getCounter.add(1, { clientId })
+
+    const span = this.traceService.startSpan(`${PolicyDataStoreService.name}.getPolicies`, {
+      attributes: { clientId }
+    })
+
     const policyStore = await this.policyDataStoreRepository.getLatestDataStore(clientId)
 
-    return policyStore ? PolicyStore.parse(policyStore.data) : null
+    const response = policyStore ? PolicyStore.parse(policyStore.data) : null
+
+    span.end()
+
+    return response
   }
 
   async setPolicies(clientId: string, payload: PolicyStore) {
+    this.setCounter.add(1, { clientId })
+
+    const span = this.traceService.startSpan(`${PolicyDataStoreService.name}.setPolicies`, {
+      attributes: { clientId }
+    })
+
     const client = await this.clientService.findById(clientId)
 
     if (!client) {
@@ -46,10 +72,14 @@ export class PolicyDataStoreService extends SignatureService {
 
     const success = await this.clusterService.sync(clientId)
 
-    return {
+    const response = {
       latestSync: { success },
       policy: PolicyStore.parse(data),
       version
     }
+
+    span.end()
+
+    return response
   }
 }

apps/armory/src/managed-data-store/http/rest/controller/data-store.controller.ts

@@ -93,8 +93,8 @@ export class DataStoreController {
     status: HttpStatus.CREATED,
     type: SetEntityStoreResponseDto
   })
-  setEntities(@Query('clientId') clientId: string, @Body() body: SetEntityStoreDto) {
-    return this.entityDataStoreService.setEntities(clientId, body)
+  async setEntities(@Query('clientId') clientId: string, @Body() body: SetEntityStoreDto) {
+    return await this.entityDataStoreService.setEntities(clientId, body)
   }
 
   @Post('/policies')
@@ -105,11 +105,11 @@ export class DataStoreController {
     status: HttpStatus.CREATED,
     type: SetPolicyStoreResponseDto
   })
-  setPolicies(
+  async setPolicies(
     @Query('clientId') clientId: string,
     @Body() body: SetPolicyStoreDto
   ): Promise<SetPolicyStoreResponseDto> {
-    return this.policyDataStoreService.setPolicies(clientId, body)
+    return await this.policyDataStoreService.setPolicies(clientId, body)
   }
 
   @Post('/sync')

apps/armory/src/orchestration/__test__/e2e/authorization-request.spec.ts

@@ -1,5 +1,5 @@
 import { ConfigModule } from '@narval/config-module'
-import { LoggerModule, secret } from '@narval/nestjs-shared'
+import { LoggerModule, OpenTelemetryModule, secret } from '@narval/nestjs-shared'
 import { Action, AuthorizationRequest, Eip712TypedData, FIXTURE, Request } from '@narval/policy-engine-shared'
 import { getQueueToken } from '@nestjs/bull'
 import { HttpStatus, INestApplication } from '@nestjs/common'
@@ -56,6 +56,7 @@ describe('Authorization Request', () => {
         }),
         QueueModule.forRoot(),
         PersistenceModule,
+        OpenTelemetryModule.forTest(),
         OrchestrationModule
       ]
     })

apps/armory/src/orchestration/orchestration.module.ts

@@ -1,11 +1,10 @@
 import { BullAdapter } from '@bull-board/api/bullAdapter'
 import { BullBoardModule } from '@bull-board/nestjs'
-import { ConfigModule } from '@narval/config-module'
 import { HttpModule } from '@nestjs/axios'
 import { BullModule } from '@nestjs/bull'
 import { Module } from '@nestjs/common'
 import { ConditionalModule } from '@nestjs/config'
-import { Env, isEnv, load } from '../armory.config'
+import { Env, isEnv } from '../armory.config'
 import { AUTHORIZATION_REQUEST_PROCESSING_QUEUE, DEFAULT_HTTP_MODULE_PROVIDERS } from '../armory.constant'
 import { ClientModule } from '../client/client.module'
 import { DataFeedModule } from '../data-feed/data-feed.module'
@@ -23,7 +22,6 @@ import { AuthorizationRequestProcessingConsumer } from './queue/consumer/authori
 import { AuthorizationRequestProcessingProducer } from './queue/producer/authorization-request-processing.producer'
 
 const INFRASTRUCTURE_MODULES = [
-  ConfigModule.forRoot({ load: [load] }),
   HttpModule,
   PersistenceModule,
   // TODO (@wcalderipe, 11/01/24): Figure out why can't I have a wrapper to

apps/armory/src/orchestration/queue/consumer/__test__/integration/authorization-request-processing.consumer.spec.ts

@@ -1,5 +1,5 @@
 import { ConfigModule } from '@narval/config-module'
-import { LoggerModule, secret } from '@narval/nestjs-shared'
+import { LoggerModule, OpenTelemetryModule, secret } from '@narval/nestjs-shared'
 import {
   Action,
   AuthorizationRequest,
@@ -99,6 +99,7 @@ describe(AuthorizationRequestProcessingConsumer.name, () => {
         BullModule.registerQueue({
           name: AUTHORIZATION_REQUEST_PROCESSING_QUEUE
         }),
+        OpenTelemetryModule.forTest(),
         HttpModule,
         PolicyEngineModule
       ],