Open Policy Agent engine evaluation per tenant (#170)

Note: - Engine signing is currently a proof of concept. It's unclear how it will evolve with remote signing, so I'd rather not modify it at this point. - Building OPA targeting WASM involves a process with many steps that interact with the OS and its file system. The current implementation addresses the happy path and a few obvious errors. Therefore, error handling for edge cases is not yet implemented, and I expect these to be addressed as we encounter issues in the building process. - The whole implementation is in a single place (see `wasm-build.util.ts`). Changelog: - Added a core Engine interface along with an Open Policy Agent implementation of it, effectively hiding OPA as an implementation detail. Everything outside the `apps/policy-engine/src/open-policy-agent` directory SHOULD NOT have knowledge of OPA and Rego. - Added temporary code in the BootstrapService to add a development tenant pointing to the devtool stores. - Added a `resource` directory for the server to access files from the disk. - Added a `RESOURCE_PATH` environment variable because we can't depend on `__dirname` to resolve the path. Its value changes based on how the application is built. With webpack, it minimizes, and `__dirname` always points to the lowest level of the directory tree. By contrast, in tests, the directory tree is preserved because files are transformed on the fly. - Added the resource directory as a NestJS assets directory in `project.json`. - Changed the Rego core directory to `apps/policy-engine/src/resource`. - Removed most of the legacy OPA code. - Changed the tenant module location to the engine module due to numerous circular dependency issues in the DI container. - Note: it felt like a good example of unclear module boundaries, and I'd rather roll it back and see how it evolves than trying to address problems we SHOULD NOT have. Co-authored-by: samuel <[email protected]>
narval-xyz · Mar 21, 2024 · 286ce66 · 286ce66
1 parent 4703b62
commit 286ce66
Show file tree

Hide file tree

Showing 112 changed files with 1,925 additions and 1,894 deletions.
diff --git a/.github/workflows/policy-engine.yml b/.github/workflows/policy-engine.yml
@@ -41,6 +41,11 @@ jobs:
         with:
           node-version: '20.4.0'
 
+      - name: Install Open Policy Agent CLI
+        uses: open-policy-agent/setup-opa@v2
+        with:
+          version: latest
+
       - name: Install dependencies
         run: |
           make install/ci
@@ -103,5 +108,5 @@ jobs:
         with:
           version: latest
 
-      - name: Run OPA Tests
+      - name: Test rego
         run: make policy-engine/rego/test
diff --git a/apps/policy-engine/.env.default b/apps/policy-engine/.env.default
@@ -8,6 +8,8 @@ ENGINE_UID="local-dev-engine-instance-1"
 
 MASTER_PASSWORD="unsafe-local-dev-master-password"
 
+RESOURCE_PATH=./apps/policy-engine/src/resource
+
 KEYRING_TYPE="raw"
 
 # MASTER_AWS_KMS_ARN="arn:aws:kms:us-east-2:728783560968:key/f6aa3ddb-47c3-4f31-977d-b93205bb23d1"
diff --git a/apps/policy-engine/.env.test.default b/apps/policy-engine/.env.test.default
@@ -10,4 +10,6 @@ MASTER_PASSWORD="unsafe-local-test-master-password"
 
 KEYRING_TYPE="raw"
 
+RESOURCE_PATH=./apps/policy-engine/src/resource
+
 # MASTER_AWS_KMS_ARN="arn:aws:kms:us-east-2:728783560968:key/f6aa3ddb-47c3-4f31-977d-b93205bb23d1"
diff --git a/apps/policy-engine/Makefile b/apps/policy-engine/Makefile
@@ -1,6 +1,7 @@
 POLICY_ENGINE_PROJECT_NAME := policy-engine
 POLICY_ENGINE_PROJECT_DIR := ./apps/policy-engine
 POLICY_ENGINE_DATABASE_SCHEMA := ${POLICY_ENGINE_PROJECT_DIR}/src/shared/module/persistence/schema/schema.prisma
+POLICY_ENGINE_REGO_DIST = ./dist/rego
 
 # === Start ===
 
@@ -80,7 +81,6 @@ policy-engine/db/seed:
 	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
 		ts-node -r tsconfig-paths/register --project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json ${POLICY_ENGINE_PROJECT_DIR}/src/shared/module/persistence/seed.ts
 
-
 # === Testing ===
 
 policy-engine/test/db/setup:
@@ -90,7 +90,6 @@ policy-engine/test/db/setup:
 		--skip-seed \
 		--force
 
-
 policy-engine/test/type:
 	make policy-engine/db/generate-types
 	npx tsc \
@@ -131,37 +130,21 @@ policy-engine/cli:
 # === Open Policy Agent & Rego ===
 
 policy-engine/rego/build:
-	rm -rf ./rego-build
-	mkdir -p ./rego-build
+	rm -rf ${POLICY_ENGINE_REGO_DIST}
+	mkdir -p ${POLICY_ENGINE_REGO_DIST}
 	opa build \
 		--target wasm \
 		--entrypoint main/evaluate \
-		--bundle ${POLICY_ENGINE_PROJECT_DIR}/src/opa/rego \
+		--bundle ${POLICY_ENGINE_PROJECT_DIR}/src/resource/open-policy-agent/rego \
 		--ignore "__test__" \
 		--ignore "policies" \
-		--output ./rego-build/policies.gz
-	tar -xzf ./rego-build/policies.gz -C ./rego-build/
-
-policy-engine/rego/eval:
-	npx ts-node \
-		--compiler-options "{\"module\":\"CommonJS\"}" \
-		${POLICY_ENGINE_PROJECT_DIR}/src/opa/script/evaluation.script.ts
-
-policy-engine/rego/translate:
-	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
-		ts-node -r tsconfig-paths/register \
-		--project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json ${POLICY_ENGINE_PROJECT_DIR}/src/opa/script/translate-legacy-policy.script.ts
-
-policy-engine/rego/evaluation:
-	npx dotenv -e ${POLICY_ENGINE_PROJECT_DIR}/.env -- \
-		ts-node -r tsconfig-paths/register \
-		--project ${POLICY_ENGINE_PROJECT_DIR}/tsconfig.app.json ${POLICY_ENGINE_PROJECT_DIR}/src/opa/script/evaluate-legacy-policy.script.ts
+		--output ${POLICY_ENGINE_REGO_DIST}/bundle.tar.gz
+	tar -xzf ${POLICY_ENGINE_REGO_DIST}/bundle.tar.gz -C ${POLICY_ENGINE_REGO_DIST}
 
 policy-engine/rego/test:
 	opa test \
 	--format="pretty" \
-	${POLICY_ENGINE_PROJECT_DIR}/src/opa/rego \
-	--ignore "generated" \
+	${POLICY_ENGINE_PROJECT_DIR}/src/resource/open-policy-agent/rego \
 	--verbose \
 	${ARGS}
 

diff --git a/apps/policy-engine/project.json b/apps/policy-engine/project.json
@@ -13,6 +13,7 @@
         "compiler": "tsc",
         "outputPath": "dist/apps/policy-engine",
         "main": "apps/policy-engine/src/main.ts",
+        "assets": ["apps/policy-engine/src/resource"],
         "tsConfig": "apps/policy-engine/tsconfig.app.json",
         "isolatedConfig": true,
         "webpackConfig": "apps/policy-engine/webpack.config.js"

diff --git a/...ne/src/tenant/__test__/e2e/tenant.spec.ts → ...ne/src/engine/__test__/e2e/tenant.spec.ts b/...ne/src/tenant/__test__/e2e/tenant.spec.ts → ...ne/src/engine/__test__/e2e/tenant.spec.ts
@@ -6,33 +6,55 @@ import request from 'supertest'
 import { v4 as uuid } from 'uuid'
 import { EngineService } from '../../../engine/core/service/engine.service'
 import { Config, load } from '../../../policy-engine.config'
-import { REQUEST_HEADER_API_KEY } from '../../../policy-engine.constant'
+import {
+  REQUEST_HEADER_API_KEY,
+  REQUEST_HEADER_CLIENT_ID,
+  REQUEST_HEADER_CLIENT_SECRET
+} from '../../../policy-engine.constant'
 import { KeyValueRepository } from '../../../shared/module/key-value/core/repository/key-value.repository'
 import { InMemoryKeyValueRepository } from '../../../shared/module/key-value/persistence/repository/in-memory-key-value.repository'
 import { TestPrismaService } from '../../../shared/module/persistence/service/test-prisma.service'
 import { getTestRawAesKeyring } from '../../../shared/testing/encryption.testing'
-import { CreateTenantDto } from '../../../tenant/http/rest/dto/create-tenant.dto'
+import { Tenant } from '../../../shared/type/domain.type'
+import { TenantService } from '../../core/service/tenant.service'
+import { EngineModule } from '../../engine.module'
+import { CreateTenantDto } from '../../http/rest/dto/create-tenant.dto'
 import { TenantRepository } from '../../persistence/repository/tenant.repository'
-import { TenantModule } from '../../tenant.module'
 
 describe('Tenant', () => {
   let app: INestApplication
   let module: TestingModule
   let testPrismaService: TestPrismaService
   let tenantRepository: TenantRepository
+  let tenantService: TenantService
   let engineService: EngineService
   let configService: ConfigService<Config, true>
 
   const adminApiKey = 'test-admin-api-key'
 
+  const clientId = uuid()
+
+  const dataStoreUrl = 'http://127.0.0.1:9999/test-data-store'
+
+  const dataStoreConfiguration = {
+    dataUrl: dataStoreUrl,
+    signatureUrl: dataStoreUrl
+  }
+
+  const createTenantPayload: CreateTenantDto = {
+    clientId,
+    entityDataStore: dataStoreConfiguration,
+    policyDataStore: dataStoreConfiguration
+  }
+
   beforeAll(async () => {
     module = await Test.createTestingModule({
       imports: [
         ConfigModule.forRoot({
           load: [load],
           isGlobal: true
         }),
-        TenantModule
+        EngineModule
       ]
     })
       .overrideProvider(KeyValueRepository)
@@ -46,6 +68,7 @@ describe('Tenant', () => {
     app = module.createNestApplication()
 
     engineService = module.get<EngineService>(EngineService)
+    tenantService = module.get<TenantService>(TenantService)
     tenantRepository = module.get<TenantRepository>(TenantRepository)
     testPrismaService = module.get<TestPrismaService>(TestPrismaService)
     configService = module.get<ConfigService<Config, true>>(ConfigService)
@@ -67,25 +90,16 @@ describe('Tenant', () => {
     await app.close()
   })
 
-  describe('POST /tenants', () => {
-    const clientId = uuid()
-
-    const dataStoreConfiguration = {
-      dataUrl: 'http://some.host',
-      signatureUrl: 'http://some.host'
-    }
-
-    const payload: CreateTenantDto = {
-      clientId,
-      entityDataStore: dataStoreConfiguration,
-      policyDataStore: dataStoreConfiguration
-    }
+  beforeEach(() => {
+    jest.spyOn(tenantService, 'syncDataStore').mockResolvedValue(true)
+  })
 
+  describe('POST /tenants', () => {
     it('creates a new tenant', async () => {
       const { status, body } = await request(app.getHttpServer())
         .post('/tenants')
         .set(REQUEST_HEADER_API_KEY, adminApiKey)
-        .send(payload)
+        .send(createTenantPayload)
       const actualTenant = await tenantRepository.findByClientId(clientId)
 
       expect(body).toMatchObject({
@@ -113,12 +127,15 @@ describe('Tenant', () => {
     })
 
     it('responds with an error when clientId already exist', async () => {
-      await request(app.getHttpServer()).post('/tenants').set(REQUEST_HEADER_API_KEY, adminApiKey).send(payload)
+      await request(app.getHttpServer())
+        .post('/tenants')
+        .set(REQUEST_HEADER_API_KEY, adminApiKey)
+        .send(createTenantPayload)
 
       const { status, body } = await request(app.getHttpServer())
         .post('/tenants')
         .set(REQUEST_HEADER_API_KEY, adminApiKey)
-        .send(payload)
+        .send(createTenantPayload)
 
       expect(body).toEqual({
         message: 'Tenant already exist',
@@ -131,7 +148,7 @@ describe('Tenant', () => {
       const { status, body } = await request(app.getHttpServer())
         .post('/tenants')
         .set(REQUEST_HEADER_API_KEY, 'invalid-api-key')
-        .send(payload)
+        .send(createTenantPayload)
 
       expect(body).toMatchObject({
         message: 'Forbidden resource',
@@ -140,4 +157,33 @@ describe('Tenant', () => {
       expect(status).toEqual(HttpStatus.FORBIDDEN)
     })
   })
+
+  describe('POST /tenants/sync', () => {
+    let tenant: Tenant
+
+    beforeEach(async () => {
+      jest.spyOn(tenantService, 'syncDataStore').mockResolvedValue(true)
+
+      const { body } = await request(app.getHttpServer())
+        .post('/tenants')
+        .set(REQUEST_HEADER_API_KEY, adminApiKey)
+        .send({
+          ...createTenantPayload,
+          clientId: uuid()
+        })
+
+      tenant = body
+    })
+
+    it('calls the tenant data store sync', async () => {
+      const { status, body } = await request(app.getHttpServer())
+        .post('/tenants/sync')
+        .set(REQUEST_HEADER_CLIENT_ID, tenant.clientId)
+        .set(REQUEST_HEADER_CLIENT_SECRET, tenant.clientSecret)
+        .send(createTenantPayload)
+
+      expect(body).toEqual({ ok: true })
+      expect(status).toEqual(HttpStatus.OK)
+    })
+  })
 })
Original file line number	Diff line number	Diff line change
Expand Up		@@ -10,4 +10,6 @@ MASTER_PASSWORD="unsafe-local-test-master-password"

		KEYRING_TYPE="raw"

		RESOURCE_PATH=./apps/policy-engine/src/resource

		# MASTER_AWS_KMS_ARN="arn:aws:kms:us-east-2:728783560968:key/f6aa3ddb-47c3-4f31-977d-b93205bb23d1"