Delete sync operation rebased (#117)

* removed operations, scope sync can partially succeed

* refactor bitgo and fireblocks

* tests fixed for new interface

* removed unused services imports

* adding logs to the wallet bulkUpsert method

* networkMap is a map, addressed namings, used constants instead of numbers

* namings, complete function handles partial success for scoped sync service

* lock sync when there's already an in progresss sync for this connection

* Trying larger runner

* fixed the network caching, and the lock on scope sync

* Revert "Trying larger runner"

This reverts commit bda1bb6b3154ae3f71740bbbcf7d0aca263649dc.

* raw account errors are more precise

* fireblocks raw accounts building use external network id

* fixed reads for upserts outside of tx scope

* exists take the status as a parameter

* removed unused function

* fixed fireblocks scoped sync mock server declaration

---------

Co-authored-by: Matt Schoch <[email protected]>

apps/vault/src/broker/core/exception/scoped-sync.exception.ts

@@ -0,0 +1,13 @@
+import { HttpStatus } from '@nestjs/common'
+import { ApplicationExceptionParams } from '../../../shared/exception/application.exception'
+import { BrokerException } from './broker.exception'
+
+export class ScopedSyncException extends BrokerException {
+  constructor(params?: Partial<ApplicationExceptionParams>) {
+    super({
+      message: params?.message || 'Fail to sync provider connection',
+      suggestedHttpStatusCode: params?.suggestedHttpStatusCode || HttpStatus.UNPROCESSABLE_ENTITY,
+      ...params
+    })
+  }
+}

apps/vault/src/broker/core/provider/anchorage/__test__/integration/anchorage-scoped-sync.service.spec.ts

@@ -10,12 +10,11 @@ import { TestPrismaService } from '../../../../../../shared/module/persistence/s
 import { testClient } from '../../../../../__test__/util/mock-data'
 import { NetworkSeed } from '../../../../../persistence/seed/network.seed'
 import { setupMockServer } from '../../../../../shared/__test__/mock-server'
-import { AccountService } from '../../../../service/account.service'
-import { AddressService } from '../../../../service/address.service'
 import { ConnectionService } from '../../../../service/connection.service'
-import { WalletService } from '../../../../service/wallet.service'
+import { NetworkService } from '../../../../service/network.service'
 import { ConnectionWithCredentials } from '../../../../type/connection.type'
-import { Provider, isCreateOperation } from '../../../../type/provider.type'
+import { Provider } from '../../../../type/provider.type'
+import { RawAccountError } from '../../../../type/scoped-sync.type'
 import { AnchorageScopedSyncService } from '../../anchorage-scoped-sync.service'
 import { ANCHORAGE_TEST_API_BASE_URL, getHandlers } from '../server-mock/server'
 
@@ -26,9 +25,7 @@ describe(AnchorageScopedSyncService.name, () => {
   let anchorageScopedSyncService: AnchorageScopedSyncService
   let clientService: ClientService
   let connection: ConnectionWithCredentials
-  let walletService: WalletService
-  let accountService: AccountService
-  let addressService: AddressService
+  let networkService: NetworkService
   let connectionService: ConnectionService
   let networkSeed: NetworkSeed
   let provisionService: ProvisionService
@@ -48,9 +45,7 @@ describe(AnchorageScopedSyncService.name, () => {
     testPrismaService = module.get(TestPrismaService)
     anchorageScopedSyncService = module.get(AnchorageScopedSyncService)
     connectionService = module.get(ConnectionService)
-    walletService = module.get(WalletService)
-    accountService = module.get(AccountService)
-    addressService = module.get(AddressService)
+    networkService = module.get(NetworkService)
     provisionService = module.get(ProvisionService)
     networkSeed = module.get(NetworkSeed)
     clientService = module.get(ClientService)
@@ -94,7 +89,14 @@ describe(AnchorageScopedSyncService.name, () => {
           externalId: '6a46a1977959e0529f567e8e927e3895'
         }
       ]
-      const sync = await anchorageScopedSyncService.scopedSync(connection, rawAccounts)
+      const networks = await networkService.buildProviderExternalIdIndex(Provider.ANCHORAGE)
+
+      const sync = await anchorageScopedSyncService.scopeSync({
+        connection,
+        rawAccounts,
+        networks,
+        existingAccounts: []
+      })
 
       expect(sync.wallets.length).toBe(1)
       expect(sync.accounts.length).toBe(1)
@@ -103,75 +105,76 @@ describe(AnchorageScopedSyncService.name, () => {
 
     // TODO @ptroger: revert that back to 'return empty array' when we completely move towards the scoped connections
     it('returns full connection sync when empty rawAccounts are provided', async () => {
-      const sync = await anchorageScopedSyncService.scopedSync(connection, [])
+      const networks = await networkService.buildProviderExternalIdIndex(Provider.ANCHORAGE)
+
+      const sync = await anchorageScopedSyncService.scopeSync({
+        connection,
+        rawAccounts: [],
+        networks,
+        existingAccounts: []
+      })
 
       expect(sync.wallets.length).toBe(2)
       expect(sync.accounts.length).toBe(18)
-      expect(sync.addresses.length).toBe(2)
+      expect(sync.addresses.length).toBe(18)
     })
 
-    it('returns empty array when no wallets are found for the provided rawAccounts', async () => {
+    it('adds failure when external resource is not found', async () => {
       const rawAccounts = [
         {
           provider: Provider.ANCHORAGE,
           externalId: 'notFound'
         }
       ]
-      const sync = await anchorageScopedSyncService.scopedSync(connection, rawAccounts)
+      const networks = await networkService.buildProviderExternalIdIndex(Provider.ANCHORAGE)
+
+      const sync = await anchorageScopedSyncService.scopeSync({
+        connection,
+        rawAccounts,
+        networks,
+        existingAccounts: []
+      })
 
       expect(sync.wallets.length).toBe(0)
       expect(sync.accounts.length).toBe(0)
       expect(sync.addresses.length).toBe(0)
-    })
-    it('skips duplicate for the same connection', async () => {
-      const rawAccounts = [
+      expect(sync.failures).toEqual([
         {
-          provider: Provider.ANCHORAGE,
-          externalId: '6a46a1977959e0529f567e8e927e3895'
+          rawAccount: rawAccounts[0],
+          message: 'Anchorage wallet not found',
+          code: RawAccountError.EXTERNAL_RESOURCE_NOT_FOUND,
+          externalResourceType: 'wallet',
+          externalResourceId: rawAccounts[0].externalId
         }
-      ]
-      const firstSync = await anchorageScopedSyncService.scopedSync(connection, rawAccounts)
-
-      await walletService.bulkCreate(firstSync.wallets.filter(isCreateOperation).map(({ create }) => create))
-      await accountService.bulkCreate(firstSync.accounts.filter(isCreateOperation).map(({ create }) => create))
-      await addressService.bulkCreate(firstSync.addresses.filter(isCreateOperation).map(({ create }) => create))
-
-      const secondSync = await anchorageScopedSyncService.scopedSync(connection, rawAccounts)
-
-      expect(secondSync.wallets.length).toBe(0)
-      expect(secondSync.accounts.length).toBe(0)
-      expect(secondSync.addresses.length).toBe(0)
+      ])
     })
 
-    it('duplicates for different connections', async () => {
+    it('adds failure when network is not found in our list', async () => {
       const rawAccounts = [
         {
           provider: Provider.ANCHORAGE,
           externalId: '6a46a1977959e0529f567e8e927e3895'
         }
       ]
-      const firstSync = await anchorageScopedSyncService.scopedSync(connection, rawAccounts)
-
-      await walletService.bulkCreate(firstSync.wallets.filter(isCreateOperation).map(({ create }) => create))
-      await accountService.bulkCreate(firstSync.accounts.filter(isCreateOperation).map(({ create }) => create))
-      await addressService.bulkCreate(firstSync.addresses.filter(isCreateOperation).map(({ create }) => create))
-
-      const secondConnection = await connectionService.create(clientId, {
-        connectionId: uuid(),
-        provider: Provider.ANCHORAGE,
-        url: ANCHORAGE_TEST_API_BASE_URL,
-        label: 'test active connection',
-        credentials: {
-          apiKey: 'test-api-key',
-          privateKey: await privateKeyToHex(await generateJwk(Alg.EDDSA))
-        }
-      })
 
-      const secondSync = await anchorageScopedSyncService.scopedSync(secondConnection, rawAccounts)
+      const sync = await anchorageScopedSyncService.scopeSync({
+        connection,
+        rawAccounts,
+        networks: new Map(),
+        existingAccounts: []
+      })
 
-      expect(secondSync.wallets.length).toBe(1)
-      expect(secondSync.accounts.length).toBe(1)
-      expect(secondSync.addresses.length).toBe(1)
+      expect(sync.wallets.length).toBe(0)
+      expect(sync.accounts.length).toBe(0)
+      expect(sync.addresses.length).toBe(0)
+      expect(sync.failures).toEqual([
+        {
+          rawAccount: rawAccounts[0],
+          message: 'Network for this account is not supported',
+          code: RawAccountError.UNLISTED_NETWORK,
+          networkId: 'BTC_S'
+        }
+      ])
     })
   })
 })