feat: Add organisation level import [GROOT-1495] (#2824)

* feat: Add organisation level import [GROOT-1495] * clean up, extract taxonomy import * add broader, related, concept schemes * docs: tidy and improve accuracy of testing notes * docs: clarify * refactor: fix test execution inconsistency * refactor: clarify testing docs * chore: [GROOT-1495] update management sdk version * chore: [GROOT-1495] update typings to match new management sdk version * test: outlined scenarios * test: help message * test: help msg when incorrect args * chore: [GROOT-1495] taxonomy import unit test * test: wip - creating concepts * chore: [GROOT-1495] fix org import unit tests * fix: fix integration tests * test: reduce number of expected concept patch requests * feat: add update integration test * refactor: remove comments * chore: remove unintended change to 'contentful' command --------- Co-authored-by: Adrian L Thomas <[email protected]> Co-authored-by: LiamStokingerContentful <[email protected]>
contentful · Nov 22, 2024 · a829969 · a829969
1 parent 7c3335e
commit a829969
Show file tree

Hide file tree

Showing 20 changed files with 20,106 additions and 11,220 deletions.
diff --git a/.gitignore b/.gitignore
@@ -194,3 +194,6 @@ contentful-import-error-log-*.json
 reports
 
 data/*
+
+# integration test output
+bin/output.json
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -34,16 +34,65 @@ This project uses the [Angular JS Commit Message Conventions](https://docs.googl
 
 # Running tests
 
+## Unit tests
+
+*Note: at the time of writing, the unit tests depend on the environment variables the integration tests use (see next heading below). Therefore please ensure they're set before executing the tests.*
+
+Simply run:
+
+```sh
+# runs Node.js unit tests without coverage.
+npm run test:unit
+
+# Run all unit tests
+npm run test:unit:watch
+
+# Or run specific tests
+npx jest test/unit/cmds/* --watch
+```
+
+See [jest](https://jestjs.io/) documentation for more details about running tests and optional flags.
+
+
+## Integration tests
+
+To run integration tests locally, [talkback](https://github.com/ijpiantanida/talkback) is used as a proxy to record and playback http requests
+
+1. Prepare build in prep for integration tests
+```sh
+npm run build:standalone
+```
+
+1. In another terminal shell run your preferred tests
+```sh
+# Ensure environment variables are set to for the Ecosystem Integration Test Org (`Contentful - Ecosystem (for integration test org)` in password vault)
+export CONTENTFUL_INTEGRATION_TEST_CMA_TOKEN='<cma_auth_token>'
+export CLI_E2E_ORG_ID='<organization_id>'
+
+# Run all integration tests
+npm run test:integration
+
+# Or run specific tests
+npm run talkback-proxy
+npx jest test/integration/cmds/space/* --watch
+```
+
+## Updating Snapshots
+
+You might need to update snapshots and it's challenging with the recordings.
+
+Tip: run tests without recordings to update the snapshots.
+
+```
+npx jest test/integration/cmds/<path to the affected test file> --updateSnapshot
+```
+
 This project has unit and integration tests. Both of these run on both Node.js and Browser environments.
 
 Both of these test environments are setup to deal with Babel and code transpiling, so there's no need to worry about that
 
-- `npm test` runs all three kinds of tests and generates a coverage report
-- `npm run test:unit` runs Node.js unit tests without coverage.
-
 # Other tasks
 
 - `npm run clean` removes any built files
-- `npm run build` builds node package
 - `npm run build:standalone` build standalone binary version
 - `npm run dev` live rebuild - very useful when working on a feature
diff --git a/README.md b/README.md
@@ -104,51 +104,7 @@ contentful config add --raw-proxy
    npm link
    ```
 
-## :robot: Testing
 
-### Integration tests
-
-To run integration tests locally, you'll need the following:
-
-1. Set environment variables in `.jest/env.js` (can be found in 1Password)
-   ```js
-   process.env.CONTENTFUL_INTEGRATION_TEST_CMA_TOKEN = '<cma_auth_token>'
-   process.env.CLI_E2E_ORG_ID = '<organization_id>'
-   ```
-2. Run [talkback](https://github.com/ijpiantanida/talkback) proxy to record and playback http requests
-   ```sh
-   npm run talkback-proxy
-   ```
-3. In another terminal shell run your preferred tests
-   ```sh
-   ## Run all integration tests
-   npm run test:jest
-   ## Or run specific tests
-   npx jest test/integration/cmds/space/* --watch
-   ```
-
-### Unit tests
-
-Simply run:
-
-```sh
-# Run all unit tests
-npm run test:unit:watch
-# Or run specific tests
-npx jest test/unit/cmds/* --watch
-```
-
-See [jest](https://jestjs.io/) documentation for more details about running tests and optional flags.
-
-### Updating Snapshots
-
-You might need to update snapshots and it's challenging with the recordings.
-
-Tip: run tests without recordings to update the snapshots.
-
-```
-npx jest test/integration/cmds/<path to the affected test file> --updateSnapshot
-```
 
 ## :question: Support
 

diff --git a/lib/cmds/organization_cmds/import.ts b/lib/cmds/organization_cmds/import.ts
@@ -0,0 +1,167 @@
+/* eslint-disable @typescript-eslint/no-empty-function */
+import Listr from 'listr'
+import { noop } from 'lodash'
+import type { Argv } from 'yargs'
+import { handleAsyncError as handle } from '../../utils/async'
+import { createPlainClient } from '../../utils/contentful-clients'
+import { getHeadersFromOption } from '../../utils/headers'
+import { TaxonomyJson } from './taxonomy/taxonomy'
+import PQueue from 'p-queue'
+import {
+  displayErrorLog,
+  setupLogging,
+  writeErrorLogFile
+} from 'contentful-batch-libs/dist/logging'
+import taxonomyImport from './taxonomy/taxonomy-import'
+import { PlainClientAPI } from 'contentful-management'
+import { readContentFile } from './utils/read-content-file'
+
+module.exports.command = 'import'
+
+module.exports.desc = 'import organization level entities'
+
+module.exports.builder = (yargs: Argv) => {
+  return yargs
+    .usage('Usage: contentful organization import')
+    .option('management-token', {
+      alias: 'mt',
+      describe: 'Contentful management API token',
+      type: 'string'
+    })
+    .option('organization-id', {
+      alias: 'oid',
+      describe: 'ID of Organization with source data',
+      type: 'string',
+      demandOption: true
+    })
+    .option('header', {
+      alias: 'H',
+      type: 'string',
+      describe: 'Pass an additional HTTP Header'
+    })
+    .option('content-file', {
+      alias: 'f',
+      type: 'string',
+      describe: 'Content file with entities that need to be imported',
+      demandOption: true
+    })
+    .option('silent', {
+      alias: 's',
+      type: 'boolean',
+      describe: 'Suppress any log output',
+      default: false
+    })
+    .option('error-log-file', {
+      describe: 'Full path to the error log file',
+      type: 'string'
+    })
+}
+
+export interface OrgImportParams {
+  context: { managementToken: string }
+  header?: string
+  organizationId: string
+  contentFile: string
+  silent?: boolean
+  errorLogFile?: string
+}
+
+export interface OrgImportContext {
+  data: {
+    taxonomy?: TaxonomyJson['taxonomy']
+  }
+  requestQueue: PQueue
+  cmaClient: PlainClientAPI
+}
+
+const ONE_SECOND = 1000
+
+interface ErrorMessage {
+  ts: string
+  level: 'error'
+  error: Error
+}
+
+async function importCommand(params: OrgImportParams) {
+  const { context, header, organizationId, contentFile, silent, errorLogFile } =
+    params
+  const { managementToken } = context
+
+  const cmaClient = await createPlainClient({
+    accessToken: managementToken,
+    feature: 'org-import',
+    headers: getHeadersFromOption(header),
+    throttle: 8,
+    logHandler: noop
+  })
+
+  const importContext: OrgImportContext = {
+    data: {},
+    requestQueue: new PQueue({
+      concurrency: 1,
+      interval: ONE_SECOND,
+      intervalCap: 1,
+      carryoverConcurrencyCount: true
+    }),
+    cmaClient
+  }
+
+  const log: ErrorMessage[] = []
+  setupLogging(log)
+
+  const tasks = new Listr(
+    [
+      {
+        title: 'Import organization level entities',
+        task: async () => {
+          return new Listr([
+            {
+              title: 'Read content file',
+              task: async ctx => {
+                const data = await readContentFile(contentFile)
+
+                if (data.taxonomy) {
+                  ctx.data.taxonomy = data.taxonomy
+                }
+              }
+            },
+            {
+              title: 'Import taxonomy',
+              task: async ctx => {
+                return taxonomyImport(params, ctx)
+              }
+            }
+          ])
+        }
+      }
+    ],
+    { renderer: silent ? 'silent' : 'default' }
+  )
+
+  await tasks
+    .run(importContext)
+    .catch(err => {
+      importContext.requestQueue.clear()
+
+      log.push({
+        ts: new Date().toISOString(),
+        level: 'error',
+        error: err as Error
+      })
+    })
+    .then(() => {
+      if (log.length > 0) {
+        displayErrorLog(log)
+        const errorLogFilePath =
+          errorLogFile ||
+          `${Date.now()}-${organizationId}-import-org-error-log.json`
+        writeErrorLogFile(errorLogFilePath, log)
+      }
+    })
+}
+
+module.exports.organizationImport = importCommand
+
+module.exports.handler = handle(importCommand)
+
+export default importCommand
diff --git a/lib/cmds/organization_cmds/taxonomy-transform.ts b/lib/cmds/organization_cmds/taxonomy-transform.ts
@@ -10,7 +10,7 @@ import { ensureDir, getPath, readFileP, writeFileP } from '../../utils/fs'
 import { getHeadersFromOption } from '../../utils/headers'
 import { success, log } from '../../utils/log'
 import * as Papa from 'papaparse'
-import { Taxonomy } from './utils/taxonomy'
+import { Taxonomy } from './taxonomy/taxonomy'
 
 module.exports.command = 'taxonomy-transform'
 

diff --git a/...organization_cmds/utils/concept-scheme.ts → ...anization_cmds/taxonomy/concept-scheme.ts b/...organization_cmds/utils/concept-scheme.ts → ...anization_cmds/taxonomy/concept-scheme.ts
@@ -1,16 +1,20 @@
 import { CreateConceptSchemeProps } from 'contentful-management'
 import { defaultLocale } from '../taxonomy-transform'
 
+export type CreateConceptSchemeWithIdProps = CreateConceptSchemeProps & {
+  sys: { id: string }
+}
+
 export class ConceptScheme {
-  private model: CreateConceptSchemeProps & { id: string }
+  private model: CreateConceptSchemeProps & { sys: { id: string } }
 
   public constructor(
     id: string,
     init: Partial<Omit<CreateConceptSchemeProps, 'id'>> & {
       prefLabel: CreateConceptSchemeProps['prefLabel']
     }
   ) {
-    this.model = { id, ...init }
+    this.model = { sys: { id }, ...init }
   }
 
   toJson() {

diff --git a/lib/cmds/organization_cmds/utils/concept.ts → ...mds/organization_cmds/taxonomy/concept.ts b/lib/cmds/organization_cmds/utils/concept.ts → ...mds/organization_cmds/taxonomy/concept.ts
@@ -1,16 +1,20 @@
 import { CreateConceptProps } from 'contentful-management'
 import { defaultLocale } from '../taxonomy-transform'
 
+export type CreateConceptWithIdProps = CreateConceptProps & {
+  sys: { id: string }
+}
+
 export class Concept {
-  private model: CreateConceptProps & { id: string }
+  private model: CreateConceptWithIdProps
 
   public constructor(
     id: string,
     init: Partial<Omit<CreateConceptProps, 'id'>> & {
       prefLabel: CreateConceptProps['prefLabel']
     }
   ) {
-    this.model = { id, ...init }
+    this.model = { sys: { id }, ...init }
   }
 
   toJson() {
-Original file line number
+Diff line change
@@ Expand Up / @@ -194,3 +194,6 @@ contentful-import-error-log-*.json @@
     reports
     data/*
+    # integration test output
+    bin/output.json