From 7887ff43c870a0c8d08d63bb0abce017ab7aaa78 Mon Sep 17 00:00:00 2001 From: Anna Khismatullina Date: Mon, 18 Nov 2024 23:21:49 +0700 Subject: [PATCH] Integrate import to workspace initialization Signed-off-by: Anna Khismatullina --- .vscode/launch.json | 2 +- common/config/rush/pnpm-lock.yaml | 47 ++++++- dev/.env | 3 +- dev/docker-compose.yaml | 9 +- dev/import-tool/README.md | 8 +- .../{src => docs}/clickup/README.md | 0 dev/import-tool/{src => docs}/huly/README.md | 0 .../huly/example-workspace/Documentation.yaml | 1 + .../Documentation/Getting Started.md | 0 .../Documentation/User Guide.md | 0 .../Documentation/User Guide/Installation.md | 0 .../Documentation/files/architecture.png | Bin .../huly/example-workspace/Project Alpha.yaml | 1 + .../Project Alpha/1.Project Setup.md | 0 .../1.Project Setup/2.Configure CI.md | 0 .../Project Alpha/4.Update Docs.md | 0 .../Project Alpha/files/config.yaml | 0 .../{src => docs}/notion/README.md | 0 dev/import-tool/package.json | 31 +---- dev/import-tool/src/index.ts | 30 ++++- dev/tool/src/index.ts | 7 +- packages/importer/.eslintrc.js | 7 + packages/importer/.npmignore | 4 + packages/importer/jest.config.js | 7 + packages/importer/package.json | 68 ++++++++++ .../importer}/src/clickup/clickup.ts | 23 ++-- .../importer}/src/huly/unified.ts | 60 ++++++--- .../importer}/src/importer/builder.ts | 33 +++++ .../importer}/src/importer/dowloader.ts | 0 .../importer/src/importer/frontUploader.ts | 54 +++++--- .../importer}/src/importer/importer.ts | 127 +++++++++++++----- packages/importer/src/importer/logger.ts | 18 +++ .../importer}/src/importer/preprocessor.ts | 0 .../importer/src/importer/storageUploader.ts | 59 ++++++++ packages/importer/src/importer/uploader.ts | 33 +++++ packages/importer/src/index.ts | 23 ++++ .../importer}/src/notion/notion.ts | 8 +- packages/importer/tsconfig.json | 10 ++ rush.json | 5 + server/account-service/package.json | 1 - server/account-service/src/index.ts | 6 - server/tool/package.json | 1 + server/tool/src/index.ts | 20 ++- server/tool/src/initializer.ts | 26 +++- server/tool/src/plugin.ts | 2 +- server/workspace-service/src/index.ts | 9 +- 46 files changed, 595 insertions(+), 148 deletions(-) rename dev/import-tool/{src => docs}/clickup/README.md (100%) rename dev/import-tool/{src => docs}/huly/README.md (100%) rename dev/import-tool/{src => docs}/huly/example-workspace/Documentation.yaml (94%) rename dev/import-tool/{src => docs}/huly/example-workspace/Documentation/Getting Started.md (100%) rename dev/import-tool/{src => docs}/huly/example-workspace/Documentation/User Guide.md (100%) rename dev/import-tool/{src => docs}/huly/example-workspace/Documentation/User Guide/Installation.md (100%) rename dev/import-tool/{src => docs}/huly/example-workspace/Documentation/files/architecture.png (100%) rename dev/import-tool/{src => docs}/huly/example-workspace/Project Alpha.yaml (95%) rename dev/import-tool/{src => docs}/huly/example-workspace/Project Alpha/1.Project Setup.md (100%) rename dev/import-tool/{src => docs}/huly/example-workspace/Project Alpha/1.Project Setup/2.Configure CI.md (100%) rename dev/import-tool/{src => docs}/huly/example-workspace/Project Alpha/4.Update Docs.md (100%) rename dev/import-tool/{src => docs}/huly/example-workspace/Project Alpha/files/config.yaml (100%) rename dev/import-tool/{src => docs}/notion/README.md (100%) create mode 100644 packages/importer/.eslintrc.js create mode 100644 packages/importer/.npmignore create mode 100644 packages/importer/jest.config.js create mode 100644 packages/importer/package.json rename {dev/import-tool => packages/importer}/src/clickup/clickup.ts (93%) rename {dev/import-tool => packages/importer}/src/huly/unified.ts (91%) rename {dev/import-tool => packages/importer}/src/importer/builder.ts (94%) rename {dev/import-tool => packages/importer}/src/importer/dowloader.ts (100%) rename dev/import-tool/src/importer/uploader.ts => packages/importer/src/importer/frontUploader.ts (53%) rename {dev/import-tool => packages/importer}/src/importer/importer.ts (80%) create mode 100644 packages/importer/src/importer/logger.ts rename {dev/import-tool => packages/importer}/src/importer/preprocessor.ts (100%) create mode 100644 packages/importer/src/importer/storageUploader.ts create mode 100644 packages/importer/src/importer/uploader.ts create mode 100644 packages/importer/src/index.ts rename {dev/import-tool => packages/importer}/src/notion/notion.ts (98%) create mode 100644 packages/importer/tsconfig.json diff --git a/.vscode/launch.json b/.vscode/launch.json index cb08a80418..04471cd613 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -626,7 +626,7 @@ "name": "Debug Huly import", "type": "node", "request": "launch", - "args": ["src/__start.ts", "import", "/home/anna/xored/huly/platform/dev/import-tool/src/huly/example-workspace", "-u", "user1", "-pw", "1234", "-ws", "ws12"], + "args": ["src/__start.ts", "import", "/home/anna/xored/huly/platform/dev/import-tool/docs/huly/example-workspace", "-u", "user1", "-pw", "1234", "-ws", "ws12"], "env": { "FRONT_URL": "http://localhost:8087" }, diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 13b158e773..7fc05bb533 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -296,6 +296,9 @@ dependencies: '@rush-temp/import-tool': specifier: file:./projects/import-tool.tgz version: file:projects/import-tool.tgz + '@rush-temp/importer': + specifier: file:./projects/importer.tgz + version: file:projects/importer.tgz '@rush-temp/inventory': specifier: file:./projects/inventory.tgz version: file:projects/inventory.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2) @@ -23870,7 +23873,7 @@ packages: dev: false file:projects/import-tool.tgz: - resolution: {integrity: sha512-0Q1/hHZxEdYFPr2qqfovlVJRA8JyvWsKtY3ubHrffnaAMbbN6BNWt6Jf+GzwyGeIwImLI2Oud2x/WqOFb/USdg==, tarball: file:projects/import-tool.tgz} + resolution: {integrity: sha512-+aINV0yg8OL4e+HL4NEtM879eZSQEDyUwuIKcRl2jtzlaGqoUnAoBx56ds3ynehouocaZxhUz+FuRsNA8j6YdQ==, tarball: file:projects/import-tool.tgz} name: '@rush-temp/import-tool' version: 0.0.0 dependencies: @@ -23917,6 +23920,46 @@ packages: - supports-color dev: false + file:projects/importer.tgz: + resolution: {integrity: sha512-NPWdDMTZZVuM4tlCiUWR3m8XTpr0zD9iNzJBgZZ76JoXwu8Tvimw0CVVCQl2twn7UayHyBLgiJ1Gy5nZHbk2+w==, tarball: file:projects/importer.tgz} + name: '@rush-temp/importer' + version: 0.0.0 + dependencies: + '@types/csvtojson': 2.0.0 + '@types/jest': 29.5.12 + '@types/js-yaml': 4.0.9 + '@types/mime-types': 2.1.4 + '@types/node': 20.11.19 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) + commander: 8.3.0 + cross-env: 7.0.3 + csvtojson: 2.0.10 + esbuild: 0.20.1 + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) + js-yaml: 4.1.0 + mime-types: 2.1.35 + prettier: 3.2.5 + ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.6.2) + ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.6.2) + typescript: 5.6.2 + yjs: 13.6.19 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - '@swc/core' + - '@swc/wasm' + - babel-jest + - babel-plugin-macros + - node-notifier + - supports-color + dev: false + file:projects/inventory-assets.tgz(esbuild@0.20.1)(ts-node@10.9.2): resolution: {integrity: sha512-Uayr70kuiNfwBgNoFcu1rkWhdHhnbg7aEDcbozbhn5Eyel/B6he2uUYxPZc2gl1VUiEA8KBGszSAEXMs0YER0A==, tarball: file:projects/inventory-assets.tgz} id: file:projects/inventory-assets.tgz @@ -30317,7 +30360,7 @@ packages: dev: false file:projects/server-tool.tgz(@types/node@20.11.19)(esbuild@0.20.1)(ts-node@10.9.2): - resolution: {integrity: sha512-Pd5QWUOAPkgU3vgG4J4yXrQJuskV46yXxzATYcWPzY+4LQRJ4jshHlld/rI2EGJKJNr5fy6trH1p8LJ+xvLdeA==, tarball: file:projects/server-tool.tgz} + resolution: {integrity: sha512-KrWvamRcvwO2YhpEiGjKBl2gVkkEkeF40fR5siXFNaDzrjNkdyPdAqSM//pGlaBSlzXA3PBG6Z8G8IfwgVEvXg==, tarball: file:projects/server-tool.tgz} id: file:projects/server-tool.tgz name: '@rush-temp/server-tool' version: 0.0.0 diff --git a/dev/.env b/dev/.env index 6650314f86..389fa9b39e 100644 --- a/dev/.env +++ b/dev/.env @@ -1,3 +1,4 @@ STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin" MONGO_URL=mongodb://mongodb:27017?compressors=snappy -DB_URL_PG=postgresql://postgres:example@postgres:5432 \ No newline at end of file +DB_URL_PG=postgresql://postgres:example@postgres:5432 +WS_INIT_SCRIPTS=../../init diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index d92ff909c7..8fda5ce992 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -125,6 +125,7 @@ services: - stats volumes: - ./branding.json:/var/cfg/branding.json + - ${WS_INIT_SCRIPTS}:/init-scripts environment: # - WS_OPERATION=create - SERVER_SECRET=secret @@ -138,7 +139,7 @@ services: - ACCOUNTS_URL=http://host.docker.internal:3000 - BRANDING_PATH=/var/cfg/branding.json # - PARALLEL=2 - - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml + - INIT_REPO_DIR=/init-scripts - INIT_WORKSPACE=test restart: unless-stopped workspacepg: @@ -151,6 +152,7 @@ services: - stats volumes: - ./branding.json:/var/cfg/branding.json + - ${WS_INIT_SCRIPTS}:/init-scripts environment: # - WS_OPERATION=create - SERVER_SECRET=secret @@ -164,8 +166,8 @@ services: - MODEL_ENABLED=* - ACCOUNTS_URL=http://host.docker.internal:3000 - BRANDING_PATH=/var/cfg/branding.json + - INIT_REPO_DIR=/init-scripts # - PARALLEL=2 - - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml # - INIT_WORKSPACE=onboarding restart: unless-stopped workspace_cockroach: @@ -178,6 +180,7 @@ services: - stats volumes: - ./branding.json:/var/cfg/branding.json + - ${WS_INIT_SCRIPTS}:/init-scripts environment: # - WS_OPERATION=create - SERVER_SECRET=secret @@ -190,8 +193,8 @@ services: - MODEL_ENABLED=* - ACCOUNTS_URL=http://host.docker.internal:3000 - BRANDING_PATH=/var/cfg/branding.json + - INIT_REPO_DIR=/init-scripts # - PARALLEL=2 - - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml # - INIT_WORKSPACE=onboarding restart: unless-stopped collaborator: diff --git a/dev/import-tool/README.md b/dev/import-tool/README.md index 4e1c0d56b8..59de5713b5 100644 --- a/dev/import-tool/README.md +++ b/dev/import-tool/README.md @@ -5,9 +5,9 @@ Tool for importing data into Huly workspace. ## Recommended Import Method ### Unified Format Import -The recommended way to import data into Huly is using our [Unified Import Format](./src/huly/README.md). This format provides a straightforward way to migrate data from any system by converting it into an intermediate, human-readable structure. +The recommended way to import data into Huly is using our [Unified Import Format](./docs/huly/README.md). This format provides a straightforward way to migrate data from any system by converting it into an intermediate, human-readable structure. -See the [complete guide](./src/huly/README.md) and [example workspace](./src/huly/example-workspace) to get started. +See the [complete guide](./docs/huly/README.md) and [example workspace](./docs/huly/example-workspace) to get started. ### Why Use Unified Format? - Simple, human-readable format using YAML and Markdown @@ -19,7 +19,7 @@ See the [complete guide](./src/huly/README.md) and [example workspace](./src/hul We also support direct import from some platforms: -1. **Notion**: see [Import from Notion Guide](./src/notion/README.md) -2. **ClickUp**: see [Import from ClickUp Guide](./src/clickup/README.md) +1. **Notion**: see [Import from Notion Guide](./docs/notion/README.md) +2. **ClickUp**: see [Import from ClickUp Guide](./docs/clickup/README.md) These direct imports are suitable for simple migrations, but for complex cases or systems not listed above, please use the Unified Format. \ No newline at end of file diff --git a/dev/import-tool/src/clickup/README.md b/dev/import-tool/docs/clickup/README.md similarity index 100% rename from dev/import-tool/src/clickup/README.md rename to dev/import-tool/docs/clickup/README.md diff --git a/dev/import-tool/src/huly/README.md b/dev/import-tool/docs/huly/README.md similarity index 100% rename from dev/import-tool/src/huly/README.md rename to dev/import-tool/docs/huly/README.md diff --git a/dev/import-tool/src/huly/example-workspace/Documentation.yaml b/dev/import-tool/docs/huly/example-workspace/Documentation.yaml similarity index 94% rename from dev/import-tool/src/huly/example-workspace/Documentation.yaml rename to dev/import-tool/docs/huly/example-workspace/Documentation.yaml index 3279e182fb..781c2b6f12 100644 --- a/dev/import-tool/src/huly/example-workspace/Documentation.yaml +++ b/dev/import-tool/docs/huly/example-workspace/Documentation.yaml @@ -1,5 +1,6 @@ class: document:class:Teamspace title: Documentation +emoji: 📖 private: false autoJoin: true owners: diff --git a/dev/import-tool/src/huly/example-workspace/Documentation/Getting Started.md b/dev/import-tool/docs/huly/example-workspace/Documentation/Getting Started.md similarity index 100% rename from dev/import-tool/src/huly/example-workspace/Documentation/Getting Started.md rename to dev/import-tool/docs/huly/example-workspace/Documentation/Getting Started.md diff --git a/dev/import-tool/src/huly/example-workspace/Documentation/User Guide.md b/dev/import-tool/docs/huly/example-workspace/Documentation/User Guide.md similarity index 100% rename from dev/import-tool/src/huly/example-workspace/Documentation/User Guide.md rename to dev/import-tool/docs/huly/example-workspace/Documentation/User Guide.md diff --git a/dev/import-tool/src/huly/example-workspace/Documentation/User Guide/Installation.md b/dev/import-tool/docs/huly/example-workspace/Documentation/User Guide/Installation.md similarity index 100% rename from dev/import-tool/src/huly/example-workspace/Documentation/User Guide/Installation.md rename to dev/import-tool/docs/huly/example-workspace/Documentation/User Guide/Installation.md diff --git a/dev/import-tool/src/huly/example-workspace/Documentation/files/architecture.png b/dev/import-tool/docs/huly/example-workspace/Documentation/files/architecture.png similarity index 100% rename from dev/import-tool/src/huly/example-workspace/Documentation/files/architecture.png rename to dev/import-tool/docs/huly/example-workspace/Documentation/files/architecture.png diff --git a/dev/import-tool/src/huly/example-workspace/Project Alpha.yaml b/dev/import-tool/docs/huly/example-workspace/Project Alpha.yaml similarity index 95% rename from dev/import-tool/src/huly/example-workspace/Project Alpha.yaml rename to dev/import-tool/docs/huly/example-workspace/Project Alpha.yaml index 0e13f3656c..ee5389ad33 100644 --- a/dev/import-tool/src/huly/example-workspace/Project Alpha.yaml +++ b/dev/import-tool/docs/huly/example-workspace/Project Alpha.yaml @@ -1,6 +1,7 @@ class: tracker:class:Project title: Project Alpha identifier: ALPHA +emoji: 🦄 private: false autoJoin: true owners: diff --git a/dev/import-tool/src/huly/example-workspace/Project Alpha/1.Project Setup.md b/dev/import-tool/docs/huly/example-workspace/Project Alpha/1.Project Setup.md similarity index 100% rename from dev/import-tool/src/huly/example-workspace/Project Alpha/1.Project Setup.md rename to dev/import-tool/docs/huly/example-workspace/Project Alpha/1.Project Setup.md diff --git a/dev/import-tool/src/huly/example-workspace/Project Alpha/1.Project Setup/2.Configure CI.md b/dev/import-tool/docs/huly/example-workspace/Project Alpha/1.Project Setup/2.Configure CI.md similarity index 100% rename from dev/import-tool/src/huly/example-workspace/Project Alpha/1.Project Setup/2.Configure CI.md rename to dev/import-tool/docs/huly/example-workspace/Project Alpha/1.Project Setup/2.Configure CI.md diff --git a/dev/import-tool/src/huly/example-workspace/Project Alpha/4.Update Docs.md b/dev/import-tool/docs/huly/example-workspace/Project Alpha/4.Update Docs.md similarity index 100% rename from dev/import-tool/src/huly/example-workspace/Project Alpha/4.Update Docs.md rename to dev/import-tool/docs/huly/example-workspace/Project Alpha/4.Update Docs.md diff --git a/dev/import-tool/src/huly/example-workspace/Project Alpha/files/config.yaml b/dev/import-tool/docs/huly/example-workspace/Project Alpha/files/config.yaml similarity index 100% rename from dev/import-tool/src/huly/example-workspace/Project Alpha/files/config.yaml rename to dev/import-tool/docs/huly/example-workspace/Project Alpha/files/config.yaml diff --git a/dev/import-tool/src/notion/README.md b/dev/import-tool/docs/notion/README.md similarity index 100% rename from dev/import-tool/src/notion/README.md rename to dev/import-tool/docs/notion/README.md diff --git a/dev/import-tool/package.json b/dev/import-tool/package.json index 336f00f6fd..8acc809953 100644 --- a/dev/import-tool/package.json +++ b/dev/import-tool/package.json @@ -34,43 +34,26 @@ "cross-env": "~7.0.3", "@hcengineering/platform-rig": "^0.6.0", "@typescript-eslint/eslint-plugin": "^6.11.0", - "eslint-plugin-import": "^2.26.0", - "eslint-plugin-promise": "^6.1.1", - "eslint-plugin-n": "^15.4.0", "eslint": "^8.54.0", "ts-node": "^10.8.0", "esbuild": "^0.20.0", - "@types/mime-types": "~2.1.1", "@types/node": "~20.11.16", "@typescript-eslint/parser": "^6.11.0", - "eslint-config-standard-with-typescript": "^40.0.0", - "prettier": "^3.1.0", "typescript": "^5.3.3", "jest": "^29.7.0", "ts-jest": "^29.1.1", "@types/jest": "^29.5.5", - "@types/csvtojson": "^2.0.0", - "@types/js-yaml": "^4.0.9" + "eslint-config-standard-with-typescript": "^40.0.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-n": "^15.4.0", + "eslint-plugin-promise": "^6.1.1", + "prettier": "^3.1.0" }, "dependencies": { - "@hcengineering/attachment": "^0.6.14", - "@hcengineering/collaboration": "^0.6.0", - "@hcengineering/document": "^0.6.0", - "@hcengineering/text": "^0.6.5", - "@hcengineering/model-attachment": "^0.6.0", - "@hcengineering/model-core": "^0.6.0", "@hcengineering/core": "^0.6.32", "@hcengineering/platform": "^0.6.11", - "@hcengineering/server-tool": "^0.6.0", "@hcengineering/server-client": "^0.6.0", - "@hcengineering/rank": "^0.6.4", - "@hcengineering/tracker": "^0.6.24", - "commander": "^8.1.0", - "mime-types": "~2.1.34", - "csvtojson": "^2.0.10", - "@hcengineering/task": "^0.6.20", - "@hcengineering/contact": "^0.6.24", - "@hcengineering/chunter": "^0.6.20", - "js-yaml": "^4.1.0" + "@hcengineering/importer": "^0.6.1", + "commander": "^8.1.0" } } diff --git a/dev/import-tool/src/index.ts b/dev/import-tool/src/index.ts index 3ca141dd71..ff40a87eaa 100644 --- a/dev/import-tool/src/index.ts +++ b/dev/import-tool/src/index.ts @@ -20,11 +20,29 @@ import serverClientPlugin, { selectWorkspace } from '@hcengineering/server-client' import { program } from 'commander' -import { importNotion } from './notion/notion' import { setMetadata } from '@hcengineering/platform' -import { FrontFileUploader, type FileUploader } from './importer/uploader' -import { ClickupImporter } from './clickup/clickup' -import { UnifiedFormatImporter } from './huly/unified' +import { + UnifiedFormatImporter, + ClickupImporter, + importNotion, + FrontFileUploader, + type FileUploader, + type Logger +} from '@hcengineering/importer' + +class ConsoleLogger implements Logger { + log (msg: string, data?: any): void { + console.log(msg, data) + } + + warn (msg: string, data?: any): void { + console.warn(msg, data) + } + + error (msg: string, data?: any): void { + console.error(msg, data) + } +} /** * @public @@ -127,7 +145,7 @@ export function importTool (): void { .action(async (file: string, cmd) => { const { workspace, user, password } = cmd await authorize(user, password, workspace, async (client, uploader) => { - const importer = new ClickupImporter(client, uploader) + const importer = new ClickupImporter(client, uploader, new ConsoleLogger()) await importer.importClickUpTasks(file) }) }) @@ -142,7 +160,7 @@ export function importTool (): void { .action(async (dir: string, cmd) => { const { workspace, user, password } = cmd await authorize(user, password, workspace, async (client, uploader) => { - const importer = new UnifiedFormatImporter(client, uploader) + const importer = new UnifiedFormatImporter(client, uploader, new ConsoleLogger()) await importer.importFolder(dir) }) }) diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 829c9eca48..d195400056 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -60,7 +60,7 @@ import serverClientPlugin, { } from '@hcengineering/server-client' import { createBackupPipeline, getConfig } from '@hcengineering/server-pipeline' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' -import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool' +import { FileModelLogger } from '@hcengineering/server-tool' import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' import path from 'path' @@ -173,11 +173,6 @@ export function devTool ( return elasticUrl } - const initScriptUrl = process.env.INIT_SCRIPT_URL - if (initScriptUrl !== undefined) { - setMetadata(toolPlugin.metadata.InitScriptURL, initScriptUrl) - } - setMetadata(accountPlugin.metadata.Transactors, transactorUrl) setMetadata(serverClientPlugin.metadata.Endpoint, accountsUrl) setMetadata(serverToken.metadata.Secret, serverSecret) diff --git a/packages/importer/.eslintrc.js b/packages/importer/.eslintrc.js new file mode 100644 index 0000000000..72235dc283 --- /dev/null +++ b/packages/importer/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.json' + } +} diff --git a/packages/importer/.npmignore b/packages/importer/.npmignore new file mode 100644 index 0000000000..e3ec093c38 --- /dev/null +++ b/packages/importer/.npmignore @@ -0,0 +1,4 @@ +* +!/lib/** +!CHANGELOG.md +/lib/**/__tests__/ diff --git a/packages/importer/jest.config.js b/packages/importer/jest.config.js new file mode 100644 index 0000000000..2cfd408b67 --- /dev/null +++ b/packages/importer/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'], + roots: ["./src"], + coverageReporters: ["text-summary", "html"] +} diff --git a/packages/importer/package.json b/packages/importer/package.json new file mode 100644 index 0000000000..ee8a5ce371 --- /dev/null +++ b/packages/importer/package.json @@ -0,0 +1,68 @@ +{ + "name": "@hcengineering/importer", + "version": "0.6.1", + "main": "lib/index.js", + "svelte": "src/index.ts", + "types": "types/index.d.ts", + "files": [ + "lib/**/*", + "types/**/*", + "tsconfig.json" + ], + "author": "Anticrm Platform Contributors", + "license": "EPL-2.0", + "scripts": { + "build": "compile", + "build:watch": "compile", + "test": "jest --passWithNoTests --silent", + "format": "format src", + "_phase:build": "compile transpile src", + "_phase:test": "jest --passWithNoTests --silent", + "_phase:format": "format src", + "_phase:validate": "compile validate" + }, + "devDependencies": { + "@hcengineering/platform-rig": "^0.6.0", + "@typescript-eslint/eslint-plugin": "^6.11.0", + "@typescript-eslint/parser": "^6.11.0", + "eslint": "^8.54.0", + "eslint-config-standard-with-typescript": "^40.0.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-n": "^15.4.0", + "eslint-plugin-promise": "^6.1.1", + "prettier": "^3.1.0", + "typescript": "^5.3.3", + "jest": "^29.7.0", + "ts-jest": "^29.1.1", + "@types/jest": "^29.5.5", + "@types/node": "~20.11.16", + "@types/mime-types": "~2.1.1", + "@types/csvtojson": "^2.0.0", + "@types/js-yaml": "^4.0.9" + }, + "dependencies": { + "@hcengineering/attachment": "^0.6.14", + "@hcengineering/chunter": "^0.6.20", + "@hcengineering/collaboration": "^0.6.0", + "@hcengineering/contact": "^0.6.24", + "@hcengineering/core": "^0.6.32", + "@hcengineering/document": "^0.6.0", + "@hcengineering/model-attachment": "^0.6.0", + "@hcengineering/model-core": "^0.6.0", + "@hcengineering/platform": "^0.6.11", + "@hcengineering/rank": "^0.6.4", + "@hcengineering/server-core": "^0.6.1", + "@hcengineering/task": "^0.6.20", + "@hcengineering/text": "^0.6.5", + "@hcengineering/tracker": "^0.6.24", + "@hcengineering/view": "^0.6.13", + "commander": "^8.1.0", + "mime-types": "~2.1.34", + "csvtojson": "^2.0.10", + "js-yaml": "^4.1.0" + }, + "repository": "https://github.com/hcengineering/platform", + "publishConfig": { + "registry": "https://npm.pkg.github.com" + } +} diff --git a/dev/import-tool/src/clickup/clickup.ts b/packages/importer/src/clickup/clickup.ts similarity index 93% rename from dev/import-tool/src/clickup/clickup.ts rename to packages/importer/src/clickup/clickup.ts index b39d00a67b..431c759bfd 100644 --- a/dev/import-tool/src/clickup/clickup.ts +++ b/packages/importer/src/clickup/clickup.ts @@ -25,9 +25,9 @@ import { type ImportProject, type ImportProjectType } from '../importer/importer' -import { type FileUploader } from '../importer/uploader' +import { type Logger } from '../importer/logger' import { BaseMarkdownPreprocessor } from '../importer/preprocessor' - +import { type FileUploader } from '../importer/uploader' interface ClickupTask { 'Task ID': string 'Task Name': string @@ -86,7 +86,8 @@ class ClickupImporter { constructor ( private readonly client: TxOperations, - private readonly fileUploader: FileUploader + private readonly fileUploader: FileUploader, + private readonly logger: Logger ) {} async importClickUpTasks (file: string): Promise { @@ -102,13 +103,13 @@ class ClickupImporter { spaces } - console.log('========================================') - console.log('IMPORT DATA STRUCTURE: ', JSON.stringify(importData, null, 4)) - console.log('========================================') + this.logger.log('========================================') + this.logger.log('IMPORT DATA STRUCTURE: ', JSON.stringify(importData, null, 4)) + this.logger.log('========================================') const postprocessor = new ClickupMarkdownPreprocessor(this.personsByName) - await new WorkspaceImporter(this.client, this.fileUploader, importData, postprocessor).performImport() - console.log('========================================') - console.log('IMPORT SUCCESS ') + await new WorkspaceImporter(this.client, this.logger, this.fileUploader, importData, postprocessor).performImport() + this.logger.log('========================================') + this.logger.log('IMPORT SUCCESS ') } private async processTasksCsv (file: string, process: (json: ClickupTask) => Promise | void): Promise { @@ -136,8 +137,8 @@ class ClickupImporter { statuses.add(clickupTask.Status) }) - console.log(projects) - console.log(statuses) + this.logger.log('Projects: ' + JSON.stringify(projects)) + this.logger.log('Statuses: ' + JSON.stringify(statuses)) const importProjectType = this.createClickupProjectType(Array.from(statuses)) diff --git a/dev/import-tool/src/huly/unified.ts b/packages/importer/src/huly/unified.ts similarity index 91% rename from dev/import-tool/src/huly/unified.ts rename to packages/importer/src/huly/unified.ts index 67713179a4..6ba2e4e031 100644 --- a/dev/import-tool/src/huly/unified.ts +++ b/packages/importer/src/huly/unified.ts @@ -18,7 +18,7 @@ import contact, { type Person, type PersonAccount } from '@hcengineering/contact import { type Class, type Doc, generateId, type Ref, type Space, type TxOperations } from '@hcengineering/core' import document, { type Document } from '@hcengineering/document' import { MarkupMarkType, type MarkupNode, MarkupNodeType, traverseNode, traverseNodeMarks } from '@hcengineering/text' -import tracker, { type Issue } from '@hcengineering/tracker' +import tracker, { type Issue, Project } from '@hcengineering/tracker' import * as fs from 'fs' import * as yaml from 'js-yaml' import { contentType } from 'mime-types' @@ -35,6 +35,7 @@ import { type ImportWorkspace, WorkspaceImporter } from '../importer/importer' +import { type Logger } from '../importer/logger' import { BaseMarkdownPreprocessor } from '../importer/preprocessor' import { type FileUploader } from '../importer/uploader' @@ -59,14 +60,17 @@ interface UnifiedSpaceSettings { title: string private?: boolean autoJoin?: boolean + archived?: boolean owners?: string[] members?: string[] description?: string + emoji?: string } interface UnifiedProjectSettings extends UnifiedSpaceSettings { class: 'tracker:class:Project' identifier: string + id?: 'tracker:project:DefaultProject' projectType?: string defaultIssueStatus?: string } @@ -97,6 +101,7 @@ interface UnifiedWorkspaceSettings { class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor { constructor ( private readonly urlProvider: (id: string) => string, + private readonly logger: Logger, private readonly metadataByFilePath: Map, private readonly metadataById: Map, DocMetadata>, private readonly attachMetadataByPath: Map, @@ -130,7 +135,7 @@ class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor { const attachmentMeta = this.attachMetadataByPath.get(fullPath) if (attachmentMeta === undefined) { - console.warn(`Attachment image not found for ${fullPath}`) + this.logger.error(`Attachment image not found for ${fullPath}`) return } @@ -160,7 +165,7 @@ class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor { this.updateAttachmentMetadata(fullPath, attachmentMeta, id, spaceId, sourceMeta) } } else { - console.log('Unknown link type, leave it as is:', href) + this.logger.log('Unknown link type, leave it as is: ' + href) } }) } @@ -218,7 +223,7 @@ class HulyMarkdownPreprocessor extends BaseMarkdownPreprocessor { private getSourceMetadata (id: Ref): DocMetadata | null { const sourceMeta = this.metadataById.get(id) if (sourceMeta == null) { - console.warn(`Source metadata not found for ${id}`) + this.logger.error(`Source metadata not found for ${id}`) return null } return sourceMeta @@ -266,7 +271,8 @@ export class UnifiedFormatImporter { constructor ( private readonly client: TxOperations, - private readonly fileUploader: FileUploader + private readonly fileUploader: FileUploader, + private readonly logger: Logger ) {} async importFolder (folderPath: string): Promise { @@ -275,21 +281,28 @@ export class UnifiedFormatImporter { const workspaceData = await this.processImportFolder(folderPath) - console.log('========================================') - console.log('IMPORT DATA STRUCTURE: ', JSON.stringify(workspaceData, null, 4)) - console.log('========================================') + this.logger.log('========================================') + this.logger.log('IMPORT DATA STRUCTURE: ' + JSON.stringify(workspaceData)) + this.logger.log('========================================') - console.log('Importing documents...') + this.logger.log('Importing documents...') const preprocessor = new HulyMarkdownPreprocessor( this.fileUploader.getFileUrl, + this.logger, this.metadataByFilePath, this.metadataById, this.attachMetadataByPath, this.personsByName ) - await new WorkspaceImporter(this.client, this.fileUploader, workspaceData, preprocessor).performImport() - - console.log('Importing attachments...') + await new WorkspaceImporter( + this.client, + this.logger, + this.fileUploader, + workspaceData, + preprocessor + ).performImport() + + this.logger.log('Importing attachments...') const attachments: ImportAttachment[] = Array.from(this.attachMetadataByPath.values()) .filter((attachment) => attachment.parentId !== undefined) .map((attachment) => { @@ -305,10 +318,10 @@ export class UnifiedFormatImporter { spaceId: attachment.spaceId } }) - await new WorkspaceImporter(this.client, this.fileUploader, { attachments }).performImport() + await new WorkspaceImporter(this.client, this.logger, this.fileUploader, { attachments }).performImport() - console.log('========================================') - console.log('IMPORT SUCCESS') + this.logger.log('========================================') + this.logger.log('IMPORT SUCCESS') } private async processImportFolder (folderPath: string): Promise { @@ -336,11 +349,11 @@ export class UnifiedFormatImporter { const spacePath = path.join(folderPath, spaceName) try { - console.log(`Processing ${spaceName}...`) + this.logger.log(`Processing ${spaceName}...`) const spaceConfig = yaml.load(fs.readFileSync(yamlPath, 'utf8')) as UnifiedSpaceSettings - if (spaceConfig.class === undefined) { - console.warn(`Skipping ${spaceName}: not a space - no class specified`) + if (spaceConfig?.class === undefined) { + this.logger.error(`Skipping ${spaceName}: not a space - no class specified`) continue } @@ -392,7 +405,7 @@ export class UnifiedFormatImporter { const issueHeader = (await this.readYamlHeader(issuePath)) as UnifiedIssueHeader if (issueHeader.class === undefined) { - console.warn(`Skipping ${issueFile}: not an issue`) + this.logger.error(`Skipping ${issueFile}: not an issue`) continue } @@ -470,7 +483,7 @@ export class UnifiedFormatImporter { const docHeader = (await this.readYamlHeader(docPath)) as UnifiedDocumentHeader if (docHeader.class === undefined) { - console.warn(`Skipping ${docFile}: not a document`) + this.logger.error(`Skipping ${docFile}: not a document`) continue } @@ -534,11 +547,14 @@ export class UnifiedFormatImporter { private async processProject (projectHeader: UnifiedProjectSettings): Promise { return { class: tracker.class.Project, + id: projectHeader.id as Ref, title: projectHeader.title, identifier: projectHeader.identifier, private: projectHeader.private ?? false, autoJoin: projectHeader.autoJoin ?? true, + archived: projectHeader.archived ?? false, description: projectHeader.description, + emoji: projectHeader.emoji, defaultIssueStatus: projectHeader.defaultIssueStatus !== undefined ? { name: projectHeader.defaultIssueStatus } : undefined, owners: @@ -555,7 +571,9 @@ export class UnifiedFormatImporter { title: spaceHeader.title, private: spaceHeader.private ?? false, autoJoin: spaceHeader.autoJoin ?? true, + archived: spaceHeader.archived ?? false, description: spaceHeader.description, + emoji: spaceHeader.emoji, owners: spaceHeader.owners !== undefined ? spaceHeader.owners.map((email) => this.findAccountByEmail(email)) : [], members: spaceHeader.members !== undefined ? spaceHeader.members.map((email) => this.findAccountByEmail(email)) : [], @@ -564,7 +582,7 @@ export class UnifiedFormatImporter { } private async readYamlHeader (filePath: string): Promise { - console.log('Read YAML header from: ', filePath) + this.logger.log('Read YAML header from: ' + filePath) const content = fs.readFileSync(filePath, 'utf8') const match = content.match(/^---\n([\s\S]*?)\n---/) if (match != null) { diff --git a/dev/import-tool/src/importer/builder.ts b/packages/importer/src/importer/builder.ts similarity index 94% rename from dev/import-tool/src/importer/builder.ts rename to packages/importer/src/importer/builder.ts index d47a84a5f6..c9f0ccbc48 100644 --- a/dev/import-tool/src/importer/builder.ts +++ b/packages/importer/src/importer/builder.ts @@ -256,6 +256,18 @@ export class ImportWorkspaceBuilder { errors.push('defaultIssueStatus not found: ' + project.defaultIssueStatus.name) } + if (project.id !== undefined && project.id !== tracker.project.DefaultProject) { + errors.push('update operation is only allowed for tracker:project:DefaultProject') + } + + if (project.archived !== undefined) { + errors.push(...this.validateType(project.archived, 'boolean', 'archived')) + } + + if (project.emoji !== undefined) { + errors.push(...this.validateEmoji(project.emoji)) + } + errors.push(...this.validateProjectIdentifier(project.identifier)) return errors } @@ -303,9 +315,22 @@ export class ImportWorkspaceBuilder { errors.push(...this.validateType(teamspace.description, 'string', 'description')) } + if (teamspace.archived !== undefined) { + errors.push(...this.validateType(teamspace.archived, 'boolean', 'archived')) + } + + if (teamspace.emoji !== undefined) { + errors.push(...this.validateType(teamspace.emoji, 'string', 'emoji')) + } + + if (teamspace.emoji !== undefined) { + errors.push(...this.validateEmoji(teamspace.emoji)) + } + if (!this.validateStringDefined(teamspace.title)) { errors.push('title is required') } + if (teamspace.class !== document.class.Teamspace) { errors.push('invalid class: ' + teamspace.class) } @@ -446,6 +471,14 @@ export class ImportWorkspaceBuilder { issue.subdocs = childIssues } + private validateEmoji (emoji: string): string[] { + const errors: string[] = [] + if (typeof emoji === 'string' && emoji.codePointAt(0) == null) { + errors.push('Invalid emoji: ' + emoji) + } + return errors + } + private validateType (value: unknown, type: 'string' | 'number' | 'boolean', fieldName: string): string[] { const errors: string[] = [] switch (type) { diff --git a/dev/import-tool/src/importer/dowloader.ts b/packages/importer/src/importer/dowloader.ts similarity index 100% rename from dev/import-tool/src/importer/dowloader.ts rename to packages/importer/src/importer/dowloader.ts diff --git a/dev/import-tool/src/importer/uploader.ts b/packages/importer/src/importer/frontUploader.ts similarity index 53% rename from dev/import-tool/src/importer/uploader.ts rename to packages/importer/src/importer/frontUploader.ts index d2ae4fb7d9..95fe9b09b7 100644 --- a/dev/import-tool/src/importer/uploader.ts +++ b/packages/importer/src/importer/frontUploader.ts @@ -13,19 +13,27 @@ // limitations under the License. // import { - type Ref, - type Blob as PlatformBlob, type CollaborativeDoc, concatLink, - makeCollabJsonId + makeCollabJsonId, + Markup, + type Blob as PlatformBlob, + type Ref } from '@hcengineering/core' +import { FileUploader, UploadResult } from './uploader' + +interface FileUploadError { + key: string + error: string +} -export interface FileUploader { - uploadFile: (name: string, file: Blob) => Promise> - uploadCollaborativeDoc: (collabId: CollaborativeDoc, data: Buffer) => Promise> - getFileUrl: (id: string) => string +interface FileUploadSuccess { + key: string + id: string } +type FileUploadResult = FileUploadSuccess | FileUploadError + export class FrontFileUploader implements FileUploader { constructor ( private readonly frontUrl: string, @@ -35,11 +43,11 @@ export class FrontFileUploader implements FileUploader { this.getFileUrl = this.getFileUrl.bind(this) } - public async uploadFile (name: string, file: Blob): Promise> { + public async uploadFile (name: string, blob: Blob): Promise { const form = new FormData() - form.append('file', file, name) + form.append('file', blob, name) - const res = await fetch(concatLink(this.frontUrl, '/files'), { + const response = await fetch(concatLink(this.frontUrl, '/files'), { method: 'POST', headers: { Authorization: 'Bearer ' + this.token @@ -47,20 +55,36 @@ export class FrontFileUploader implements FileUploader { body: form }) - if (res.ok && res.status === 200) { - return name as Ref + if (response.status !== 200) { + return { success: false, error: response.statusText } + } + + const responseText = await response.text() + if (responseText === undefined) { + return { success: false, error: response.statusText } + } + + const uploadResult = JSON.parse(responseText) as FileUploadResult[] + if (!Array.isArray(uploadResult) || uploadResult.length === 0) { + return { success: false, error: response.statusText } + } + + const result = uploadResult[0] + if ('error' in result) { + return { success: false, error: result.error } } - throw new Error('Failed to upload file') + return { success: true, id: result.id as Ref } } public getFileUrl (id: string): string { return concatLink(this.frontUrl, `/files/${this.workspaceId}/${id}?file=${id}&workspace=${this.workspaceId}`) } - public async uploadCollaborativeDoc (collabId: CollaborativeDoc, data: Buffer): Promise> { + public async uploadCollaborativeDoc (collabId: CollaborativeDoc, content: Markup): Promise { + const buffer = Buffer.from(content) const blobId = makeCollabJsonId(collabId) - const blob = new Blob([data], { type: 'application/json' }) + const blob = new Blob([buffer], { type: 'application/json' }) return await this.uploadFile(blobId, blob) } } diff --git a/dev/import-tool/src/importer/importer.ts b/packages/importer/src/importer/importer.ts similarity index 80% rename from dev/import-tool/src/importer/importer.ts rename to packages/importer/src/importer/importer.ts index 93f285ee70..e00de25a4a 100644 --- a/dev/import-tool/src/importer/importer.ts +++ b/packages/importer/src/importer/importer.ts @@ -51,8 +51,10 @@ import tracker, { type Project, TimeReportDayType } from '@hcengineering/tracker' +import view from '@hcengineering/view' import { type MarkdownPreprocessor, NoopMarkdownPreprocessor } from './preprocessor' import { type FileUploader } from './uploader' +import { Logger } from './logger' export interface ImportWorkspace { projectTypes?: ImportProjectType[] @@ -82,7 +84,9 @@ export interface ImportSpace { title: string private: boolean autoJoin?: boolean + archived?: boolean description?: string + emoji?: string owners?: Ref[] members?: Ref[] docs: T[] @@ -107,6 +111,7 @@ export interface ImportDocument extends ImportDoc { export interface ImportProject extends ImportSpace { class: Ref> + id?: Ref identifier: string projectType?: ImportProjectType defaultIssueStatus?: ImportStatus @@ -147,6 +152,7 @@ export class WorkspaceImporter { constructor ( private readonly client: TxOperations, + private readonly logger: Logger, private readonly fileUploader: FileUploader, private readonly workspaceData: ImportWorkspace, private readonly preprocessor: MarkdownPreprocessor = new NoopMarkdownPreprocessor() @@ -234,9 +240,9 @@ export class WorkspaceImporter { } async importTeamspace (space: ImportTeamspace): Promise> { - console.log('Creating teamspace: ', space.title) + this.logger.log('Creating teamspace: ' + space.title) const teamspaceId = await this.createTeamspace(space) - console.log('Teamspace created: ', teamspaceId) + this.logger.log('Teamspace created: ' + teamspaceId) for (const doc of space.docs) { await this.createDocumentWithSubdocs(doc, document.ids.NoParent, teamspaceId) } @@ -248,9 +254,9 @@ export class WorkspaceImporter { parentId: Ref, teamspaceId: Ref ): Promise> { - console.log('Creating document: ', doc.title) + this.logger.log('Creating document: ' + doc.title) const documentId = await this.createDocument(doc, parentId, teamspaceId) - console.log('Document created: ', documentId) + this.logger.log('Document created: ' + documentId) for (const child of doc.subdocs) { await this.createDocumentWithSubdocs(child, documentId, teamspaceId) } @@ -259,16 +265,19 @@ export class WorkspaceImporter { async createTeamspace (space: ImportTeamspace): Promise> { const teamspaceId = generateId() + const codePoint = space.emoji?.codePointAt(0) const data = { type: document.spaceType.DefaultTeamspaceType, description: space.description ?? '', title: space.title, name: space.title, private: space.private, + color: codePoint, + icon: codePoint === undefined ? undefined : view.ids.IconWithEmoji, owners: space.owners ?? [], members: space.members ?? [], autoJoin: space.autoJoin, - archived: false + archived: space.archived ?? false } await this.client.createDoc(document.class.Teamspace, core.space.Space, data, teamspaceId) return teamspaceId @@ -304,9 +313,17 @@ export class WorkspaceImporter { } async importProject (project: ImportProject): Promise> { - console.log('Creating project: ', project.title) - const projectId = await this.createProject(project) - console.log('Project created: ' + projectId) + let projectId: Ref + if (project.id === tracker.project.DefaultProject) { + this.logger.log('Setting up default project: ' + project.title) + projectId = tracker.project.DefaultProject + await this.updateProject(projectId, project) + this.logger.log('Default project updated: ' + projectId) + } else { + this.logger.log('Creating project: ', project.title) + projectId = await this.createProject(project) + this.logger.log('Project created: ' + projectId) + } const projectDoc = await this.client.findOne(tracker.class.Project, { _id: projectId }) if (projectDoc === undefined) { @@ -314,7 +331,7 @@ export class WorkspaceImporter { } for (const issue of project.docs) { - await this.createIssueWithSubissues(issue, tracker.ids.NoParent, projectDoc, []) + await this.createIssueWithSubissues(issue, tracker.ids.NoParent, projectDoc, projectId, []) } return projectId } @@ -323,31 +340,68 @@ export class WorkspaceImporter { issue: ImportIssue, parentId: Ref, project: Project, + spaceId: Ref, parentsInfo: IssueParentInfo[] ): Promise<{ id: Ref, identifier: string }> { - console.log('Creating issue: ', issue.title) - const issueResult = await this.createIssue(issue, project, parentId, parentsInfo) - console.log('Issue created: ', issueResult) + this.logger.log('Creating issue: ' + issue.title) + const issueResult = await this.createIssue(issue, project, parentId, spaceId, parentsInfo) + this.logger.log('Issue created: ' + JSON.stringify(issueResult)) if (issue.subdocs.length > 0) { const parentsInfoEx = [ { parentId: issueResult.id, parentTitle: issue.title, - space: project._id, + space: spaceId, identifier: issueResult.identifier }, ...parentsInfo ] for (const child of issue.subdocs) { - await this.createIssueWithSubissues(child as ImportIssue, issueResult.id, project, parentsInfoEx) + await this.createIssueWithSubissues(child as ImportIssue, issueResult.id, project, spaceId, parentsInfoEx) } } return issueResult } + async updateProject (projectId: Ref, project: ImportProject): Promise> { + const oldProject = await this.client.findOne(tracker.class.Project, { _id: projectId }) + if (oldProject === undefined) { + throw new Error('Project not found: ' + projectId) + } + const codePoint = project.emoji?.codePointAt(0) + const projectData = { + name: project.title, + private: project.private, + description: project.description ?? oldProject.description, + members: project.members ?? oldProject.members, + owners: project.owners ?? oldProject.owners, + archived: project.archived ?? oldProject.archived, + autoJoin: project.autoJoin ?? oldProject.autoJoin, + identifier: + project.identifier !== undefined + ? await this.uniqueProjectIdentifier(project.identifier) + : oldProject.identifier, + sequence: oldProject.sequence, + color: codePoint ?? oldProject.color, + icon: codePoint === undefined ? undefined : view.ids.IconWithEmoji, + defaultIssueStatus: + project.defaultIssueStatus !== undefined + ? this.issueStatusByName.get(project.defaultIssueStatus.name) + : oldProject.defaultIssueStatus, + defaultTimeReportDay: oldProject.defaultTimeReportDay, + type: + project.projectType !== undefined + ? this.projectTypeByName.get(project.projectType.name) ?? tracker.ids.ClassingProjectType + : oldProject.type + } + + await this.client.updateDoc(tracker.class.Project, core.space.Space, projectId, projectData) + return projectId + } + async createProject (project: ImportProject): Promise> { const projectId = generateId() @@ -362,6 +416,7 @@ export class WorkspaceImporter { : tracker.status.Backlog const identifier = await this.uniqueProjectIdentifier(project.identifier) + const codePoint = project.emoji?.codePointAt(0) const projectData = { name: project.title, description: project.description ?? '', @@ -372,6 +427,8 @@ export class WorkspaceImporter { autoJoin: project.autoJoin, identifier, sequence: 0, + color: codePoint, + icon: codePoint != null ? view.ids.IconWithEmoji : undefined, defaultIssueStatus: defaultIssueStatus ?? tracker.status.Backlog, defaultTimeReportDay: TimeReportDayType.PreviousWorkDay, type: projectType as Ref @@ -388,20 +445,21 @@ export class WorkspaceImporter { issue: ImportIssue, project: Project, parentId: Ref, + spaceId: Ref, parentsInfo: IssueParentInfo[] ): Promise<{ id: Ref, identifier: string }> { const issueId = issue.id ?? generateId() const content = await issue.descrProvider() const collabId = makeCollabId(tracker.class.Issue, issueId, 'description') - const contentId = await this.createCollaborativeContent(issueId, collabId, content, project._id) + const contentId = await this.createCollaborativeContent(issueId, collabId, content, spaceId) const { number, identifier } = issue.number !== undefined ? { number: issue.number, identifier: `${project.identifier}-${issue.number}` } - : await this.getNextIssueIdentifier(project) + : await this.getNextIssueIdentifier(project, spaceId) const kind = await this.getIssueKind(project) - const rank = await this.getIssueRank(project) + const rank = await this.getIssueRank(project, spaceId) const status = await this.findIssueStatusByName(issue.status.name) const priority = issue.priority !== undefined @@ -436,7 +494,7 @@ export class WorkspaceImporter { await this.client.addCollection( tracker.class.Issue, - project._id, + spaceId, parentId, tracker.class.Issue, 'subIssues', @@ -445,16 +503,19 @@ export class WorkspaceImporter { ) if (issue.comments !== undefined) { - await this.importComments(issueId, issue.comments, project._id) + await this.importComments(issueId, issue.comments, spaceId) } return { id: issueId, identifier } } - private async getNextIssueIdentifier (project: Project): Promise<{ number: number, identifier: string }> { + private async getNextIssueIdentifier ( + project: Project, + spaceId: Ref + ): Promise<{ number: number, identifier: string }> { const incResult = await this.client.updateDoc( tracker.class.Project, core.space.Space, - project._id, + spaceId, { $inc: { sequence: 1 } }, true ) @@ -467,15 +528,15 @@ export class WorkspaceImporter { const taskKind = project?.type !== undefined ? { parent: project.type } : {} const kind = await this.client.findOne(task.class.TaskType, taskKind) if (kind === undefined) { - throw new Error(`Task type not found for project: ${project._id}`) + throw new Error(`Task type not found for project: ${project.name}`) } return kind } - private async getIssueRank (project: Project): Promise { + private async getIssueRank (project: Project, spaceId: Ref): Promise { const lastIssue = await this.client.findOne( tracker.class.Issue, - { space: project._id }, + { space: spaceId }, { sort: { rank: SortingOrder.Descending } } ) return makeRank(lastIssue?.rank, undefined) @@ -529,7 +590,7 @@ export class WorkspaceImporter { ): Promise { const blob = await attachment.blobProvider() if (blob === null) { - console.warn('Failed to read attachment file: ', attachment.title) + this.logger.error('Failed to read attachment file: ' + attachment.title) return } @@ -538,7 +599,7 @@ export class WorkspaceImporter { try { await this.createAttachment(attachment.id ?? generateId(), file, spaceId, parentId, parentClass) } catch { - console.warn('Failed to upload attachment file: ', attachment.title) + this.logger.error('Failed to upload attachment file: ', attachment.title) } } @@ -550,7 +611,10 @@ export class WorkspaceImporter { parentClass: Ref>> ): Promise> { const attachmentId = generateId() - const blobId = await this.fileUploader.uploadFile(id, file) + const uploadResult = await this.fileUploader.uploadFile(id, file) + if (!uploadResult.success) { + throw new Error('Failed to upload attachment file: ' + file.name) + } await this.client.addCollection( attachment.class.Attachment, spaceId, @@ -558,7 +622,7 @@ export class WorkspaceImporter { parentClass, 'attachments', { - file: blobId, + file: uploadResult.id, lastModified: Date.now(), name: file.name, size: file.size, @@ -580,9 +644,12 @@ export class WorkspaceImporter { const processedJson = this.preprocessor.process(json, id, spaceId) const markup = jsonToMarkup(processedJson) - const buffer = Buffer.from(markup) - return await this.fileUploader.uploadCollaborativeDoc(collabId, buffer) + const result = await this.fileUploader.uploadCollaborativeDoc(collabId, markup) + if (result.success) { + return result.id + } + throw new Error('Failed to upload collaborative document: ' + id) } async findIssueStatusByName (name: string): Promise> { diff --git a/packages/importer/src/importer/logger.ts b/packages/importer/src/importer/logger.ts new file mode 100644 index 0000000000..5d734933af --- /dev/null +++ b/packages/importer/src/importer/logger.ts @@ -0,0 +1,18 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +export interface Logger { + log: (msg: string, data?: any) => void + error: (msg: string, data?: any) => void +} diff --git a/dev/import-tool/src/importer/preprocessor.ts b/packages/importer/src/importer/preprocessor.ts similarity index 100% rename from dev/import-tool/src/importer/preprocessor.ts rename to packages/importer/src/importer/preprocessor.ts diff --git a/packages/importer/src/importer/storageUploader.ts b/packages/importer/src/importer/storageUploader.ts new file mode 100644 index 0000000000..3668d55985 --- /dev/null +++ b/packages/importer/src/importer/storageUploader.ts @@ -0,0 +1,59 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { saveCollabJson } from '@hcengineering/collaboration' +import { + CollaborativeDoc, + Markup, + MeasureContext, + Blob as PlatformBlob, + Ref, + WorkspaceIdWithUrl +} from '@hcengineering/core' +import type { StorageAdapter } from '@hcengineering/server-core' +import { FileUploader, UploadResult } from './uploader' + +export class StorageFileUploader implements FileUploader { + constructor ( + private readonly ctx: MeasureContext, + private readonly storageAdapter: StorageAdapter, + private readonly wsUrl: WorkspaceIdWithUrl + ) { + this.uploadFile = this.uploadFile.bind(this) + } + + public async uploadFile (id: string, blob: Blob): Promise { + try { + const arrayBuffer = await blob.arrayBuffer() + const buffer = Buffer.from(arrayBuffer) + await this.storageAdapter.put(this.ctx, this.wsUrl, id, buffer, blob.type, buffer.byteLength) + return { success: true, id: id as Ref } + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) } + } + } + + public async uploadCollaborativeDoc (collabId: CollaborativeDoc, content: Markup): Promise { + try { + const blobId = await saveCollabJson(this.ctx, this.storageAdapter, this.wsUrl, collabId, content) + return { success: true, id: blobId } + } catch (error) { + return { success: false, error: error instanceof Error ? error.message : String(error) } + } + } + + public getFileUrl (id: string): string { + return '' + } +} diff --git a/packages/importer/src/importer/uploader.ts b/packages/importer/src/importer/uploader.ts new file mode 100644 index 0000000000..7f2a4cc0c4 --- /dev/null +++ b/packages/importer/src/importer/uploader.ts @@ -0,0 +1,33 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// +import { type CollaborativeDoc, type Blob as PlatformBlob, type Markup, type Ref } from '@hcengineering/core' + +export interface SuccessUploadResult { + success: true + id: Ref +} + +export interface FailureUploadResult { + success: false + error: string +} + +export type UploadResult = SuccessUploadResult | FailureUploadResult + +export interface FileUploader { + uploadFile: (name: string, blob: Blob) => Promise + uploadCollaborativeDoc: (collabId: CollaborativeDoc, content: Markup) => Promise + getFileUrl: (id: string) => string +} diff --git a/packages/importer/src/index.ts b/packages/importer/src/index.ts new file mode 100644 index 0000000000..83ab51cf82 --- /dev/null +++ b/packages/importer/src/index.ts @@ -0,0 +1,23 @@ +// +// Copyright © 2024 Hardcore Engineering Inc. +// +// Licensed under the Eclipse Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. You may +// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// +// See the License for the specific language governing permissions and +// limitations under the License. +// + +export * from './huly/unified' +export * from './clickup/clickup' +export * from './notion/notion' + +export * from './importer/uploader' +export * from './importer/storageUploader' +export * from './importer/frontUploader' +export * from './importer/logger' diff --git a/dev/import-tool/src/notion/notion.ts b/packages/importer/src/notion/notion.ts similarity index 98% rename from dev/import-tool/src/notion/notion.ts rename to packages/importer/src/notion/notion.ts index aff260f759..c61b56f207 100644 --- a/dev/import-tool/src/notion/notion.ts +++ b/packages/importer/src/notion/notion.ts @@ -441,11 +441,13 @@ async function importPageDocument ( preProcessMarkdown(json, documentMetaMap, fileUploader) } const markup = jsonToMarkup(json) - const buffer = Buffer.from(markup) const id = docMeta.id as Ref const collabId = makeCollabId(document.class.Document, id, 'content') - const blobId = await fileUploader.uploadCollaborativeDoc(collabId, buffer) + const uploadResult = await fileUploader.uploadCollaborativeDoc(collabId, markup) + if (!uploadResult.success) { + throw new Error('Failed to upload collaborative document: ' + docMeta.id) + } const parent = (parentMeta?.id as Ref) ?? document.ids.NoParent @@ -454,7 +456,7 @@ async function importPageDocument ( const attachedData: Data = { title: docMeta.name, - content: blobId, + content: uploadResult.id, parent, attachments: 0, embeddings: 0, diff --git a/packages/importer/tsconfig.json b/packages/importer/tsconfig.json new file mode 100644 index 0000000000..59e4fd4297 --- /dev/null +++ b/packages/importer/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "./node_modules/@hcengineering/platform-rig/profiles/default/tsconfig.json", + + "compilerOptions": { + "rootDir": "./src", + "outDir": "./lib", + "declarationDir": "./types", + "tsBuildInfoFile": ".build/build.tsbuildinfo" + } +} \ No newline at end of file diff --git a/rush.json b/rush.json index a2f8e5ed0e..fb79acc469 100644 --- a/rush.json +++ b/rush.json @@ -455,6 +455,11 @@ "projectFolder": "packages/api-client", "shouldPublish": false }, + { + "packageName": "@hcengineering/importer", + "projectFolder": "packages/importer", + "shouldPublish": false + }, { "packageName": "@hcengineering/collaboration", "projectFolder": "server/collaboration", diff --git a/server/account-service/package.json b/server/account-service/package.json index 937390de69..cb7c4c5048 100644 --- a/server/account-service/package.json +++ b/server/account-service/package.json @@ -53,7 +53,6 @@ "koa-router": "^12.0.1", "koa-bodyparser": "^4.4.1", "@koa/cors": "^5.0.0", - "@hcengineering/server-tool": "^0.6.0", "@hcengineering/server-token": "^0.6.11", "@hcengineering/analytics": "^0.6.0" } diff --git a/server/account-service/src/index.ts b/server/account-service/src/index.ts index 3bfd42de8c..884289f5e9 100644 --- a/server/account-service/src/index.ts +++ b/server/account-service/src/index.ts @@ -17,7 +17,6 @@ import { registerProviders } from '@hcengineering/auth-providers' import { metricsAggregate, type BrandingMap, type MeasureContext } from '@hcengineering/core' import platform, { Severity, Status, addStringsLoader, setMetadata } from '@hcengineering/platform' import serverToken, { decodeToken } from '@hcengineering/server-token' -import toolPlugin from '@hcengineering/server-tool' import cors from '@koa/cors' import { type IncomingHttpHeaders } from 'http' import Koa from 'koa' @@ -87,11 +86,6 @@ export function serveAccount (measureCtx: MeasureContext, brandings: BrandingMap setMetadata(serverToken.metadata.Secret, serverSecret) - const initScriptUrl = process.env.INIT_SCRIPT_URL - if (initScriptUrl !== undefined) { - setMetadata(toolPlugin.metadata.InitScriptURL, initScriptUrl) - } - const hasSignUp = process.env.DISABLE_SIGNUP !== 'true' const methods = getMethods(hasSignUp) diff --git a/server/tool/package.json b/server/tool/package.json index 387660b94d..51c9436fe2 100644 --- a/server/tool/package.json +++ b/server/tool/package.json @@ -46,6 +46,7 @@ "@hcengineering/contact": "^0.6.24", "@hcengineering/client-resources": "^0.6.27", "@hcengineering/client": "^0.6.18", + "@hcengineering/importer": "^0.6.1", "@hcengineering/model": "^0.6.11", "@hcengineering/rank": "^0.6.4", "uuid": "^8.3.2", diff --git a/server/tool/src/index.ts b/server/tool/src/index.ts index 2fb7408b86..b7a945988a 100644 --- a/server/tool/src/index.ts +++ b/server/tool/src/index.ts @@ -192,14 +192,20 @@ export async function initializeWorkspace ( progress: (value: number) => Promise ): Promise { const initWS = branding?.initWorkspace ?? getMetadata(toolPlugin.metadata.InitWorkspace) - const scriptUrl = getMetadata(toolPlugin.metadata.InitScriptURL) - ctx.info('Init script details', { scriptUrl, initWS }) - if (initWS === undefined || scriptUrl === undefined) return + const initRepoDir = getMetadata(toolPlugin.metadata.InitRepoDir) + ctx.info('Init script details', { initWS, initRepoDir }) + if (initWS === undefined || initRepoDir === undefined) return + + const initScriptFile = path.resolve(initRepoDir, 'script.yaml') + if (!fs.existsSync(initScriptFile)) { + ctx.warn('Init script file not found in init directory', { initScriptFile }) + return + } + try { - // `https://raw.githubusercontent.com/hcengineering/init/main/script.yaml` - const req = await fetch(scriptUrl) - const text = await req.text() + const text = fs.readFileSync(initScriptFile, 'utf8') const scripts = yaml.load(text) as any as InitScript[] + let script: InitScript | undefined if (initWS !== undefined) { script = scripts.find((it) => it.name === initWS) @@ -211,7 +217,7 @@ export async function initializeWorkspace ( return } - const initializer = new WorkspaceInitializer(ctx, storageAdapter, wsUrl, client) + const initializer = new WorkspaceInitializer(ctx, storageAdapter, wsUrl, client, initRepoDir) await initializer.processScript(script, logger, progress) } catch (err: any) { ctx.error('Failed to initialize workspace', { error: err }) diff --git a/server/tool/src/initializer.ts b/server/tool/src/initializer.ts index 250115c633..1ee620a67d 100644 --- a/server/tool/src/initializer.ts +++ b/server/tool/src/initializer.ts @@ -15,9 +15,11 @@ import core, { } from '@hcengineering/core' import { ModelLogger } from '@hcengineering/model' import { makeRank } from '@hcengineering/rank' +import { StorageFileUploader, UnifiedFormatImporter } from '@hcengineering/importer' import type { StorageAdapter } from '@hcengineering/server-core' import { jsonToMarkup, parseMessageMarkdown } from '@hcengineering/text' import { v4 as uuid } from 'uuid' +import path from 'path' const fieldRegexp = /\${\S+?}/ @@ -35,7 +37,7 @@ export type InitStep = | UpdateStep | FindStep | UploadStep - + | ImportStep export interface CreateStep { type: 'create' _class: Ref> @@ -82,6 +84,11 @@ export interface UploadStep { resultVariable?: string } +export interface ImportStep { + type: 'import' + path: string +} + export type Props = Data & Partial & { space: Ref } export class WorkspaceInitializer { @@ -93,7 +100,8 @@ export class WorkspaceInitializer { private readonly ctx: MeasureContext, private readonly storageAdapter: StorageAdapter, private readonly wsUrl: WorkspaceIdWithUrl, - private readonly client: TxOperations + private readonly client: TxOperations, + private readonly initRepoDir: string ) {} async processScript ( @@ -118,6 +126,8 @@ export class WorkspaceInitializer { await this.processFind(step, vars) } else if (step.type === 'upload') { await this.processUpload(step, vars, logger) + } else if (step.type === 'import') { + await this.processImport(step, vars, logger) } await progress(Math.round(((index + 1) * 100) / script.steps.length)) @@ -152,6 +162,18 @@ export class WorkspaceInitializer { } } + private async processImport (step: ImportStep, vars: Record, logger: ModelLogger): Promise { + try { + const uploader = new StorageFileUploader(this.ctx, this.storageAdapter, this.wsUrl) + const initPath = path.resolve(this.initRepoDir, step.path) + const importer = new UnifiedFormatImporter(this.client, uploader, logger) + await importer.importFolder(initPath) + } catch (error) { + logger.error('Import failed', error) + throw error + } + } + private async processFind(step: FindStep, vars: Record): Promise { const query = this.fillProps(step.query, vars) const res = await this.client.findOne(step._class, { ...(query as any) }) diff --git a/server/tool/src/plugin.ts b/server/tool/src/plugin.ts index 880560884d..3a3136caec 100644 --- a/server/tool/src/plugin.ts +++ b/server/tool/src/plugin.ts @@ -11,7 +11,7 @@ export const toolId = 'tool' as Plugin const toolPlugin = plugin(toolId, { metadata: { InitWorkspace: '' as Metadata, - InitScriptURL: '' as Metadata + InitRepoDir: '' as Metadata } }) diff --git a/server/workspace-service/src/index.ts b/server/workspace-service/src/index.ts index 438642aafb..a97425cd3e 100644 --- a/server/workspace-service/src/index.ts +++ b/server/workspace-service/src/index.ts @@ -82,12 +82,13 @@ export function serveWorkspaceAccount ( if (initWS !== undefined) { setMetadata(toolPlugin.metadata.InitWorkspace, initWS) } - const initScriptUrl = process.env.INIT_SCRIPT_URL - if (initScriptUrl !== undefined) { - setMetadata(toolPlugin.metadata.InitScriptURL, initScriptUrl) + + const initRepoDir = process.env.INIT_REPO_DIR + if (initRepoDir !== undefined) { + setMetadata(toolPlugin.metadata.InitRepoDir, initRepoDir) } - setMetadata(serverClientPlugin.metadata.UserAgent, 'WorkspaceService') + setMetadata(serverClientPlugin.metadata.UserAgent, 'WorkspaceService') setMetadata(serverNotification.metadata.InboxOnlyNotifications, true) let canceled = false