Integrate import to workspace initialization

Signed-off-by: Anna Khismatullina <[email protected]>
hcengineering · Dec 4, 2024 · 7887ff4 · 7887ff4
1 parent 1526ae2
commit 7887ff4
Show file tree

Hide file tree

Showing 46 changed files with 595 additions and 148 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -626,7 +626,7 @@
       "name": "Debug Huly import",
       "type": "node",
       "request": "launch",
-      "args": ["src/__start.ts", "import", "/home/anna/xored/huly/platform/dev/import-tool/src/huly/example-workspace", "-u", "user1", "-pw", "1234", "-ws", "ws12"],
+      "args": ["src/__start.ts", "import", "/home/anna/xored/huly/platform/dev/import-tool/docs/huly/example-workspace", "-u", "user1", "-pw", "1234", "-ws", "ws12"],
       "env": {
         "FRONT_URL": "http://localhost:8087"
       },

diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml
diff --git a/dev/.env b/dev/.env
@@ -1,3 +1,4 @@
 STORAGE_CONFIG="minio|minio?accessKey=minioadmin&secretKey=minioadmin"
 MONGO_URL=mongodb://mongodb:27017?compressors=snappy
-DB_URL_PG=postgresql://postgres:example@postgres:5432
+DB_URL_PG=postgresql://postgres:example@postgres:5432
+WS_INIT_SCRIPTS=../../init
diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml
@@ -125,6 +125,7 @@ services:
       - stats
     volumes:
       - ./branding.json:/var/cfg/branding.json
+      - ${WS_INIT_SCRIPTS}:/init-scripts
     environment:
       # - WS_OPERATION=create
       - SERVER_SECRET=secret
@@ -138,7 +139,7 @@ services:
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - BRANDING_PATH=/var/cfg/branding.json
       # - PARALLEL=2
-      - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml
+      - INIT_REPO_DIR=/init-scripts
       - INIT_WORKSPACE=test
     restart: unless-stopped
   workspacepg:
@@ -151,6 +152,7 @@ services:
       - stats
     volumes:
       - ./branding.json:/var/cfg/branding.json
+      - ${WS_INIT_SCRIPTS}:/init-scripts
     environment:
       # - WS_OPERATION=create
       - SERVER_SECRET=secret
@@ -164,8 +166,8 @@ services:
       - MODEL_ENABLED=*
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - BRANDING_PATH=/var/cfg/branding.json
+      - INIT_REPO_DIR=/init-scripts
       # - PARALLEL=2
-      - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml
       # - INIT_WORKSPACE=onboarding
     restart: unless-stopped    
   workspace_cockroach:
@@ -178,6 +180,7 @@ services:
       - stats
     volumes:
       - ./branding.json:/var/cfg/branding.json
+      - ${WS_INIT_SCRIPTS}:/init-scripts
     environment:
       # - WS_OPERATION=create
       - SERVER_SECRET=secret
@@ -190,8 +193,8 @@ services:
       - MODEL_ENABLED=*
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - BRANDING_PATH=/var/cfg/branding.json
+      - INIT_REPO_DIR=/init-scripts
       # - PARALLEL=2
-      - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml
       # - INIT_WORKSPACE=onboarding
     restart: unless-stopped        
   collaborator:

diff --git a/dev/import-tool/README.md b/dev/import-tool/README.md
@@ -5,9 +5,9 @@ Tool for importing data into Huly workspace.
 ## Recommended Import Method
 
 ### Unified Format Import
-The recommended way to import data into Huly is using our [Unified Import Format](./src/huly/README.md). This format provides a straightforward way to migrate data from any system by converting it into an intermediate, human-readable structure.
+The recommended way to import data into Huly is using our [Unified Import Format](./docs/huly/README.md). This format provides a straightforward way to migrate data from any system by converting it into an intermediate, human-readable structure.
 
-See the [complete guide](./src/huly/README.md) and [example workspace](./src/huly/example-workspace) to get started.
+See the [complete guide](./docs/huly/README.md) and [example workspace](./docs/huly/example-workspace) to get started.
 
 ### Why Use Unified Format?
 - Simple, human-readable format using YAML and Markdown
@@ -19,7 +19,7 @@ See the [complete guide](./src/huly/README.md) and [example workspace](./src/hul
 
 We also support direct import from some platforms:
 
-1. **Notion**: see [Import from Notion Guide](./src/notion/README.md)
-2. **ClickUp**: see [Import from ClickUp Guide](./src/clickup/README.md)
+1. **Notion**: see [Import from Notion Guide](./docs/notion/README.md)
+2. **ClickUp**: see [Import from ClickUp Guide](./docs/clickup/README.md)
 
 These direct imports are suitable for simple migrations, but for complex cases or systems not listed above, please use the Unified Format.
diff --git a/dev/import-tool/src/clickup/README.md → dev/import-tool/docs/clickup/README.md b/dev/import-tool/src/clickup/README.md → dev/import-tool/docs/clickup/README.md
diff --git a/dev/import-tool/src/huly/README.md → dev/import-tool/docs/huly/README.md b/dev/import-tool/src/huly/README.md → dev/import-tool/docs/huly/README.md
diff --git a/...huly/example-workspace/Documentation.yaml → ...huly/example-workspace/Documentation.yaml b/...huly/example-workspace/Documentation.yaml → ...huly/example-workspace/Documentation.yaml
@@ -1,5 +1,6 @@
 class: document:class:Teamspace
 title: Documentation
+emoji: 📖
 private: false
 autoJoin: true
 owners:

diff --git a/...orkspace/Documentation/Getting Started.md → ...orkspace/Documentation/Getting Started.md b/...orkspace/Documentation/Getting Started.md → ...orkspace/Documentation/Getting Started.md
diff --git a/...ple-workspace/Documentation/User Guide.md → ...ple-workspace/Documentation/User Guide.md b/...ple-workspace/Documentation/User Guide.md → ...ple-workspace/Documentation/User Guide.md
diff --git a/.../Documentation/User Guide/Installation.md → .../Documentation/User Guide/Installation.md b/.../Documentation/User Guide/Installation.md → .../Documentation/User Guide/Installation.md
diff --git a/...pace/Documentation/files/architecture.png → ...pace/Documentation/files/architecture.png b/...pace/Documentation/files/architecture.png → ...pace/Documentation/files/architecture.png
diff --git a/...huly/example-workspace/Project Alpha.yaml → ...huly/example-workspace/Project Alpha.yaml b/...huly/example-workspace/Project Alpha.yaml → ...huly/example-workspace/Project Alpha.yaml
@@ -1,6 +1,7 @@
 class: tracker:class:Project
 title: Project Alpha
 identifier: ALPHA
+emoji: 🦄
 private: false
 autoJoin: true
 owners:

diff --git a/...orkspace/Project Alpha/1.Project Setup.md → ...orkspace/Project Alpha/1.Project Setup.md b/...orkspace/Project Alpha/1.Project Setup.md → ...orkspace/Project Alpha/1.Project Setup.md
diff --git a/...t Alpha/1.Project Setup/2.Configure CI.md → ...t Alpha/1.Project Setup/2.Configure CI.md b/...t Alpha/1.Project Setup/2.Configure CI.md → ...t Alpha/1.Project Setup/2.Configure CI.md
diff --git a/...-workspace/Project Alpha/4.Update Docs.md → ...-workspace/Project Alpha/4.Update Docs.md b/...-workspace/Project Alpha/4.Update Docs.md → ...-workspace/Project Alpha/4.Update Docs.md
diff --git a/...workspace/Project Alpha/files/config.yaml → ...workspace/Project Alpha/files/config.yaml b/...workspace/Project Alpha/files/config.yaml → ...workspace/Project Alpha/files/config.yaml
diff --git a/dev/import-tool/src/notion/README.md → dev/import-tool/docs/notion/README.md b/dev/import-tool/src/notion/README.md → dev/import-tool/docs/notion/README.md
diff --git a/dev/import-tool/package.json b/dev/import-tool/package.json
@@ -34,43 +34,26 @@
     "cross-env": "~7.0.3",
     "@hcengineering/platform-rig": "^0.6.0",
     "@typescript-eslint/eslint-plugin": "^6.11.0",
-    "eslint-plugin-import": "^2.26.0",
-    "eslint-plugin-promise": "^6.1.1",
-    "eslint-plugin-n": "^15.4.0",
     "eslint": "^8.54.0",
     "ts-node": "^10.8.0",
     "esbuild": "^0.20.0",
-    "@types/mime-types": "~2.1.1",
     "@types/node": "~20.11.16",
     "@typescript-eslint/parser": "^6.11.0",
-    "eslint-config-standard-with-typescript": "^40.0.0",
-    "prettier": "^3.1.0",
     "typescript": "^5.3.3",
     "jest": "^29.7.0",
     "ts-jest": "^29.1.1",
     "@types/jest": "^29.5.5",
-    "@types/csvtojson": "^2.0.0",
-    "@types/js-yaml": "^4.0.9"
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "prettier": "^3.1.0"
   },
   "dependencies": {
-    "@hcengineering/attachment": "^0.6.14",
-    "@hcengineering/collaboration": "^0.6.0",
-    "@hcengineering/document": "^0.6.0",
-    "@hcengineering/text": "^0.6.5",
-    "@hcengineering/model-attachment": "^0.6.0",
-    "@hcengineering/model-core": "^0.6.0",
     "@hcengineering/core": "^0.6.32",
     "@hcengineering/platform": "^0.6.11",
-    "@hcengineering/server-tool": "^0.6.0",
     "@hcengineering/server-client": "^0.6.0",
-    "@hcengineering/rank": "^0.6.4",
-    "@hcengineering/tracker": "^0.6.24",
-    "commander": "^8.1.0",
-    "mime-types": "~2.1.34",
-    "csvtojson": "^2.0.10",
-    "@hcengineering/task": "^0.6.20",
-    "@hcengineering/contact": "^0.6.24",
-    "@hcengineering/chunter": "^0.6.20",
-    "js-yaml": "^4.1.0"
+    "@hcengineering/importer": "^0.6.1",
+    "commander": "^8.1.0"
   }
 }
diff --git a/dev/import-tool/src/index.ts b/dev/import-tool/src/index.ts
@@ -20,11 +20,29 @@ import serverClientPlugin, {
   selectWorkspace
 } from '@hcengineering/server-client'
 import { program } from 'commander'
-import { importNotion } from './notion/notion'
 import { setMetadata } from '@hcengineering/platform'
-import { FrontFileUploader, type FileUploader } from './importer/uploader'
-import { ClickupImporter } from './clickup/clickup'
-import { UnifiedFormatImporter } from './huly/unified'
+import {
+  UnifiedFormatImporter,
+  ClickupImporter,
+  importNotion,
+  FrontFileUploader,
+  type FileUploader,
+  type Logger
+} from '@hcengineering/importer'
+
+class ConsoleLogger implements Logger {
+  log (msg: string, data?: any): void {
+    console.log(msg, data)
+  }
+
+  warn (msg: string, data?: any): void {
+    console.warn(msg, data)
+  }
+
+  error (msg: string, data?: any): void {
+    console.error(msg, data)
+  }
+}
 
 /**
  * @public
@@ -127,7 +145,7 @@ export function importTool (): void {
     .action(async (file: string, cmd) => {
       const { workspace, user, password } = cmd
       await authorize(user, password, workspace, async (client, uploader) => {
-        const importer = new ClickupImporter(client, uploader)
+        const importer = new ClickupImporter(client, uploader, new ConsoleLogger())
         await importer.importClickUpTasks(file)
       })
     })
@@ -142,7 +160,7 @@ export function importTool (): void {
     .action(async (dir: string, cmd) => {
       const { workspace, user, password } = cmd
       await authorize(user, password, workspace, async (client, uploader) => {
-        const importer = new UnifiedFormatImporter(client, uploader)
+        const importer = new UnifiedFormatImporter(client, uploader, new ConsoleLogger())
         await importer.importFolder(dir)
       })
     })

diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts
@@ -60,7 +60,7 @@ import serverClientPlugin, {
 } from '@hcengineering/server-client'
 import { createBackupPipeline, getConfig } from '@hcengineering/server-pipeline'
 import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token'
-import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool'
+import { FileModelLogger } from '@hcengineering/server-tool'
 import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service'
 import path from 'path'
 
@@ -173,11 +173,6 @@ export function devTool (
     return elasticUrl
   }
 
-  const initScriptUrl = process.env.INIT_SCRIPT_URL
-  if (initScriptUrl !== undefined) {
-    setMetadata(toolPlugin.metadata.InitScriptURL, initScriptUrl)
-  }
-
   setMetadata(accountPlugin.metadata.Transactors, transactorUrl)
   setMetadata(serverClientPlugin.metadata.Endpoint, accountsUrl)
   setMetadata(serverToken.metadata.Secret, serverSecret)

diff --git a/packages/importer/.eslintrc.js b/packages/importer/.eslintrc.js
@@ -0,0 +1,7 @@
+module.exports = {
+  extends: ['./node_modules/@hcengineering/platform-rig/profiles/default/eslint.config.json'],
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: './tsconfig.json'
+  }
+}
diff --git a/packages/importer/.npmignore b/packages/importer/.npmignore
@@ -0,0 +1,4 @@
+*
+!/lib/**
+!CHANGELOG.md
+/lib/**/__tests__/
diff --git a/packages/importer/jest.config.js b/packages/importer/jest.config.js
@@ -0,0 +1,7 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'],
+  roots: ["./src"],
+  coverageReporters: ["text-summary", "html"]
+}
diff --git a/packages/importer/package.json b/packages/importer/package.json
@@ -0,0 +1,68 @@
+{
+  "name": "@hcengineering/importer",
+  "version": "0.6.1",
+  "main": "lib/index.js",
+  "svelte": "src/index.ts",
+  "types": "types/index.d.ts",
+  "files": [
+    "lib/**/*",
+    "types/**/*",
+    "tsconfig.json"
+  ],
+  "author": "Anticrm Platform Contributors",
+  "license": "EPL-2.0",
+  "scripts": {
+    "build": "compile",
+    "build:watch": "compile",
+    "test": "jest --passWithNoTests --silent",
+    "format": "format src",
+    "_phase:build": "compile transpile src",
+    "_phase:test": "jest --passWithNoTests --silent",
+    "_phase:format": "format src",
+    "_phase:validate": "compile validate"
+  },
+  "devDependencies": {
+    "@hcengineering/platform-rig": "^0.6.0",
+    "@typescript-eslint/eslint-plugin": "^6.11.0",
+    "@typescript-eslint/parser": "^6.11.0",
+    "eslint": "^8.54.0",
+    "eslint-config-standard-with-typescript": "^40.0.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-n": "^15.4.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "prettier": "^3.1.0",
+    "typescript": "^5.3.3",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.1.1",
+    "@types/jest": "^29.5.5",
+    "@types/node": "~20.11.16",
+    "@types/mime-types": "~2.1.1",
+    "@types/csvtojson": "^2.0.0",
+    "@types/js-yaml": "^4.0.9"
+  },
+  "dependencies": {
+    "@hcengineering/attachment": "^0.6.14",
+    "@hcengineering/chunter": "^0.6.20",
+    "@hcengineering/collaboration": "^0.6.0",
+    "@hcengineering/contact": "^0.6.24",
+    "@hcengineering/core": "^0.6.32",
+    "@hcengineering/document": "^0.6.0",
+    "@hcengineering/model-attachment": "^0.6.0",
+    "@hcengineering/model-core": "^0.6.0",
+    "@hcengineering/platform": "^0.6.11",
+    "@hcengineering/rank": "^0.6.4",
+    "@hcengineering/server-core": "^0.6.1",
+    "@hcengineering/task": "^0.6.20",
+    "@hcengineering/text": "^0.6.5",
+    "@hcengineering/tracker": "^0.6.24",
+    "@hcengineering/view": "^0.6.13",
+    "commander": "^8.1.0",
+    "mime-types": "~2.1.34",
+    "csvtojson": "^2.0.10",
+    "js-yaml": "^4.1.0"
+  },
+  "repository": "https://github.com/hcengineering/platform",
+  "publishConfig": {
+    "registry": "https://npm.pkg.github.com"
+  }
+}