Merge branch 'main' of github.com:specklesystems/speckle-server into …

…gergo/multiRegionFileImport

.circleci/config.yml

@@ -16,7 +16,7 @@ workflows:
                 - main
                 - hotfix*
 
-      - test-server:
+      - test-server: &test-server-job-definition
           context:
             - speckle-server-licensing
             - stripe-integration
@@ -32,6 +32,8 @@ workflows:
           requires:
             - docker-publish-postgres-container
 
+      - test-server-multiregion: *test-server-job-definition
+
       - test-frontend-2:
           filters: *filters-allow-all
 
@@ -190,6 +192,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-frontend:
@@ -205,6 +208,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-frontend-2:
@@ -220,6 +224,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-webhooks:
@@ -235,6 +240,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-file-imports:
@@ -250,6 +256,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-previews:
@@ -265,6 +272,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-test-container:
@@ -280,6 +288,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-postgres-container:
@@ -301,6 +310,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - docker-publish-docker-compose-ingress:
@@ -316,6 +326,7 @@ workflows:
             - test-objectsender
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-preview-service
 
       - publish-helm-chart:
@@ -356,6 +367,7 @@ workflows:
             - get-version
             - test-server
             - test-server-no-ff
+            - test-server-multiregion
             - test-ui-components
             - test-frontend-2
             - test-viewer
@@ -573,6 +585,48 @@ jobs:
       FF_GATEKEEPER_MODULE_ENABLED: 'false'
       FF_BILLING_INTEGRATION_ENABLED: 'false'
 
+  test-server-multiregion:
+    <<: *test-server-job
+    docker:
+      - image: cimg/node:18.19.0
+      - image: cimg/redis:7.2.4
+      - image: 'speckle/speckle-postgres'
+        environment:
+          POSTGRES_DB: speckle2_test
+          POSTGRES_PASSWORD: speckle
+          POSTGRES_USER: speckle
+        command: -c 'max_connections=1000'
+      - image: 'speckle/speckle-postgres'
+        environment:
+          POSTGRES_DB: speckle2_test
+          POSTGRES_PASSWORD: speckle
+          POSTGRES_USER: speckle
+        command: -c 'max_connections=1000' -c 'port=5433'
+      - image: 'minio/minio'
+        command: server /data --console-address ":9001"
+    environment:
+      # Same as test-server:
+      NODE_ENV: test
+      DATABASE_URL: 'postgres://speckle:[email protected]:5432/speckle2_test'
+      PGDATABASE: speckle2_test
+      POSTGRES_MAX_CONNECTIONS_SERVER: 20
+      PGUSER: speckle
+      SESSION_SECRET: 'keyboard cat'
+      STRATEGY_LOCAL: 'true'
+      CANONICAL_URL: 'http://127.0.0.1:3000'
+      S3_ENDPOINT: 'http://127.0.0.1:9000'
+      S3_ACCESS_KEY: 'minioadmin'
+      S3_SECRET_KEY: 'minioadmin'
+      S3_BUCKET: 'speckle-server'
+      S3_CREATE_BUCKET: 'true'
+      REDIS_URL: 'redis://127.0.0.1:6379'
+      S3_REGION: '' # optional, defaults to 'us-east-1'
+      AUTOMATE_ENCRYPTION_KEYS_PATH: 'test/assets/automate/encryptionKeys.json'
+      FF_BILLING_INTEGRATION_ENABLED: 'true'
+      # These are the only 2 different env keys:
+      MULTI_REGION_CONFIG_PATH: '../../.circleci/multiregion.test-ci.json'
+      RUN_TESTS_IN_MULTIREGION_MODE: true
+
   test-frontend-2:
     docker: &docker-node-browsers-image
       - image: cimg/node:18.19.0-browsers

.circleci/multiregion.test-ci.json

@@ -0,0 +1,14 @@
+{
+  "main": {
+    "postgres": {
+      "connectionUri": "postgresql://speckle:[email protected]:5432/speckle2_test"
+    }
+  },
+  "regions": {
+    "region1": {
+      "postgres": {
+        "connectionUri": "postgresql://speckle:[email protected]:5433/speckle2_test"
+      }
+    }
+  }
+}

.gitignore

@@ -74,4 +74,5 @@ redis-data/
 .tshy-build
 
 # Server
-multiregion.json
+multiregion.json
+multiregion.test.json

package.json

@@ -21,6 +21,7 @@
     "dev:docker": "docker compose -f ./docker-compose-deps.yml",
     "dev:docker:up": "docker compose -f ./docker-compose-deps.yml up -d",
     "dev:docker:down": "docker compose -f ./docker-compose-deps.yml down",
+    "dev:docker:restart": "yarn dev:docker:down && yarn dev:docker:up",
     "dev:kind:up": "ctlptl apply --filename ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:down": "ctlptl delete -f ./.circleci/deployment/cluster-config.yaml",
     "dev:kind:helm:up": "yarn dev:kind:up && tilt up --file ./.circleci/deployment/Tiltfile.helm --context kind-speckle-server",

packages/frontend-2/components/projects/AddDialog.vue

@@ -118,7 +118,8 @@ const emit = defineEmits<{
 const isWorkspacesEnabled = useIsWorkspacesEnabled()
 const createProject = useCreateProject()
 const router = useRouter()
-const { handleSubmit, meta } = useForm<FormValues>()
+const logger = useLogger()
+const { handleSubmit, meta, isSubmitting } = useForm<FormValues>()
 const { result: workspaceResult } = useQuery(projectWorkspaceSelectQuery, null, () => ({
   enabled: isWorkspacesEnabled.value
 }))
@@ -127,45 +128,62 @@ const visibility = ref(ProjectVisibility.Unlisted)
 const selectedWorkspace = ref<ProjectsAddDialog_WorkspaceFragment>()
 const showConfirmDialog = ref(false)
 const confirmActionType = ref<'navigate' | 'close' | null>(null)
+const isClosing = ref(false)
 
 const open = defineModel<boolean>('open', { required: true })
 
 const mp = useMixpanel()
 
 const onSubmit = handleSubmit(async (values) => {
-  const workspaceId = props.workspaceId || selectedWorkspace.value?.id
-
-  await createProject({
-    name: values.name,
-    description: values.description,
-    visibility: visibility.value,
-    ...(workspaceId ? { workspaceId } : {})
-  })
-  emit('created')
-  mp.track('Stream Action', {
-    type: 'action',
-    name: 'create',
-    // eslint-disable-next-line camelcase
-    workspace_id: props.workspaceId
-  })
-  open.value = false
+  if (isClosing.value) return // Prevent submission while closing
+
+  try {
+    isClosing.value = true
+    const workspaceId = props.workspaceId || selectedWorkspace.value?.id
+
+    await createProject({
+      name: values.name,
+      description: values.description,
+      visibility: visibility.value,
+      ...(workspaceId ? { workspaceId } : {})
+    })
+    emit('created')
+    mp.track('Stream Action', {
+      type: 'action',
+      name: 'create',
+      // eslint-disable-next-line camelcase
+      workspace_id: props.workspaceId
+    })
+    open.value = false
+  } catch (error) {
+    isClosing.value = false
+    logger.error('Failed to create project:', error)
+  }
 })
 
 const workspaces = computed(
   () => workspaceResult.value?.activeUser?.workspaces.items ?? []
 )
 const hasWorkspaces = computed(() => workspaces.value.length > 0)
+
 const dialogButtons = computed((): LayoutDialogButton[] => {
+  const isDisabled = isSubmitting.value || isClosing.value
+
   return [
     {
       text: 'Cancel',
-      props: { color: 'outline' },
+      props: {
+        color: 'outline',
+        disabled: isDisabled
+      },
       onClick: confirmCancel
     },
     {
       text: 'Create',
       props: {
-        submit: true
+        submit: true,
+        loading: isDisabled,
+        disabled: isDisabled
       },
       onClick: onSubmit
     }
@@ -206,6 +224,7 @@ const handleConfirmAction = () => {
 watch(open, (newVal, oldVal) => {
   if (newVal && !oldVal) {
     selectedWorkspace.value = undefined
+    isClosing.value = false
   }
 })
 </script>

packages/server/.env-example

@@ -147,3 +147,8 @@ OIDC_CLIENT_SECRET="gLb9IEutYQ0npyvA8iHxPsObY3duGB0w"
 # OTEL_TRACE_URL=""
 # OTEL_TRACE_KEY=""
 # OTEL_TRACE_VALUE=""
+
+############################################################
+# Multi region settings
+############################################################
+MULTI_REGION_CONFIG_PATH="multiregion.json"

packages/server/.env.test-example

@@ -4,4 +4,6 @@
 
 PORT=0
 POSTGRES_URL=postgres://speckle:[email protected]/speckle2_test
-POSTGRES_USER=''
+POSTGRES_USER=''
+MULTI_REGION_CONFIG_PATH="multiregion.test.json"
+#RUN_TESTS_IN_MULTIREGION_MODE=true

packages/server/.mocharc.js

@@ -14,7 +14,7 @@ const ignore = [
 /** @type {import("mocha").MochaOptions} */
 const config = {
   spec: ['modules/**/*.spec.js', 'modules/**/*.spec.ts', 'logging/**/*.spec.js'],
-  require: ['ts-node/register', 'test/hooks.js'],
+  require: ['ts-node/register', 'test/hooks.ts'],
   ...(ignore.length ? { ignore } : {}),
   slow: 0,
   timeout: '150000',

packages/server/assets/workspacesCore/typedefs/workspaces.graphql

@@ -147,6 +147,7 @@ type WorkspaceMutations {
   # We are, for the moment, doing the check in the resolver
   deleteDomain(input: WorkspaceDomainDeleteInput!): Workspace!
     @hasScope(scope: "workspace:update")
+  deleteSsoProvider(workspaceId: String!): Boolean!
   invites: WorkspaceInviteMutations!
   projects: WorkspaceProjectMutations! @hasServerRole(role: SERVER_USER)
 }

packages/server/modules/cli/commands/db/migrate/latest.ts

@@ -1,13 +1,28 @@
 import knex from '@/db/knex'
 import { logger } from '@/logging/logging'
+import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
+import { mochaHooks } from '@/test/hooks'
 import { CommandModule } from 'yargs'
 
 const command: CommandModule = {
   command: 'latest',
   describe: 'Run all migrations that have not yet been run',
   async handler() {
     logger.info('Running latest migration...')
-    await knex.migrate.latest()
+
+    // In tests we want different logic - just run beforeAll
+    if (isTestEnv()) {
+      // Run before hooks, to properly initialize everything
+      await (mochaHooks.beforeAll as () => Promise<void>)()
+    } else {
+      const regionDbs = await getRegisteredRegionClients()
+      const dbs = [knex, ...Object.values(regionDbs)]
+      for (const db of dbs) {
+        await db.migrate.latest()
+      }
+    }
+
     logger.info('Completed running migration')
   }
 }

packages/server/modules/cli/commands/db/migrate/rollback.ts

@@ -1,13 +1,30 @@
 import knex from '@/db/knex'
 import { logger } from '@/logging/logging'
+import { getRegisteredRegionClients } from '@/modules/multiregion/dbSelector'
+import { isTestEnv } from '@/modules/shared/helpers/envHelper'
+import { mochaHooks, resetPubSubFactory } from '@/test/hooks'
 import { CommandModule } from 'yargs'
 
 const command: CommandModule = {
   command: 'rollback',
   describe: 'Roll back all migrations',
   async handler() {
     logger.info('Rolling back migrations...')
-    await knex.migrate.rollback(undefined, true)
+
+    if (isTestEnv()) {
+      // Run before hooks, to properly initialize everything first
+      await (mochaHooks.beforeAll as () => Promise<void>)()
+    }
+
+    const regionDbs = await getRegisteredRegionClients()
+    const dbs = [knex, ...Object.values(regionDbs)]
+
+    for (const db of dbs) {
+      const resetPubSub = resetPubSubFactory({ db })
+      await resetPubSub()
+      await db.migrate.rollback(undefined, true)
+    }
+
     logger.info('Completed rolling back migrations')
   }
 }

packages/server/modules/comments/tests/comments.spec.js

@@ -1290,7 +1290,7 @@ describe('Comments @comments', () => {
 
       before(async () => {
         // Truncate comments
-        truncateTables([Comments.name])
+        await truncateTables([Comments.name])
 
         // Create a single comment with a blob
         const createCommentResult = await createComment({