# This is a combination of 2 commits.

# This is the 1st commit message:

Build on GitLab

# The commit message #2 will be skipped:

# fixup! Build on GitLab

explorer/.gitignore

@@ -52,3 +52,4 @@ _templates
 /files/anvil-catalog/out/
 /files/ncpi-catalog/out/
 /files/ncpi-catalog-dug/out/
+/build.tar.bz2

explorer/.gitlab/.gitignore

@@ -0,0 +1,2 @@
+/*.js
+/*.js.map

explorer/.gitlab/Dockerfile

@@ -0,0 +1,3 @@
+FROM node:16.15.1-bullseye
+
+RUN apt-get update && apt-get install -y curl

explorer/.gitlab/README.md

@@ -0,0 +1,39 @@
+The files in this directory define how GitLab builds Data Browser
+distributions and uploads them to the GitLab package registry. The Azul
+pipeline will download each distribution and deploy it to the S3 bucket
+backing the Data Browser CloudFront distribution.
+
+There is generally one instance of the Data Browser per Azul atlas. An atlas
+is a set of releases. Each release is an Azul catalog. There could be one or
+more atlases hosted in an Azul deployment. Since we don't want to hard-code
+the atlases here, or duplicate substantial amounts of GitLab pipeline YAML,
+we'd like to dynamically generate the GitLab jobs that produce the
+distributions. That's made possible in GitLab via the parent-child pipeline
+mechanism.
+
+`ci.yaml` defines the parent pipeline. The first job in the parent pipeline
+builds the image to be used for all subsequent jobs in the pipeline. This
+image is defined in the `Dockerfile`. The first job uses the Azul runner
+image, a minimal image containing a `docker` client. The Azul runner image is
+pushed manually when the Azul project is first set up in a GitLab instance.
+The GitLab job token that is issued to pipelines running for this project
+must be granted access to the pull from the registry for the Azul project, so
+that the runner can pull the Azul runner image for the first job. The next
+job in the parent pipeline generates the GitLab pipeline YAML for the child
+pipeline. The second job triggers a child pipeline using the dynamically
+generated configuration. The generation is done by a NodeJS script written in 
+Typescript, not because Typescript is particularly good at generating YAML but 
+simply because we need NodeJS for building the Data Browser distribution. We 
+didn't want to pull in yet another scripting language into the image.
+
+The child pipeline contains two stages: one stage builds the distributions, the
+other stage uploads them to the package registry. There is one job per atlas in
+each of these two stages.
+
+The `deployments` directory contains shell scripts defining the environment
+variables for the build. There is one file per deployment and atlas. The
+deployment is fixed as a job variable in the *Settings*, *CI/CD*, *Variables*
+section of the GitLab project. The name of the variable is
+`AZUL_DEPLOYMENT_STAGE`. In other words, you pick the Azul deployment by
+configuring the job variable, and the pipeline builds a distribution per
+atlas in that deployment.

explorer/.gitlab/ci.child.yml.template.ts

@@ -0,0 +1,78 @@
+import * as YAML from 'yaml';
+import * as fs from 'fs';
+
+let stage = process.env.AZUL_DEPLOYMENT_STAGE;
+
+let atlases = fs.readdirSync(
+    `deployments/${stage}`,
+    {withFileTypes: true}
+).filter(entry => entry.isDirectory())
+    .map(entry => entry.name);
+
+console.log(
+    YAML.stringify(
+        {
+            'stages': [
+                'build',
+                'publish'
+            ],
+            '.on_branch_push': {
+                'rules': [
+                    {
+                        'if': '$CI_PIPELINE_SOURCE == "parent_pipeline"'
+                    }
+                ]
+            },
+            ...Object.fromEntries(atlases.map(atlas => Object.entries({
+                [`.${atlas}_base`]: {
+                    'image': '$DOCKER_IMAGE:$DOCKER_TAG',
+                    'extends': [
+                        '.on_branch_push'
+                    ],
+                    'dependencies': [],
+                    'before_script': [
+                        'cd explorer',
+                        'source .gitlab/environment.sh',
+                        `source .gitlab/deployments/$AZUL_DEPLOYMENT_STAGE/${atlas}/environment.sh`
+                    ]
+                },
+                [`${atlas}_build`]: {
+                    'extends': `.${atlas}_base`,
+                    'stage': 'build',
+                    'script': [
+                        '(cd files && npm ci)',
+                        'npm ci',
+                        'rm -rf out',
+                        'npm run build:anvil-cmg',
+                        [
+                            'tar -cvj',
+                            '-f .gitlab/distribution.tar.bz2',
+                            '--transform s#^out/explore#explore/${data_browser_host_path}#',
+                            '--show-transformed',
+                            'out'
+                        ].join(' ')
+                    ],
+                    'artifacts': {
+                        'paths': [
+                            'explorer/dist.tar.bz2'
+                        ]
+                    }
+                },
+                [`${atlas}_publish`]: {
+                    'extends': `.${atlas}_base`,
+                    'stage': 'publish',
+                    'script': [
+                        // PUT /projects/:id/packages/generic/:package_name/:package_version/:file_name?status=:status
+                        'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" ' +
+                        '--upload-file dist.tar.bz2 ' +
+                        '"${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/dist/1.0/' +
+                        `dist-${stage}-${atlas}.tar.bz2"`
+                    ],
+                    'dependencies': [
+                        `${atlas}_build`
+                    ]
+                }
+            })).flat())
+        }
+    )
+)

explorer/.gitlab/ci.yaml

@@ -0,0 +1,43 @@
+stages:
+  - build_image
+  - generate_child_yaml
+  - trigger_child
+
+variables:
+  DOCKER_IMAGE: $CI_REGISTRY_IMAGE
+  DOCKER_TAG: $CI_PIPELINE_ID
+
+.on_branch_push:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH
+
+build_image:
+  extends: .on_branch_push
+  stage: build_image
+  rules:
+    - if: $CI_COMMIT_BRANCH
+  script:
+    - cd explorer/.gitlab
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY_IMAGE
+    - docker build --tag $DOCKER_IMAGE:$DOCKER_TAG .
+    - docker push $DOCKER_IMAGE:$DOCKER_TAG
+
+generate_child_yaml:
+  extends: .on_branch_push
+  stage: generate_child_yaml
+  image: $DOCKER_IMAGE:$DOCKER_TAG
+  script:
+    - cd explorer/.gitlab
+    - npm ci
+    - npx tsc
+    - node ci.child.yml.template.js > ci.child.yml
+  artifacts:
+    paths:
+      - explorer/.gitlab/ci.child.yml
+
+trigger_child:
+  stage: trigger_child
+  trigger:
+    include:
+      - artifact: explorer/.gitlab/ci.child.yml
+        job: generate_child_yaml

explorer/.gitlab/deployments/anvildev/anvil/environment.sh

@@ -0,0 +1 @@
+export data_browser_build_env=dev

explorer/.gitlab/deployments/anvilprod/anvil/environment.sh

@@ -0,0 +1 @@
+export data_browser_build_env=prod

explorer/.gitlab/environment.sh

@@ -0,0 +1,4 @@
+export GATSBY_GTM_ID=GTM-M2J5NTJ
+export GATSBY_GTM_AUTH=CzDpc0__fhyqfREDehPK8Q
+export GATSBY_ENV_NAME=env-83
+export data_browser_host_path=anvil-cmg

explorer/.gitlab/package.json

@@ -0,0 +1,10 @@
+{
+    "private": "true",
+    "name": "dot-gitlab",
+    "version": "1.0.0",
+    "dependencies": {
+        "@types/node": "^18.15.11",
+        "typescript": "^5.0.4",
+        "yaml": "^2.2.0"
+    }
+}

explorer/.gitlab/tsconfig.json

@@ -0,0 +1,14 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "target": "es2020",
+        "sourceMap": true,
+        "outDir": "."
+    },
+    "exclude": [
+        "node_modules"
+    ],
+    "include": [
+        "*.ts"
+    ]
+}