.gitlab-ci.yml

variables:
  # Configure to use whatever the gitlab OIDC stack defines
  DEPLOYMENT_ROLE_ARN: arn:aws:iam::677474147593:role/gitlab-oidc-xw-batch-deployment
  # For the ECR repo: The role above needs access to it!
  DBT_ECR_AWS_REGION: eu-central-1
  DBT_ECR_AWS_ACCOUNT_ID: 677474147593
  DBT_ECR_APP_NAME: dbt-run
  # TODO: change this on a real prod to "latest" (or change the cdk definition of the fargate task)
  DBT_ECR_IMAGE_TAG: inactive
  # Change pip's cache directory to be inside the project directory since we can
  # only cache local items.
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

default:
  tags:
    # Our shared runner is tagged with "docker", so all jobs need that as a tag to run on that runner
    - docker

  # Most of the jobs need an image with both nodejs and python on it, otherwise the tests won't work
  # Installing one or the other into the official nodejs/python images takes too much time
  # TODO: try to install one or the other locally and cache it?
  image: nikolaik/python-nodejs:python3.10-nodejs18-bullseye

  # Pip's cache doesn't store the python packages
  # https://pip.pypa.io/en/stable/reference/pip_install/#caching
  #
  # If you want to also cache the installed packages, you have to install
  # them in a virtualenv and cache it as well.
  cache:
    - key:
        files:
          - xw-batch/requirements.txt
          - xw-batch/requirements-dev.txt
      paths:
        - .cache/pip
        - xw-batch/.venv/
    - key:
        files:
          - xw-batch/package-lock.json
        # Needed because only the file content is otherwise used in the hash of the cache and that
        # can result in the same hash as for the gitlab-aws-oidc/package-lock.json (=same content)
        # and so overwrite the cached content.
        prefix: "xw-batch"
      paths:
        - xw-batch/node_modules/
        - .cache/npm


## When to create pipelines
# prevent duplicate pipeline runs by only running the MR Event instead of push-to-branch event.
# Otherwise, it runs on the default branch (=merges) and tags.
# See https://docs.gitlab.com/ee/ci/yaml/workflow.html#workflowrules-templates and
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml
include:
  - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'


## Snippets for re-usability
.activate-xw-batch-environment: &activate-xw-batch-environment
  - cd xw-batch
  - python3 --version  # For debugging
  - echo "Activating environments"
  - source .venv/bin/activate
  - export PATH=${PATH}:$(pwd)/node_modules/.bin # to make the cdk command available
  # Rest is for debugging
  - env
  - ls -la . node_modules/.bin .venv/bin/
  - node --version
  - npm root
  - command -v npx
  - command -v cdk

.assume-deployment-role-script: &assume-deployment-role-script
  - echo "Getting AWS credentials for the deployment"
  # for debugging: If your runner runs in aws, this will already output something sensible!
  - aws sts get-caller-identity || true
  - >
    export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s"
    $(aws sts assume-role-with-web-identity
    --role-arn "${DEPLOYMENT_ROLE_ARN}"
    --role-session-name "GitLabRunner-${CI_PROJECT_ID}-${CI_PIPELINE_ID}"
    --web-identity-token $CI_JOB_JWT_V2
    --duration-seconds 3600
    --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]'
    --output text))
  - echo "Testing aws access"
  # This should now output something or something different from the first call!
  - aws sts get-caller-identity


## List of stages for jobs, and their order of execution
stages:
  - install-packages
  - run-static-checks
  - run-tests
  - deploy-oidc
  - deploy-infra
  - deploy-dbt

## Jobs
install-xw-batch-packages:
  # via caching, all other default jobs will have python stuff installed
  stage: install-packages
  rules:
    - changes:
        - xw-batch/**/*
        - .gitlab-ci.yml
  script:
    - cd xw-batch
    - echo "Creating venv"
    - python3 -m venv .venv
    - source .venv/bin/activate
    - echo "Installing python packages"
    - python3 -m pip install -r requirements.txt
    - python3 -m pip install -r requirements-dev.txt
    - echo "Installing awscli for deployments"
    - python3 -m pip install awscli
    - echo "Installing node packages"
    - npm ci --cache $CI_PROJECT_DIR/.cache/npm --prefer-offline
    - echo "Installation of packages complete."

mypy:
  stage: run-static-checks
  rules:
    - changes:
        - xw-batch/**/*
        - .gitlab-ci.yml

  script:
    - *activate-xw-batch-environment
    - echo "Running mypy..."
    - python3 -m mypy xw_batch tests lambdas glue
    - echo "No mypy issues found."

black:
  stage: run-static-checks
  rules:
    - changes:
        - xw-batch/**/*
        - .gitlab-ci.yml
  script:
    - *activate-xw-batch-environment
    - echo "Running black..."
    - python3 -m black --check --verbose -- xw_batch tests lambdas glue
    - echo "No black issues found."

isort:
  stage: run-static-checks
  rules:
    - changes:
        - xw-batch/**/*
        - .gitlab-ci.yml
  script:
    - *activate-xw-batch-environment
    - echo "Running isort..."
    - python3 -m isort --check-only xw_batch tests lambdas glue
    - echo "No isort issues found."

flake8:
  stage: run-static-checks
  rules:
    - changes:
        - xw-batch/**/*
        - .gitlab-ci.yml
  script:
    - *activate-xw-batch-environment
    - echo "Running flake8..."
    - python3 -m flake8 xw_batch tests lambdas glue
    - echo "No flake8 issues found."

pytest:
  stage: run-tests
  rules:
    - changes:
        - xw-batch/**/*
        - .gitlab-ci.yml
  script:
    - *activate-xw-batch-environment
    - echo "Running pytest..."
    - python3 -m pytest tests
    - echo "No pytest issues found."

deploy-gitlab-oidc-stack:
  stage: deploy-oidc
  rules:
    # Only main has access to the deployment role!
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      changes:
        - gitlab-aws-oidc/**/*
        - .gitlab-ci.yml
  cache:
    - key:
        files:
          - gitlab-aws-oidc/package-lock.json
        # See above with the default cache for explanation
        prefix: "gitlab-aws-oidc"
      paths:
        - gitlab-aws-oidc/node_modules/
        - .cache/npm
  script:
    - echo "Installing cdk"
    - cd gitlab-aws-oidc
    - npm ci --cache $CI_PROJECT_DIR/.cache/npm --prefer-offline
    - export PATH=$PATH:./node_modules/.bin
    - cdk --version
    - echo "Finished installing cdk"
    - echo "Installing awscli for deployments"
    - python3 -m pip install awscli
    - echo "Finished installing awscli"
    - *assume-deployment-role-script
    - echo "Running a deployment of the OIDC stack"
    - cdk synth
    - cdk deploy --ci --debug --require-approval=never --progress=events
    - echo "Deployment complete"

diff-xw-batch-stack:
  stage: deploy-infra
  rules:
    # Only main has access to the deployment role!
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      changes:
        - xw-batch/**/*
        - .gitlab-ci.yml
  script:
    - *activate-xw-batch-environment
    - *assume-deployment-role-script
    - echo "Running a pseudo deployment of the xw-batch stack (just diff)"
    - cdk synth
    # just a diff as the current account is Jans dev account and this would play badly with the actual dev stuff
    - cdk diff dev/XwBatchStack
    - echo "Deployment complete"

deploy-dbt:
  stage: deploy-dbt
  rules:
    # Only main has access to the deployment role!
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      changes:
        - dbt/**/*
        - .gitlab-ci.yml
  cache: []
  image:
    # We want the aws cli v2, which has no way to install it with pip :-(
    name: amazon/aws-cli
    entrypoint: [ "" ]
  before_script:
    - amazon-linux-extras install docker
    - aws --version
    - docker --version
  script:
    # We have aws cli via the docker image and don't need cdk, so no need to activate the venv
    - *assume-deployment-role-script
    - echo "Running a pseudo deployment of dbt using an inactive tag"
    # No activate-xw-batch-environment, so we are still in the repo root
    - cd dbt
    - export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text)
    - export REGISTRY_URL=${DBT_ECR_AWS_ACCOUNT_ID}.dkr.ecr.${DBT_ECR_AWS_REGION}.amazonaws.com
    - export IMG=${REGISTRY_URL}/${DBT_ECR_APP_NAME}:${DBT_ECR_IMAGE_TAG}
    - aws ecr get-login-password --region ${DBT_ECR_AWS_REGION} | docker login --username AWS --password-stdin "${REGISTRY_URL}"
    - docker build -t "${IMG}" .
    - docker push "${IMG}"
    - echo "Deployment complete"