Merge pull request #151 from kbase/develop

Develop -> Master

.github/workflows/build_prodrc_pr.yaml

@@ -0,0 +1,28 @@
+---
+name: Build Prod RC Image
+'on':
+  pull_request:
+    branches:
+    - master
+    - main
+    types:
+    - opened
+    - synchronize
+    - ready_for_review
+jobs:
+  docker_build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out GitHub Repo
+      if: github.event.pull_request.draft == false
+      with:
+        ref: "${{ github.event.pull_request.head.sha }}"
+      uses: actions/checkout@v2
+    - name: Build and Push to Packages
+      if: github.event.pull_request.draft == false
+      env:
+        PR: "${{ github.event.pull_request.number }}"
+        SHA: "${{ github.event.pull_request.head.sha }}"
+        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
+        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
+      run: "./.github/workflows/scripts/build_prodrc_pr.sh\n"

.github/workflows/build_test_pr.yaml

@@ -0,0 +1,27 @@
+---
+name: Build Test Image
+'on':
+  pull_request:
+    branches:
+    - develop
+    types:
+    - opened
+    - synchronize
+    - ready_for_review
+jobs:
+  docker_build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out GitHub Repo
+      if: github.event.pull_request.draft == false
+      with:
+        ref: "${{ github.event.pull_request.head.sha }}"
+      uses: actions/checkout@v2
+    - name: Build and Push to Packages
+      if: github.event.pull_request.draft == false
+      env:
+        PR: "${{ github.event.pull_request.number }}"
+        SHA: "${{ github.event.pull_request.head.sha }}"
+        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
+        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
+      run: "./.github/workflows/scripts/build_test_pr.sh\n"

.github/workflows/run_tests.yml

@@ -13,10 +13,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.6
+    - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6
+        python-version: 3.9
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/scripts/build_prodrc_pr.sh

@@ -0,0 +1,16 @@
+#! /usr/bin/env bash
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io
+docker build --build-arg BUILD_DATE="$DATE" \
+             --build-arg COMMIT="$COMMIT" \
+             --build-arg BRANCH="$GITHUB_HEAD_REF" \
+             --build-arg PULL_REQUEST="$PR" \
+             --label us.kbase.vcs-pull-req="$PR" \
+             -t ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" .
+docker push ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR"

.github/workflows/scripts/build_test_pr.sh

@@ -0,0 +1,17 @@
+#! /usr/bin/env bash
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo $(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')"-develop")
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+echo $DOCKER_TOKEN | docker login ghcr.io -u $DOCKER_ACTOR --password-stdin
+docker build --build-arg BUILD_DATE="$DATE" \
+             --build-arg COMMIT="$COMMIT" \
+             --build-arg BRANCH="$GITHUB_HEAD_REF" \
+             --build-arg PULL_REQUEST="$PR" \
+             --label us.kbase.vcs-pull-req="$PR" \
+             -t ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" .
+docker push ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR"
+

.github/workflows/scripts/deploy_tag.sh

@@ -0,0 +1,34 @@
+#! /usr/bin/env bash
+
+# Usage: ./deploy_tag.sh -e TARGET -o ORG -r REPO -s DEV_PROD -t IMAGE_TAG
+#
+# Example 1: ./deploy_tag.sh -o "kbase" -r "narrative-traefiker" -s "dev" -t "pr-9001" -e "ci"
+# Example 2: ./deploy_tag.sh -o "kbase" -r "narrative" -s "prod" -t "latest" -e "next"
+#
+# Where:
+#   -o ORG is the organization (`kbase`, `kbaseapps`, etc.)
+#   -r REPO is the repository (e.g. `narrative`)
+#   -s DEV_PROD determines whether to pull the development {APPNAME}-develop or production {APPNAME} image.
+#   -t IMAGE_TAG is the *current* Docker image tag, typically `pr-#` or `latest`
+#   -e TARGET is one of: `appdsshev`, `ci`, or `next`
+#
+# Be sure to set $TOKEN first!
+# See: https://docs.github.com/en/packages/getting-started-with-github-container-registry/migrating-to-github-container-registry-for-docker-images#authenticating-with-the-container-registry
+
+
+while getopts e:o:r:s:t: option
+  do
+   case "${option}"
+    in
+      e) TARGET=${OPTARG};;
+      o) ORG=${OPTARG};;
+      r) REPO=${OPTARG};;
+      s) DEV_PROD=${OPTARG};;
+      t) IMAGE_TAG=${OPTARG};;
+    esac
+done
+
+curl -H "Authorization: token $TOKEN" \
+    -H 'Accept: application/vnd.github.everest-preview+json' \
+    "https://api.github.com/repos/$ORG/$REPO/dispatches" \
+    -d '{"event_type":"Tag '"$DEV_PROD"' '"$IMAGE_TAG"' for '"$TARGET"'", "client_payload": {"image_tag": "'"$IMAGE_TAG"'","target": "'"$TARGET"'","dev_prod": "'"$DEV_PROD"'"}}'

.github/workflows/scripts/tag_environments.sh

@@ -0,0 +1,22 @@
+
+#! /usr/bin/env bash
+# Add vars for PR & environments to yaml, as called from external script
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+if [ $DEV_PROD = "dev" ] || [ $DEV_PROD = "develop" ]
+then
+  IMAGE=$MY_APP"-develop"
+else
+  IMAGE=$MY_APP
+fi
+
+echo "Dev or Prod:" $DEV_PROD
+docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io
+docker pull ghcr.io/"$MY_ORG"/"$IMAGE":"$IMAGE_TAG"
+docker tag ghcr.io/"$MY_ORG"/"$IMAGE":"$IMAGE_TAG" ghcr.io/"$MY_ORG"/"$IMAGE":"$TARGET"
+docker push ghcr.io/"$MY_ORG"/"$IMAGE":"$TARGET"

.github/workflows/scripts/tag_prod_latest.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io
+docker pull ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR"
+docker tag ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" ghcr.io/"$MY_ORG"/"$MY_APP":"latest"
+docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest"

.github/workflows/scripts/tag_test_latest.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+export MY_ORG=$(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $1}')
+export MY_APP=$(echo $(echo "${GITHUB_REPOSITORY}" | awk -F / '{print $2}')"-develop")
+export DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+export COMMIT=$(echo "$SHA" | cut -c -7)
+
+docker login -u "$DOCKER_ACTOR" -p "$DOCKER_TOKEN" ghcr.io
+docker pull ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR"
+docker tag ghcr.io/"$MY_ORG"/"$MY_APP":"pr-""$PR" ghcr.io/"$MY_ORG"/"$MY_APP":"latest"
+docker push ghcr.io/"$MY_ORG"/"$MY_APP":"latest"

.github/workflows/tag_environments.yaml

@@ -0,0 +1,19 @@
+---
+name: Tag Image For Deploy
+'on':
+  repository_dispatch
+jobs:
+  tag_environments:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out GitHub Repo
+      uses: actions/checkout@v2
+    - name: Tag Deploy Environments  
+      env:
+        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
+        DOCKER_TOKEN: ${{ secrets.GHCR_TOKEN  }}
+        IMAGE_TAG: ${{ github.event.client_payload.image_tag }}
+        SHA: ${{ github.event.pull_request.head.sha }}
+        TARGET: ${{ github.event.client_payload.target }}
+        DEV_PROD: ${{ github.event.client_payload.dev_prod }}
+      run: './.github/workflows/scripts/tag_environments.sh'

.github/workflows/tag_prod_latest.yaml

@@ -0,0 +1,27 @@
+---
+name: Tag Prod Latest
+'on':
+  pull_request:
+    branches:
+    - master
+    - main
+    types:
+    - closed
+jobs:
+  docker_tag:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out GitHub Repo
+      if: github.event_name == 'pull_request' && github.event.action == 'closed' &&
+        github.event.pull_request.merged == true
+      with:
+        ref: "${{ github.event.pull_request.head.sha }}"
+      uses: actions/checkout@v2
+    - name: Build and Push to Packages
+      if: github.event.pull_request.draft == false
+      env:
+        PR: "${{ github.event.pull_request.number }}"
+        SHA: "${{ github.event.pull_request.head.sha }}"
+        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
+        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
+      run: "./.github/workflows/scripts/tag_prod_latest.sh\n"

.github/workflows/tag_test_latest.yaml

@@ -0,0 +1,26 @@
+---
+name: Tag Latest Test Image
+'on':
+  pull_request:
+    branches:
+    - develop
+    types:
+    - closed
+jobs:
+  docker_tag:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out GitHub Repo
+      if: github.event_name == 'pull_request' && github.event.action == 'closed' &&
+        github.event.pull_request.merged == true
+      with:
+        ref: "${{ github.event.pull_request.head.sha }}"
+      uses: actions/checkout@v2
+    - name: Build and Push to Packages
+      if: github.event.pull_request.draft == false
+      env:
+        PR: "${{ github.event.pull_request.number }}"
+        SHA: "${{ github.event.pull_request.head.sha }}"
+        DOCKER_ACTOR: "${{ secrets.GHCR_USERNAME }}"
+        DOCKER_TOKEN: "${{ secrets.GHCR_TOKEN }}"
+      run: "./.github/workflows/scripts/tag_test_latest.sh\n"

.gitignore

@@ -103,4 +103,5 @@ ENV/
 data/
 .DS_Store
 .virtualenvs/
-test.env
+test.env
+run_tests_single.sh

Dockerfile

@@ -1,10 +1,11 @@
-FROM quay.io/kbase/python:3.6-slim-stretch
+FROM python:3.9-slim-buster
 # -----------------------------------------
 RUN mkdir -p /kb/deployment/lib
 RUN apt-get update && apt-get install -y --no-install-recommends apt-utils
 RUN apt-get install -y zip && \
     apt-get install -y unzip && \
-    apt-get install -y bzip2
+    apt-get install -y bzip2 && \
+    apt-get install -y libmagic-dev
 
 
 RUN apt-get install -y htop wget

Pipfile

@@ -0,0 +1,29 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+flake8 = "*"
+
+[packages]
+uvloop = "==0.15.3"
+aiohttp = "==3.7.4"
+pytest = "==6.2.4"
+aiofiles = "==0.3.1"
+pytest-aiohttp = "==0.1.3"
+coverage = "==4.4.2"
+pytest-cov = "==2.5.1"
+hypothesis = "==6.14.5"
+globus-sdk = "==1.6.1"
+python-dotenv = "==0.14.0"
+aiohttp_cors = "==0.7.0"
+frozendict = "==2.0.3"
+pandas = "==1.3.2"
+xlrd = "==2.0.1"
+openpyxl = "==3.0.7"
+defusedxml = "==0.7.1"
+python-magic = "==0.4.24"
+
+[requires]
+python_version = "3.9"

Pipfile.README.md

@@ -0,0 +1,5 @@
+Note that requirements*.txt is currently the source of truth for which modules and versions are
+required for this software. The Pipfile is provided as a convenience for users of tools that
+consume one.
+
+You should verify that it is equivalent to requirements*.txt before using it.