Merge branch 'contextus' into new_master

Sefaria · Jan 16, 2024 · 5e0a77a · 5e0a77a
2 parents 0f6c039 + a790711
commit 5e0a77a
Show file tree

Hide file tree

Showing 125 changed files with 1,224 additions and 1,562 deletions.
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
@@ -9,4 +9,4 @@ community_bridge: # Replace with a single Community Bridge project-name e.g., cl
 liberapay: # Replace with a single Liberapay username
 issuehunt: # Replace with a single IssueHunt username
 otechie: # Replace with a single Otechie username
-custom: https://donate.sefaria.org/en # Replace with a single custom sponsorship URL
+custom: https://sefaria.nationbuilder.com # Replace with a single custom sponsorship URL
diff --git a/.github/workflows/continuous.yaml b/.github/workflows/continuous.yaml
@@ -40,8 +40,8 @@ jobs:
         uses: google-github-actions/auth@v1
         with:
           token_format: 'access_token'
-          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
-          service_account: '${{ secrets.DEV_GKE_SA }}'
+          workload_identity_provider: 'projects/${{ secrets.CONTEXTUS_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
+          service_account: '${{ secrets.CONTEXTUS_GKE_SA }}'
       - name: Login to GCR
         uses: docker/login-action@v1
         with:
@@ -73,8 +73,7 @@ jobs:
         uses: docker/metadata-action@v3
         with:
           images: |
-            gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
-            us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
+            us-east1-docker.pkg.dev/${{ secrets.PROD_GKE_PROJECT }}/containers/contextus-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
           # generate Docker tags based on the following events/attributes
           tags: |
             type=ref,event=branch
@@ -85,8 +84,6 @@ jobs:
       - name: build and push
         uses: docker/build-push-action@v3
         with:
-#         cache-from: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache
-#         cache-to: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache, mode=max
           context: .
           push: true
           build-args: |
@@ -117,8 +114,8 @@ jobs:
         uses: google-github-actions/auth@v1
         with:
           token_format: 'access_token'
-          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
-          service_account: '${{ secrets.DEV_GKE_SA }}'
+          workload_identity_provider: 'projects/${{ secrets.CONTEXTUS_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
+          service_account: '${{ secrets.CONTEXTUS_GKE_SA }}'
       - name: Login to GCR
         uses: docker/login-action@v1
         with:
@@ -150,8 +147,7 @@ jobs:
         uses: docker/metadata-action@v3
         with:
           images: |
-            gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
-            us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
+            us-east1-docker.pkg.dev/${{ secrets.PROD_GKE_PROJECT }}/containers/contextus-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
           # generate Docker tags based on the following events/attributes
           tags: |
             type=ref,event=branch
@@ -165,229 +161,10 @@ jobs:
       - name: build and push
         uses: docker/build-push-action@v3
         with:
-#         cache-from: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache
-#         cache-to: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache,mode=max
           context: .
           push: true
           build-args: |
-            SRC_IMG=us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-web-${{ steps.branch-name.outputs.current_branch }}:sha-${{ steps.get-sha.outputs.sha_short }}
+            SRC_IMG=us-east1-docker.pkg.dev/${{ secrets.PROD_GKE_PROJECT }}/containers/contextus-web-${{ steps.branch-name.outputs.current_branch }}:sha-${{ steps.get-sha.outputs.sha_short }}
           file: ./build/${{ matrix.app }}/Dockerfile
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-#######
-# Below Tests only run on PR
-#######
-  jest-tests:
-    name: "Jest"   # This name is referenced when slacking status
-    runs-on: ubuntu-latest
-    if: ${{ github.event_name == 'pull_request' }}
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v2
-      - name: Install dependencies
-        run: |
-          npm install
-          npm run build-prod
-      - run: ls
-      - run: pwd
-      - name: Run Jest Tests
-        run: npm run jest-gha
-      - name: Handle Jest Test Results
-        run: cat /home/runner/jestResults.json; STATUS=`jq ".numFailedTestSuites" /home/runner/jestResults.json`; exit $STATUS
-        if: ${{ always() }}
-  sandbox-deploy:
-    concurrency:
-      group: dev-mongo
-      cancel-in-progress: false
-    if: ${{ github.event_name == 'pull_request' }}
-    runs-on: ubuntu-latest
-    needs: build-derived
-    permissions:
-      contents: 'read'
-      id-token: 'write'
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v2
-      - id: auth
-        name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v1
-        with:
-          token_format: 'access_token'
-          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
-          service_account: '${{ secrets.DEV_GKE_SA }}'
-      - name: Setup GCloud
-        uses: google-github-actions/setup-gcloud@v0
-        with:
-          project_id: ${{ secrets.DEV_PROJECT }}
-          install_components: 'gke-gcloud-auth-plugin'
-      - name: Get branch name
-        id: branch-raw
-        uses: tj-actions/[email protected]
-      - name: Format branch name
-        id: branch-name
-        run: >-
-          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }}
-          | awk '{print tolower($0)}'
-          | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|'
-          | sed 's/[^a-z0-9\.\-]//g')
-          >> $GITHUB_OUTPUT
-      - name: Set outputs
-        id: get-sha
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: Set up yq
-        uses: frenck/action-setup-yq@v1
-          #      - name: setup helm
-          #        uses: azure/setup-helm@v3
-      - name: Authenticate GHA Runner To Target Cluster
-        uses: google-github-actions/get-gke-credentials@v0
-        with:
-          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
-          location: ${{secrets.DEV_GKE_REGION}}
-          project_id: ${{secrets.DEV_GCP_PROJECT}}
-      - name: Deploy Sandbox
-        run: ./build/ci/sandbox-helm-deploy.sh build/ci/sandbox-values.yaml
-        env:
-          GIT_COMMIT: "${{ steps.get-sha.outputs.sha_short }}"
-          BRANCH: "${{ steps.branch-name.outputs.current_branch }}"
-          PROJECT_ID: "${{ secrets.DEV_PROJECT }}"
-          NAMESPACE: "${{secrets.DEV_SANDBOX_NAMESPACE}}"
-  sandbox-ready:
-    if: ${{ github.event_name == 'pull_request' }}
-    needs: sandbox-deploy
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Set outputs
-        id: get-sha
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: Wait for test sandbox to become available
-        # https://gist.github.com/rgl/f90ff293d56dbb0a1e0f7e7e89a81f42
-        run: ./build/ci/waitForSandbox.bash
-        env:
-          WAIT_DURATION: "3000"
-          GIT_COMMIT: "${{ steps.get-sha.outputs.sha_short }}"
-  pytest-job:
-    concurrency:
-      group: dev-mongo
-      cancel-in-progress: false
-    if: ${{ github.event_name == 'pull_request' }}
-    name: "PyTest"  # This name is referenced when slacking status
-    needs:
-      - sandbox-ready
-    permissions:
-      contents: 'read'
-      id-token: 'write'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - id: auth
-        name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v1
-        with:
-          token_format: 'access_token'
-          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
-          service_account: '${{ secrets.DEV_GKE_SA }}'
-      - name: Setup GCloud
-        uses: google-github-actions/setup-gcloud@v0
-        with:
-          project_id: ${{ secrets.DEV_PROJECT }}
-          install_components: 'gke-gcloud-auth-plugin'
-      - name: Authenticate GHA Runner To Target Cluster
-        uses: google-github-actions/get-gke-credentials@v0
-        with:
-          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
-          location: ${{secrets.DEV_GKE_REGION}}
-          project_id: ${{secrets.DEV_GCP_PROJECT}}
-      - name: Set outputs
-        id: get-sha
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: Get branch name
-        id: branch-raw
-        uses: tj-actions/[email protected]
-      - name: Format branch name
-        id: branch-name
-        run: >-
-          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }}
-          | awk '{print tolower($0)}'
-          | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|'
-          | sed 's/[^a-z0-9\.\-]//g')
-          >> $GITHUB_OUTPUT
-      - name: Start Job
-        run: ./build/ci/createJobFromRollout.sh $GITHUB_RUN_ID $DEPLOY_ENV
-        env:
-          # dependent on GITHUB_RUN_ID, which is implicitly passed in
-          DEPLOY_ENV: sandbox-${{ steps.get-sha.outputs.sha_short }}
-      - name: Wait For Job To Finish
-        run: ./build/ci/waitForCIJob.bash
-        timeout-minutes: 60
-        env:
-          # dependent on GITHUB_RUN_ID, which is implicitly passed in
-          TEST_NAME: pytest
-      - name: Get Logs From Cluster and propogate test result
-        run: "kubectl logs --tail=-1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest; LASTLINE=`kubectl logs --tail=1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest`; STAT=${LASTLINE: -1}; exit $STAT"
-      - name: Cleanup pyTest Pod
-        run: kubectl delete jobs -l ci-run=$GITHUB_RUN_ID,test-name=pytest
-        if: always()
-  ending-notification:
-    runs-on: ubuntu-latest
-    if: ${{ always() && github.event_name == 'pull_request' }}
-    needs:
-      - pytest-job
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-      - run: npm ci
-        working-directory: ./build/notify
-      - run: node notifyEnd.js
-        working-directory: ./build/notify
-        env:
-          SLACK_TEST_SUCCESS_WEBHOOK_URL: ${{secrets.SLACK_TEST_SUCCESS_WEBHOOK_URL}}
-          SLACK_TEST_FAILURE_WEBHOOK_URL: ${{secrets.SLACK_TEST_FAILURE_WEBHOOK_URL}}
-          GITUSER_SLACK_MAP: ${{secrets.GITUSER_SLACK_MAP}}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  sandbox-cleanup:
-    name: "Clean up CI Sandbox"
-    if: ${{ always() && github.event_name == 'pull_request' }}
-    needs:
-      - pytest-job
-    runs-on: ubuntu-latest
-    permissions:
-      contents: 'read'
-      id-token: 'write'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - id: auth
-        name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v1
-        with:
-          token_format: 'access_token'
-          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
-          service_account: '${{ secrets.DEV_GKE_SA }}'
-      - name: Setup GCloud
-        uses: google-github-actions/setup-gcloud@v0
-        with:
-          project_id: ${{ secrets.DEV_PROJECT }}
-          install_components: 'gke-gcloud-auth-plugin'
-      - name: Authenticate GHA Runner To Target Cluster
-        uses: google-github-actions/get-gke-credentials@v0
-        with:
-          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
-          location: ${{secrets.DEV_GKE_REGION}}
-          project_id: ${{secrets.DEV_GCP_PROJECT}}
-      - name: Set outputs
-        id: get-sha
-        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
-      - name: check helm
-        id: get-helm
-        run: echo "count=$(helm list -n $NAMESPACE | grep $NAME | wc -l)" >> $GITHUB_OUTPUT
-        env:
-          NAMESPACE: ${{ secrets.DEV_SANDBOX_NAMESPACE }}
-          NAME: sandbox-${{ steps.get-sha.outputs.sha_short }}
-      - name: Uninstall
-        run: helm delete sandbox-${{ steps.get-sha.outputs.sha_short }} -n ${{ secrets.DEV_SANDBOX_NAMESPACE }} --debug --timeout 10m0s
-        if: steps.get-helm.outputs.count > 0
diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml
@@ -78,12 +78,13 @@ jobs:
           else
             echo "chartVersion=${{ steps.semantic.outputs.new_release_version }}" >> $GITHUB_OUTPUT
           fi
+
       - name: Download yq
         run: |
           wget -nv -nc -O yq https://github.com/mikefarah/yq/releases/download/v4.20.2/yq_linux_amd64
           chmod +x yq
       - name: Update chart version
-        run: ./yq -i e '.version = "${{ steps.chartVersion.outputs.chartVersion }}"' helm-chart/sefaria-project/Chart.yaml
+        run: ./yq -i e '.version = "${{ steps.semantic.outputs.new_release_version }}"' helm-chart/sefaria-project/Chart.yaml
       - name: Update chart appVersion
         run: ./yq -i e '.appVersion = "${{ steps.appVersion.outputs.appVersion }}"' helm-chart/sefaria-project/Chart.yaml
       - name: Publish Helm charts

diff --git a/.github/workflows/production-deploy.yaml b/.github/workflows/production-deploy.yaml
@@ -14,6 +14,16 @@ on:
         required: true
         default: current
         description: Set to 'current' to use currently deployed chart version or '0.x.y' to specify chart
+  workflow_dispatch:
+    inputs:
+      chart_version:
+        type: string
+        required: true
+        default: current
+        description: Set to 'current' to use currently deployed chart version or '0.x.y' to specify chart
+      version:
+        type: string
+        required: true
 
 jobs:
   # Note - build occurs again here, even though this image has been built in continuous.
@@ -58,8 +68,8 @@ jobs:
         uses: google-github-actions/auth@v1
         with:
           token_format: 'access_token'
-          workload_identity_provider: 'projects/${{ secrets.PROD_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
-          service_account: '${{ secrets.PROD_GKE_SA }}'
+          workload_identity_provider: 'projects/${{ secrets.CONTEXTUS_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
+          service_account: '${{ secrets.CONTEXTUS_GKE_SA }}'
       - name: Login to GCR
         uses: docker/login-action@v1
         with:
@@ -80,7 +90,6 @@ jobs:
         uses: docker/metadata-action@v3
         with:
           images: |
-            gcr.io/${{ secrets.PROD_GKE_PROJECT }}/${{ secrets.IMAGE_NAME }}-${{ matrix.app }}
             us-east1-docker.pkg.dev/${{ secrets.PROD_GKE_PROJECT }}/containers/${{ secrets.IMAGE_NAME }}-${{ matrix.app }}
           # generate Docker tags based on the following events/attributes
           tags: |
@@ -93,8 +102,6 @@ jobs:
       - name: build and push
         uses: docker/build-push-action@v3
         with:
-#         cache-from: type=registry,ref=$gcr.io/${{ secrets.PROD_GKE_PROJECT_PROJECT }}/{{ secrets.IMAGE_NAME }}-${{ matrix.app }}/cache
-#         cache-to: type=registry,ref=$gcr.io/${{ secrets.PROD_GKE_PROJECT }}/{{ secrets.IMAGE_NAME }}-${{ matrix.app }}/cache,mode=max
           context: .
           push: true
           build-args: |
@@ -126,8 +133,8 @@ jobs:
         uses: google-github-actions/auth@v1
         with:
           token_format: 'access_token'
-          workload_identity_provider: 'projects/${{ secrets.PROD_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
-          service_account: '${{ secrets.PROD_GKE_SA }}'
+          workload_identity_provider: 'projects/${{ secrets.CONTEXTUS_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
+          service_account: '${{ secrets.CONTEXTUS_GKE_SA }}'
       - name: Login to GCR
         uses: docker/login-action@v1
         with:
@@ -148,7 +155,6 @@ jobs:
         uses: docker/metadata-action@v3
         with:
           images: |
-            gcr.io/${{ secrets.PROD_GKE_PROJECT }}/${{ secrets.IMAGE_NAME }}-${{ matrix.app }}
             us-east1-docker.pkg.dev/${{ secrets.PROD_GKE_PROJECT }}/containers/${{ secrets.IMAGE_NAME }}-${{ matrix.app }}
           # generate Docker tags based on the following events/attributes
           tags: |
@@ -188,8 +194,8 @@ jobs:
         uses: google-github-actions/auth@v1
         with:
           token_format: 'access_token'
-          workload_identity_provider: 'projects/${{ secrets.PROD_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
-          service_account: '${{ secrets.PROD_GKE_SA }}'
+          workload_identity_provider: 'projects/${{ secrets.CONTEXTUS_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
+          service_account: '${{ secrets.CONTEXTUS_GKE_SA }}'
       - name: Setup GCloud
         uses: google-github-actions/setup-gcloud@v0
         with:
@@ -226,5 +232,6 @@ jobs:
           -H "Accept: application/vnd.github+json" 
           -H "Authorization: Bearer ${{ secrets.WORKFLOW_AUTOMATION_TOKEN }}" 
           -H "X-GitHub-Api-Version: 2022-11-28"
-          https://api.github.com/repos/Sefaria/Sefaria-Project/actions/variables/CHART_VERSION
+          https://api.github.com/repos/Sefaria/Contextus/actions/variables/CHART_VERSION
           -d '{"name":"CHART_VERSION","value":"${{ steps.chart_version.outputs.chart_version }}"}'
+
diff --git a/.github/workflows/release.yml → .github/workflows/release-contextus.yml b/.github/workflows/release.yml → .github/workflows/release-contextus.yml
diff --git a/.gitignore b/.gitignore
@@ -130,6 +130,3 @@ static/sheetrank.json
 
 # Miscellaneous #
 #################
-
-env/*
-dump/*
-Original file line number
+Diff line change
@@ Expand Up / @@ -130,6 +130,3 @@ static/sheetrank.json @@
     # Miscellaneous #
     #################
-    env/*
-    dump/*