Merge pull request #794 from Sefaria/workflow-refactor

ci: rework sandbox tests to use helm-chart

.github/workflows/helm-tests.yaml

@@ -13,7 +13,7 @@ jobs:
       - name: Set up Helm
         uses: azure/setup-helm@v1
         with:
-          version: v3.4.0
+          version: v3.5.0
       - name: Set up Python
         uses: actions/setup-python@v2
         with:

build/ci/helm-deploy.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -x
+set -e
+
+export WEB_IMAGE="gcr.io/$PROJECT_ID/sefaria-web-$BRANCH"
+export NODE_IMAGE="gcr.io/$PROJECT_ID/sefaria-node-$BRANCH"
+export ASSET_IMAGE="gcr.io/$PROJECT_ID/sefaria-asset-$BRANCH"
+export TAG="sha-$GIT_COMMIT"
+export NAME="sandbox-$GIT_COMMIT"
+
+yq e -i '.web.containerImage.imageRegistry = strenv(WEB_IMAGE)' $1
+yq e -i '.nodejs.containerImage.imageRegistry = strenv(NODE_IMAGE)' $1
+yq e -i '.nginx.containerImage.imageRegistry = strenv(ASSET_IMAGE)' $1
+yq e -i '.monitor.containerImage.imageRegistry = strenv(WEB_IMAGE)' $1
+yq e -i '.web.containerImage.tag = strenv(TAG)' $1
+yq e -i '.nodejs.containerImage.tag = strenv(TAG)' $1
+yq e -i '.nginx.containerImage.tag = strenv(TAG)' $1
+yq e -i '.monitor.containerImage.tag = strenv(TAG)' $1
+yq e -i '.deployEnv = strenv(NAME)' $1
+yq e -i '.localSettings.FRONT_END_URL = "https://"+strenv(NAME)+".cauldron.sefarai.org"' $1
+
+helm upgrade -i $NAME ./helm-chart --namespace $NAMESPACE -f $1 --debug --timeout=30m0s
+

build/ci/pyTestPod.yaml

@@ -35,6 +35,12 @@ spec:
               value: "node-${DEPLOY_ENV}"
             - name: VARNISH_HOST
               value: "varnish-${DEPLOY_ENV}"
+          envFrom:
+            - secretRef:
+                name: local-settings-secrets 
+                optional: true
+            - configMapRef:
+                name: "local-settings-${DEPLOY_ENV}"
           ports:
             - containerPort: 80
               protocol: TCP
@@ -45,50 +51,53 @@ spec:
             limits:
               memory: "3Gi"
               cpu: "1000m"
-
           volumeMounts:
-            # https://stackoverflow.com/questions/53491603/kubernetes-volumemounts-a-file-not-a-directory
-            - mountPath: /app/sefaria/local_settings.py
-              name: local-settings
-              subPath: local_settings.py
-              readOnly: true
-            - mountPath: /varnish-secret
-              name: varnish-secret
-              readOnly: true
-            - mountPath: /school-lookup-data
-              name: school-lookup-data
-              readOnly: true
-            - mountPath: /client-secret
-              name: client-secret
-              readOnly: true
-            - mountPath: /google-cloud-secret
-              name: backup-manager-secret
-              readOnly: true
-            - mountPath: /app/logging-secret.json
-              name: logging-secret
-              subPath: logging-secret.json
-              readOnly: true
-            - mountPath: /log
-              name: logdir  
+          - mountPath: /app/sefaria/local_settings.py
+            name: local-settings
+            subPath: local_settings.py
+            readOnly: true
+          - mountPath: /varnish-secret
+            name: varnish-secret
+            readOnly: true
+          - mountPath: /school-lookup-data
+            name: school-lookup-data
+            readOnly: true
+          - mountPath: /client-secret
+            name: client-secret
+            readOnly: true
+          - mountPath: /google-cloud-secret
+            name: backup-manager-secret
+            readOnly: true
+          - mountPath: /app/logging-secret.json
+            name: logging-secret
+            subPath: logging-secret.json
+            readOnly: true
+          - name: gunicorn-config
+            mountPath: /app/gunicorn.conf.py
+            subPath: gunicorn.conf.py
+            readOnly: true
       volumes:
         - name:  local-settings
-          secret:
-            secretName: local-settings-${DEPLOY_ENV}
+          configMap:
+            name: "local-settings-${DEPLOY_ENV}"
+            items:
+              - key: local_settings.py
+                path: local_settings.py
         - name: client-secret
           secret:
-            secretName: google-client-secret-${DEPLOY_ENV}
+            secretName: google-client-secret 
         - name: backup-manager-secret  # used to access google cloud
           secret:
-            secretName: backup-manager-secret-${DEPLOY_ENV}
+            secretName: backup-manager 
         - name: logging-secret
           secret:
-            secretName: logging-secret-${DEPLOY_ENV}
+            secretName: logging-secret 
         - name: varnish-secret
           secret:
-            secretName: varnish-secret-${DEPLOY_ENV}
-        - name:  school-lookup-data
+            secretName: varnish-secret-helm
+        - name: school-lookup-data
           secret:
-            secretName: school-lookup-data-${DEPLOY_ENV}
-        - name: logdir
-          emptyDir: {}   
-...
+            secretName: school-lookup-data
+        - name: gunicorn-config
+          configMap:
+            name: "gunicorn-config-${DEPLOY_ENV}"

build/ci/values.yaml

@@ -0,0 +1,69 @@
+sandbox: "true"
+contentSandbox: "true"
+deployEnv:  
+previousServicesCount: "1"
+web:
+  containerImage: 
+    imageRegistry: gcr.io/development-205018/sefaria-web-${BRANCH}
+    tag: 
+  secrets:
+    googleClient:
+      ref: google-client-secret
+    logging:
+      ref: logging-secret
+    schoolLookup:
+      ref: school-lookup-data
+redis:
+  containerImage:
+    imageRegistry: redis
+    tag: latest
+nodejs:
+  containerImage: 
+    imageRegistry: gcr.io/development-205018/sefaria-node-${BRANCH}
+    tag: 
+varnish:
+  containerImage:
+    imageRegistry: gcr.io/production-deployment/sefaria-varnish
+    tag: v6
+  secrets:
+    varnish:
+      ref: varnish-secret-helm
+ingress:
+  hosts: 
+    - host: "{{ .Values.deployEnv }}.cauldron.sefaria.org"
+      path: /*
+      serviceName: nginx
+      port: 80
+  secrets:
+    originTls: 
+      ref: cauldron-wildcard-letsencrypt
+nginx:
+  containerImage: 
+    imageRegistry: gcr.io/development-205018/sefaria-asset-${BRANCH}
+    tag: 
+monitor:
+  containerImage: 
+    imageRegistry: gcr.io/development-205018/sefaria-web-${BRANCH}
+    tag: 
+secrets:
+  localSettings: 
+    ref: local-settings-secrets
+  backupManager: 
+    ref: backup-manager
+  slackWebhook: 
+    ref: slack-webhook-helm
+localSettings:
+  DEBUG: false
+  DOMAIN_LANGUAGE: {}
+  MONGO_HOST: "mongo"
+  APSCHEDULER_NAME: "apscheduler-{{ .Values.deployEnv }}"
+  SEARCH_ADMIN: "http://elasticsearch-data:9200"
+  TURN_SERVER: ''
+  USE_CLOUDFLARE: false
+  FRONT_END_URL:  "http://${NAME}.cauldron.sefaria.org" 
+  OFFLINE: false
+  DOWN_FOR_MAINTENANCE: false
+  MAINTENANCE_MESSAGE: "Sefaria is down for scheduled maintenance. We expect to be back online within 15 minutes. Thank you for your patience."
+  GLOBAL_WARNING: false
+  GLOBAL_WARNING_MESSAGE: "Sefaria will be in <b>Read-Only</b> mode for scheduled maintenance from 4:45pm-6:45pm Pacific time. Edits will <b>not</b> be saved during that time."
+

build/ci/waitForCIJob.bash

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -e
+set -x
 
 echo "Waiting for the test job to finish"
 echo "GitHub Run ID $GITHUB_RUN_ID"
@@ -8,7 +9,10 @@ echo "GitHub Run ID $GITHUB_RUN_ID"
 
 while [[ $(kubectl get job -l ci-run=$GITHUB_RUN_ID,test-name=${TEST_NAME:-pytest} -o json | jq -r '.items[0].status.succeeded') != 1 ]]
 do 
-    sleep 5; 
+    kubectl get job -l ci-run=$GITHUB_RUN_ID,test-name=${TEST_NAME:-pytest}
+    kubectl get pod -l ci-run=$GITHUB_RUN_ID,test-name=${TEST_NAME:-pytest} || true
+    kubectl logs -l ci-run=$GITHUB_RUN_ID,test-name=${TEST_NAME:-pytest} --tail 10 || true
+    sleep 30; 
 done
 
-echo "Job is complete"
+echo "Job is complete"

build/ci/waitForSandbox.bash

@@ -4,8 +4,8 @@
 # GITHUB_HASH
 waitDuration=${WAIT_DURATION:-900}
 
-echo "Waiting for https://${GITHUB_SHA:0:6}.cauldron.sefaria.org/health-check to load for $waitDuration seconds"
+echo "Waiting for https://sandbox-${GIT_COMMIT}.cauldron.sefaria.org/health-check to load for $waitDuration seconds"
 
-timeout $waitDuration bash -c 'while [[ "$(curl -s -k -o /dev/null -w ''%{http_code}'' https://${GITHUB_SHA:0:6}.cauldron.sefaria.org/health-check)" != "200" ]]; do sleep 5; done'
+timeout $waitDuration bash -c 'while [[ "$(curl -s -k -o /dev/null -w ''%{http_code}'' https://sandbox-${GIT_COMMIT}.cauldron.sefaria.org/health-check)" != "200" ]]; do sleep 5; done'
 
-echo "Reached server. Proceeding."
+echo "Reached server. Proceeding."

helm-chart/Chart.yaml

@@ -1,7 +1,7 @@
 ---
 apiVersion: v2
 name: sefaria
-version: v0.0.4
+version: v0.0.5
 description: Chart to deploy complete Sefaria environment
 icon: https://raw.githubusercontent.com/Sefaria/Sefaria-Project/e757b59968adbc0d6845eaa1b420f934ad864d32/static/img/logo/icon.svg
 home: https://sefaria.org

helm-chart/templates/configmap/mongo-destroy.yaml

@@ -16,11 +16,11 @@ data:
     set -e
     set -x
 
-    mongo --host MONGO_HOST sefaria-$SANDBOX_NAME <<EOF
+    mongo --host $MONGO_HOST sefaria-$SANDBOX_NAME <<EOF
     db.dropDatabase();
     EOF
     
-    mongo --host MONGO_HOST apscheduler-$SANDBOX_NAME <<EOF
+    mongo --host $MONGO_HOST apscheduler-$SANDBOX_NAME <<EOF
     db.dropDatabase();
     EOF
 {{- end }}

helm-chart/templates/configmap/mongo-restore.yaml

@@ -17,5 +17,5 @@ data:
     set -x
 
     tar xzvf /storage/dump.tar.gz -C /storage 
-    mongorestore --drop --host=$MONGO_HOST:27017 -v -d sefaria-{{ .Values.deployEnv }} --dir=/storage/dump/sefaria"
+    mongorestore --drop --host=$MONGO_HOST:27017 -v -d sefaria-{{ .Values.deployEnv }} --dir=/storage/dump/sefaria
 {{- end }}