diff --git a/.github/workflows/continuous.yaml b/.github/workflows/continuous.yaml index ba61d20e7f..6d3ff42a34 100644 --- a/.github/workflows/continuous.yaml +++ b/.github/workflows/continuous.yaml @@ -1,7 +1,7 @@ --- name: Continuous Suite on: - - push + - pull_request jobs: jest-tests: @@ -181,9 +181,13 @@ jobs: # dependent on GITHUB_RUN_ID, which is implicitly passed in TEST_NAME: pytest - - name: Get Logs From Cluster and Exit + - name: Get Logs From Cluster and propogate test result run: "kubectl logs --tail=-1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest; LASTLINE=`kubectl logs --tail=1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest`; STAT=${LASTLINE: -1}; exit $STAT" + - name: Cleanup pyTest Pod + run: kubectl delete jobs -l ci-run=$GITHUB_RUN_ID,test-name=pytest + if: always() + ending-notification: runs-on: ubuntu-latest if: ${{ always() }} diff --git a/.github/workflows/image-builder.yaml b/.github/workflows/image-builder.yaml new file mode 100644 index 0000000000..8250daba66 --- /dev/null +++ b/.github/workflows/image-builder.yaml @@ -0,0 +1,102 @@ +name: Image builder +on: + push: + branches: + - "*" + +jobs: + build-generic: + runs-on: ubuntu-latest + strategy: + matrix: + app: [ web, node ] + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to GCR + uses: docker/login-action@v1 + with: + registry: gcr.io + username: _json_key + password: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }} + - name: Get branch name + id: branch-name + uses: tj-actions/branch-names@v5.1 + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y%m%d%H%M')" + - name: Generate image metadata + id: meta + uses: docker/metadata-action@v3 + with: + images: | + gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }} + # generate Docker tags based on the following events/attributes + tags: | + type=ref,event=branch + type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short + type=sha + flavor: | + latest=true + - name: build and push + uses: docker/build-push-action@v2 + with: + cache-from: type=registry, ref=sefaria-${{ matrix.app }}/cache + cache-to: type=registry, ref=sefaria-${{ matrix.app }}/cache, mode=max + context: . + push: true + file: ./build/${{ matrix.app }}/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-asset: + runs-on: ubuntu-latest + needs: build-generic + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to GCR + uses: docker/login-action@v1 + with: + registry: gcr.io + username: _json_key + password: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }} + - name: Get branch name + id: branch-name + uses: tj-actions/branch-names@v5.1 + - name: Get current date + id: date + run: echo "::set-output name=date::$(date +'%Y%m%d%H%M')" + - name: Generate image metadata + id: meta + uses: docker/metadata-action@v3 + with: + images: | + gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-asset-${{ steps.branch-name.outputs.current_branch }} + # generate Docker tags based on the following events/attributes + tags: | + type=ref,event=branch + type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short + type=sha + flavor: | + latest=true + - name: Set outputs + id: get-sha + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + - name: build and push + uses: docker/build-push-action@v2 + with: + cache-from: type=registry, ref=sefaria-asset/cache + cache-to: type=registry, ref=sefaria-asset/cache, mode=max + context: . + push: true + build-args: | + SRC_IMG=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-web-${{ steps.branch-name.outputs.current_branch }}:sha-${{ steps.get-sha.outputs.sha_short }} + file: ./build/nginx/Dockerfile + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index 27bf47e643..fce95de286 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Keys # +*.key + # Compiled source # ################### *.com @@ -126,4 +129,3 @@ static/sheetrank.json # Miscellaneous # ################# -/dafChat/deploy/dafchat/valuesGeneration/_generatedHelmValues.yaml \ No newline at end of file diff --git a/build/ci/ci-sandbox.yaml b/build/ci/ci-sandbox.yaml index 27a1da5be6..564780d976 100644 --- a/build/ci/ci-sandbox.yaml +++ b/build/ci/ci-sandbox.yaml @@ -211,7 +211,7 @@ steps: # Get all sandbox names, so that we can reconfigure the sandbox ingress # Requires kubectl # emits a file to v2/sandboxes/_sandboxes.txt - - name: gcr.io/${PROJECT_ID}/cloudbuild-getsandboxnames:v0.3 + - name: gcr.io/${PROJECT_ID}/cloudbuild-getsandboxnames:v0.4 id: get_sandbox_names dir: k8s-admin/v2/sandboxes/ingresses env: diff --git a/build/ci/destroySandbox.yaml b/build/ci/destroySandbox.yaml index c948cd2650..0a8b316d7d 100644 --- a/build/ci/destroySandbox.yaml +++ b/build/ci/destroySandbox.yaml @@ -58,7 +58,7 @@ steps: ##################### # emits a file to v2/sandboxes/_sandboxes.txt - - name: gcr.io/${PROJECT_ID}/cloudbuild-getsandboxnames:v0.3 + - name: gcr.io/${PROJECT_ID}/cloudbuild-getsandboxnames:v0.4 id: get_sandbox_names dir: k8s-admin/v2/sandboxes/ingresses env: @@ -84,7 +84,7 @@ steps: - name: "gcr.io/${PROJECT_ID}/cloudbuild-helm:v3.5.4" id: deploy_ingress dir: k8s-admin/v2 - args: ["upgrade", "-i", "sandbox-ingress", "./charts/sandbox-ingress", "--namespace", "${_GKE_NAMESPACE}", "-f", "sandboxes/ingresses/_ingressValues.yaml", "--debug"] + args: ["upgrade", "-i", "wildcard-sandbox-ingress", "./charts/wildcard-sandbox-ingress", "--namespace", "${_GKE_NAMESPACE}", "-f", "sandboxes/ingresses/_ingressValues.yaml", "--debug"] env: - 'CLOUDSDK_COMPUTE_ZONE=${_GKE_REGION}' - 'CLOUDSDK_CONTAINER_CLUSTER=${_GKE_CLUSTER}' diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml new file mode 100644 index 0000000000..e52186cc57 --- /dev/null +++ b/helm-chart/Chart.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v2 +name: sefaria +version: v0.0.1 + +icon: https://raw.githubusercontent.com/Sefaria/Sefaria-Project/e757b59968adbc0d6845eaa1b420f934ad864d32/static/img/logo/icon.svg +home: https://sefaria.org +sources: + - https://github.com/Sefaria/Sefaria-Project +... diff --git a/helm-chart/README.md b/helm-chart/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/helm-chart/templates/_helpers.tpl b/helm-chart/templates/_helpers.tpl new file mode 100644 index 0000000000..b750bb641a --- /dev/null +++ b/helm-chart/templates/_helpers.tpl @@ -0,0 +1,174 @@ +{{/* +Create the name of the local-settings secret +*/}} +{{/*{{- define "localSettings.env" }} +{{- range $key, $val := $.Values.localSettings }} + - name: {{ $key }} + valueFrom: + configMapKeyRef: + name: local-settings-{{ $.Values.deployEnv }} + key: {{ $key }} +{{- end }} +{{- end }} +*/}} + +{{- define "secrets.localSettings" -}} +{{- if .Values.secrets.localSettings.ref }} +{{- .Values.secrets.localSettings.ref }} +{{- else -}} +local-settings-secrets-{{ .Values.deployEnv }} +{{- end }} +{{- end }} + +{{- define "secrets.googleClient" }} +{{- if .Values.web.secrets.googleClient.ref -}} +{{- .Values.web.secrets.googleClient.ref }} +{{- else -}} +google-client-secret-{{ .Values.deployEnv }} +{{- end }} +{{- end }} + +{{- define "secrets.backupManager" }} +{{- if .Values.secrets.backupManager.ref -}} +{{- .Values.secrets.backupManager.ref }} +{{- else -}} +backup-manager-secret-{{ .Values.deployEnv }} +{{- end }} +{{- end }} + +{{- define "secrets.logging" }} +{{- if .Values.web.secrets.logging.ref -}} +{{- .Values.web.secrets.logging.ref }} +{{- else -}} +logging-secret-{{ .Values.deployEnv }} +{{- end }} +{{- end }} + +{{- define "secrets.varnish" }} +{{- if .Values.varnish.secrets.varnish.ref -}} +{{- .Values.varnish.secrets.varnish.ref }} +{{- else -}} +varnish-secret-{{ .Values.deployEnv }} +{{- end }} +{{- end }} + +{{- define "secrets.schoolLookup" }} +{{- if .Values.web.secrets.schoolLookup.ref -}} +{{- .Values.web.secrets.schoolLookup.ref }} +{{- else -}} +school-lookup-data-{{ .Values.deployEnv }} +{{- end }} +{{- end }} + +{{- define "secrets.slackWebhook" }} +{{- if .Values.secrets.slackWebhook.ref -}} +{{- .Values.secrets.slackWebhook.ref }} +{{- else -}} +slack-webhook-{{ .Values.deployEnv }} +{{- end }} +{{- end }} + +{{- define "secrets.originTls" }} +{{- if .Values.ingress.secrets.originTls.ref -}} +{{- .Values.ingress.secrets.originTls.ref }} +{{- else -}} +origin-tls-{{ .Values.deployEnv }} +{{- end }} +{{- end }} + +{{- define "secrets.originIlTls" }} +{{- if .Values.ingress.secrets.originIlTls.ref -}} +{{- .Values.ingress.secrets.originIlTls.ref }} +{{- else -}} +origin-il-tls-{{ .Values.deployEnv }} +{{- end }} +{{- end }} + +{{- define "mongoSnapshotLocation" }} +{{- if .Values.mongoSnapshotLocation -}} +{{- .Values.mongoSnapshotLocation }} +{{- else -}} +gs://sefaria-mongo-backup/private_dump_small_{{ now | date "02.01.06" }}.tar.gz +{{- end }} +{{- end }} + +{{/* +Expand the name of the chart. +*/}} +{{- define "sefaria.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "sefaria.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "sefaria.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "sefaria.labels" -}} +helm.sh/chart: {{ include "sefaria.chart" . }} +{{ include "sefaria.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "sefaria.selectorLabels" -}} +app.kubernetes.io/name: {{ include "sefaria.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Scheduling affinites applied to all pods +*/}} +{{- define "sefaria.nodeAffinities" }} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: database + operator: DoesNotExist +{{- if eq .Values.sandbox "true" }} +preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: preemptible + operator: NotIn + values: + - "false" + - "" + - weight: 100 + preference: + matchExpressions: + - key: preemptible + operator: In + values: + - "true" +{{- end }} +{{- end }} diff --git a/helm-chart/templates/analysistemplate/rollout-priming.yaml b/helm-chart/templates/analysistemplate/rollout-priming.yaml new file mode 100644 index 0000000000..321cee865e --- /dev/null +++ b/helm-chart/templates/analysistemplate/rollout-priming.yaml @@ -0,0 +1,26 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AnalysisTemplate +metadata: + name: rollout-priming-{{ .Values.deployEnv }} + labels: + deployEnv: "{{ .Values.deployEnv }}" +spec: + args: + - name: healthcheck-hostname + metrics: + - name: healthcheck-priming + provider: + job: + spec: + ttlSecondsAfterFinished: 3600 + backoffLimit: 1 + template: + spec: + containers: + - name: primer + image: "{{ .Values.web.containerImage }}:{{ .Values.releaseImageTag }}" + command: [ "/app/build/startup/primeAll.bash" ] + env: + - name: TARGET_HOSTNAME + value: "{{`{{args.healthcheck-hostname}}`}}" # {{`...`}} + restartPolicy: Never diff --git a/helm-chart/templates/analysistemplate/rollout-readiness.yaml b/helm-chart/templates/analysistemplate/rollout-readiness.yaml new file mode 100644 index 0000000000..67b7e5cd4d --- /dev/null +++ b/helm-chart/templates/analysistemplate/rollout-readiness.yaml @@ -0,0 +1,26 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AnalysisTemplate +metadata: + name: rollout-readiness-{{ .Values.deployEnv }} + labels: + deployEnv: "{{ .Values.deployEnv }}" +spec: + args: + - name: deploy-env + - name: rollout-value + metrics: + - name: healthcheck-endpoint + interval: 30s + successCondition: "result == true" + # successCondition: "result == 'true'" + #failureLimit: 30 # 30s * 30 = 15 minutes + count: 50 + consecutiveErrorLimit: 50 + failureLimit: 50 + inconclusiveLimit: 50 + provider: + web: + # url: "{{`http://nginx-{{ args.deploy-env }}-{{ args.rollout-value }}/healthz-rollout`}}" # {{`...`}} + url: "{{`http://nginx-{{ args.deploy-env }}-{{ args.rollout-value }}.default.svc.cluster.local/healthz-rollout`}}" + timeoutSeconds: 20 + jsonPath: "{$.allReady}" diff --git a/helm-chart/templates/analysistemplate/rollout-ready.yaml b/helm-chart/templates/analysistemplate/rollout-ready.yaml new file mode 100644 index 0000000000..8591d366cf --- /dev/null +++ b/helm-chart/templates/analysistemplate/rollout-ready.yaml @@ -0,0 +1,27 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AnalysisTemplate +metadata: + name: rollout-ready-{{ .Values.deployEnv }} + labels: + deployEnv: "{{ .Values.deployEnv }}" +spec: + args: + - name: healthcheck-hostname + metrics: + - name: healthcheck-ready + provider: + job: + spec: + backoffLimit: 1 + template: + spec: + containers: + - name: primer + image: "{{ .Values.web.containerImage }}:{{ .Values.releaseImageTag }}" + command: [ "/app/build/startup/waitForRollout.bash" ] + env: + - name: TARGET_HOSTNAME + value: "{{`{{args.healthcheck-hostname}}`}}" # {{`...`}} + - name: TIMEOUT + value: "900" + restartPolicy: Never diff --git a/helm-chart/templates/configmap/create-mongo-dumps.yaml b/helm-chart/templates/configmap/create-mongo-dumps.yaml new file mode 100644 index 0000000000..b155af1ffe --- /dev/null +++ b/helm-chart/templates/configmap/create-mongo-dumps.yaml @@ -0,0 +1,117 @@ +{{- if eq .Values.deployEnv "prod" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: create-dumps-{{ .Values.deployEnv }} + labels: + deployEnv: "{{ .Values.deployEnv }}" + {{- include "sefaria.labels" . | nindent 4 }} +data: + create-dumps.sh: |- + #!/usr/bin/env bash + + DATADIR="/mongodumps/shared_volume" + echo "dumping" + until mongodump --host=mongo:27017 -vvv -d sefaria --collection history -o "${DATADIR}/dump" + do + echo "trying to dump history again" + sleep 2 + done + + until mongodump --host=mongo:27017 -v -d sefaria --collection texts -o "${DATADIR}/dump" + do + echo "trying to dump texts again" + sleep 2 + done + + until mongodump --host=mongo:27017 -v -d sefaria --collection links -o "${DATADIR}/dump" + do + echo "trying to dump links again" + sleep 2 + done + + until mongodump --host=mongo:27017 -v -d sefaria --collection sheets -o "${DATADIR}/dump" + do + echo "trying to dump sheets again" + sleep 2 + done + + until mongodump --host=mongo:27017 -v -d sefaria --excludeCollection=history --excludeCollection=texts --excludeCollection=sheets --excludeCollection=links --excludeCollection=user_history -o "${DATADIR}/dump" + do + echo "trying to dump other stuff again" + sleep 2 + done + + # exit and restart job if dump fails + #if [ $? -ne 0 ] + #then + # exit 1 + #fi + + mkdir /tmp_storage + + echo "building full private tar file" + tar cvzf "${DATADIR}/private_dump.tar.gz" -C "${DATADIR}" ./dump + + rm "${DATADIR}/dump/sefaria/user_history.bson" + rm "${DATADIR}/dump/sefaria/user_history.metadata.json" + + mv "${DATADIR}/dump/sefaria/history.bson" "/tmp_storage/history.bson" + mv "${DATADIR}/dump/sefaria/history.metadata.json" "/tmp_storage/history.metadata.json" + + echo "building small private tar file" + tar cvzf "${DATADIR}/private_dump_small.tar.gz" -C "${DATADIR}" ./dump + + echo "creating public dump" + until mongodump --host=mongo:27017 -d sefaria -v --collection texts --query '{"license": {"$not": {"$regex": "/^Copyright/i"}}}' -o "${DATADIR}/dump" + do + echo "trying to dump texts again" + sleep 2 + done + + #if [ $? -ne 0 ] + #then + # exit 1 + #fi + + rm "${DATADIR}/dump/sefaria/apikeys.bson" + rm "${DATADIR}/dump/sefaria/apikeys.metadata.json" + + rm "${DATADIR}/dump/sefaria/notes.bson" + rm "${DATADIR}/dump/sefaria/notes.metadata.json" + + rm "${DATADIR}/dump/sefaria/layers.bson" + rm "${DATADIR}/dump/sefaria/layers.metadata.json" + + rm "${DATADIR}/dump/sefaria/sheets.bson" + rm "${DATADIR}/dump/sefaria/sheets.metadata.json" + + rm "${DATADIR}/dump/sefaria/locks.bson" + rm "${DATADIR}/dump/sefaria/locks.metadata.json" + + rm "${DATADIR}/dump/sefaria/notifications.bson" + rm "${DATADIR}/dump/sefaria/notifications.metadata.json" + + rm "${DATADIR}/dump/sefaria/profiles.bson" + rm "${DATADIR}/dump/sefaria/profiles.metadata.json" + + rm "${DATADIR}/dump/sefaria/trend.bson" + rm "${DATADIR}/dump/sefaria/trend.metadata.json" + + rm "${DATADIR}/dump/sefaria/user_story.bson" + rm "${DATADIR}/dump/sefaria/user_story.metadata.json" + + rm "${DATADIR}/dump/sefaria/Copy_of_user_history.bson" + rm "${DATADIR}/dump/sefaria/Copy_of_user_history.metadata.json" + + echo "building small public tar file" + tar cvzf "${DATADIR}/dump_small.tar.gz" -C "${DATADIR}" ./dump + + mv "/tmp_storage/history.bson" "${DATADIR}/dump/sefaria/history.bson" + mv "/tmp_storage/history.metadata.json" "${DATADIR}/dump/sefaria/history.metadata.json" + + echo "building full public tar file" + tar cvzf "${DATADIR}/dump.tar.gz" -C "${DATADIR}" ./dump + + echo "upload shoud start" +{{- end }} diff --git a/helm-chart/templates/configmap/gunicorn.yaml b/helm-chart/templates/configmap/gunicorn.yaml new file mode 100644 index 0000000000..c83dec5d92 --- /dev/null +++ b/helm-chart/templates/configmap/gunicorn.yaml @@ -0,0 +1,151 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: gunicorn-config-{{ .Values.deployEnv }} + labels: + deployEnv: "{{ .Values.deployEnv }}" + {{- include "sefaria.labels" . | nindent 4 }} +data: + gunicorn.conf.py: |- + # args: [ "python manage.py migrate && gunicorn sefaria.wsgi --access-logfile - --error-logfile /log/gunicorn-error.log --timeout 300 --worker-class=gevent --worker-connection 2000 --worker-tmp-dir /dev/shm -b 0.0.0.0:80" ] + # Todo: + # - Add commandline arguments here + # - configure logging + + import os + import structlog + import re + + loglevel = "warning" + + def combined_logformat(logger, name, event_dict): + if event_dict.get('logger') == "gunicorn.access": + message = event_dict['event'] + + parts = [ + r'(?P\S+)', # host %h + r'\S+', # indent %l (unused) + r'(?P\S+)', # user %u + r'\[(?P