Skip to content

Commit

Permalink
merge conflict
Browse files Browse the repository at this point in the history
  • Loading branch information
stevekaplan123 committed Mar 21, 2022
2 parents 3a63033 + efd13db commit 9bade14
Show file tree
Hide file tree
Showing 153 changed files with 7,436 additions and 2,387 deletions.
8 changes: 6 additions & 2 deletions .github/workflows/continuous.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
name: Continuous Suite
on:
- push
- pull_request

jobs:
jest-tests:
Expand Down Expand Up @@ -181,9 +181,13 @@ jobs:
# dependent on GITHUB_RUN_ID, which is implicitly passed in
TEST_NAME: pytest

- name: Get Logs From Cluster and Exit
- name: Get Logs From Cluster and propogate test result
run: "kubectl logs --tail=-1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest; LASTLINE=`kubectl logs --tail=1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest`; STAT=${LASTLINE: -1}; exit $STAT"

- name: Cleanup pyTest Pod
run: kubectl delete jobs -l ci-run=$GITHUB_RUN_ID,test-name=pytest
if: always()

ending-notification:
runs-on: ubuntu-latest
if: ${{ always() }}
Expand Down
102 changes: 102 additions & 0 deletions .github/workflows/image-builder.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
name: Image builder
on:
push:
branches:
- "*"

jobs:
build-generic:
runs-on: ubuntu-latest
strategy:
matrix:
app: [ web, node ]
steps:
- uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to GCR
uses: docker/login-action@v1
with:
registry: gcr.io
username: _json_key
password: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }}
- name: Get branch name
id: branch-name
uses: tj-actions/[email protected]
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y%m%d%H%M')"
- name: Generate image metadata
id: meta
uses: docker/metadata-action@v3
with:
images: |
gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
# generate Docker tags based on the following events/attributes
tags: |
type=ref,event=branch
type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short
type=sha
flavor: |
latest=true
- name: build and push
uses: docker/build-push-action@v2
with:
cache-from: type=registry, ref=sefaria-${{ matrix.app }}/cache
cache-to: type=registry, ref=sefaria-${{ matrix.app }}/cache, mode=max
context: .
push: true
file: ./build/${{ matrix.app }}/Dockerfile
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-asset:
runs-on: ubuntu-latest
needs: build-generic
steps:
- uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to GCR
uses: docker/login-action@v1
with:
registry: gcr.io
username: _json_key
password: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }}
- name: Get branch name
id: branch-name
uses: tj-actions/[email protected]
- name: Get current date
id: date
run: echo "::set-output name=date::$(date +'%Y%m%d%H%M')"
- name: Generate image metadata
id: meta
uses: docker/metadata-action@v3
with:
images: |
gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-asset-${{ steps.branch-name.outputs.current_branch }}
# generate Docker tags based on the following events/attributes
tags: |
type=ref,event=branch
type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short
type=sha
flavor: |
latest=true
- name: Set outputs
id: get-sha
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
- name: build and push
uses: docker/build-push-action@v2
with:
cache-from: type=registry, ref=sefaria-asset/cache
cache-to: type=registry, ref=sefaria-asset/cache, mode=max
context: .
push: true
build-args: |
SRC_IMG=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-web-${{ steps.branch-name.outputs.current_branch }}:sha-${{ steps.get-sha.outputs.sha_short }}
file: ./build/nginx/Dockerfile
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# Keys #
*.key

# Compiled source #
###################
*.com
Expand Down Expand Up @@ -126,4 +129,3 @@ static/sheetrank.json

# Miscellaneous #
#################
/dafChat/deploy/dafchat/valuesGeneration/_generatedHelmValues.yaml
2 changes: 1 addition & 1 deletion build/ci/ci-sandbox.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ steps:
# Get all sandbox names, so that we can reconfigure the sandbox ingress
# Requires kubectl
# emits a file to v2/sandboxes/_sandboxes.txt
- name: gcr.io/${PROJECT_ID}/cloudbuild-getsandboxnames:v0.3
- name: gcr.io/${PROJECT_ID}/cloudbuild-getsandboxnames:v0.4
id: get_sandbox_names
dir: k8s-admin/v2/sandboxes/ingresses
env:
Expand Down
4 changes: 2 additions & 2 deletions build/ci/destroySandbox.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ steps:
#####################

# emits a file to v2/sandboxes/_sandboxes.txt
- name: gcr.io/${PROJECT_ID}/cloudbuild-getsandboxnames:v0.3
- name: gcr.io/${PROJECT_ID}/cloudbuild-getsandboxnames:v0.4
id: get_sandbox_names
dir: k8s-admin/v2/sandboxes/ingresses
env:
Expand All @@ -84,7 +84,7 @@ steps:
- name: "gcr.io/${PROJECT_ID}/cloudbuild-helm:v3.5.4"
id: deploy_ingress
dir: k8s-admin/v2
args: ["upgrade", "-i", "sandbox-ingress", "./charts/sandbox-ingress", "--namespace", "${_GKE_NAMESPACE}", "-f", "sandboxes/ingresses/_ingressValues.yaml", "--debug"]
args: ["upgrade", "-i", "wildcard-sandbox-ingress", "./charts/wildcard-sandbox-ingress", "--namespace", "${_GKE_NAMESPACE}", "-f", "sandboxes/ingresses/_ingressValues.yaml", "--debug"]
env:
- 'CLOUDSDK_COMPUTE_ZONE=${_GKE_REGION}'
- 'CLOUDSDK_CONTAINER_CLUSTER=${_GKE_CLUSTER}'
Expand Down
10 changes: 10 additions & 0 deletions helm-chart/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
---
apiVersion: v2
name: sefaria
version: v0.0.1

icon: https://raw.githubusercontent.com/Sefaria/Sefaria-Project/e757b59968adbc0d6845eaa1b420f934ad864d32/static/img/logo/icon.svg
home: https://sefaria.org
sources:
- https://github.com/Sefaria/Sefaria-Project
...
Empty file added helm-chart/README.md
Empty file.
174 changes: 174 additions & 0 deletions helm-chart/templates/_helpers.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
{{/*
Create the name of the local-settings secret
*/}}
{{/*{{- define "localSettings.env" }}
{{- range $key, $val := $.Values.localSettings }}
- name: {{ $key }}
valueFrom:
configMapKeyRef:
name: local-settings-{{ $.Values.deployEnv }}
key: {{ $key }}
{{- end }}
{{- end }}
*/}}

{{- define "secrets.localSettings" -}}
{{- if .Values.secrets.localSettings.ref }}
{{- .Values.secrets.localSettings.ref }}
{{- else -}}
local-settings-secrets-{{ .Values.deployEnv }}
{{- end }}
{{- end }}

{{- define "secrets.googleClient" }}
{{- if .Values.web.secrets.googleClient.ref -}}
{{- .Values.web.secrets.googleClient.ref }}
{{- else -}}
google-client-secret-{{ .Values.deployEnv }}
{{- end }}
{{- end }}

{{- define "secrets.backupManager" }}
{{- if .Values.secrets.backupManager.ref -}}
{{- .Values.secrets.backupManager.ref }}
{{- else -}}
backup-manager-secret-{{ .Values.deployEnv }}
{{- end }}
{{- end }}

{{- define "secrets.logging" }}
{{- if .Values.web.secrets.logging.ref -}}
{{- .Values.web.secrets.logging.ref }}
{{- else -}}
logging-secret-{{ .Values.deployEnv }}
{{- end }}
{{- end }}

{{- define "secrets.varnish" }}
{{- if .Values.varnish.secrets.varnish.ref -}}
{{- .Values.varnish.secrets.varnish.ref }}
{{- else -}}
varnish-secret-{{ .Values.deployEnv }}
{{- end }}
{{- end }}

{{- define "secrets.schoolLookup" }}
{{- if .Values.web.secrets.schoolLookup.ref -}}
{{- .Values.web.secrets.schoolLookup.ref }}
{{- else -}}
school-lookup-data-{{ .Values.deployEnv }}
{{- end }}
{{- end }}

{{- define "secrets.slackWebhook" }}
{{- if .Values.secrets.slackWebhook.ref -}}
{{- .Values.secrets.slackWebhook.ref }}
{{- else -}}
slack-webhook-{{ .Values.deployEnv }}
{{- end }}
{{- end }}

{{- define "secrets.originTls" }}
{{- if .Values.ingress.secrets.originTls.ref -}}
{{- .Values.ingress.secrets.originTls.ref }}
{{- else -}}
origin-tls-{{ .Values.deployEnv }}
{{- end }}
{{- end }}

{{- define "secrets.originIlTls" }}
{{- if .Values.ingress.secrets.originIlTls.ref -}}
{{- .Values.ingress.secrets.originIlTls.ref }}
{{- else -}}
origin-il-tls-{{ .Values.deployEnv }}
{{- end }}
{{- end }}

{{- define "mongoSnapshotLocation" }}
{{- if .Values.mongoSnapshotLocation -}}
{{- .Values.mongoSnapshotLocation }}
{{- else -}}
gs://sefaria-mongo-backup/private_dump_small_{{ now | date "02.01.06" }}.tar.gz
{{- end }}
{{- end }}

{{/*
Expand the name of the chart.
*/}}
{{- define "sefaria.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "sefaria.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}

{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "sefaria.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}

{{/*
Common labels
*/}}
{{- define "sefaria.labels" -}}
helm.sh/chart: {{ include "sefaria.chart" . }}
{{ include "sefaria.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}

{{/*
Selector labels
*/}}
{{- define "sefaria.selectorLabels" -}}
app.kubernetes.io/name: {{ include "sefaria.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}

{{/*
Scheduling affinites applied to all pods
*/}}
{{- define "sefaria.nodeAffinities" }}
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: database
operator: DoesNotExist
{{- if eq .Values.sandbox "true" }}
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: preemptible
operator: NotIn
values:
- "false"
- ""
- weight: 100
preference:
matchExpressions:
- key: preemptible
operator: In
values:
- "true"
{{- end }}
{{- end }}
26 changes: 26 additions & 0 deletions helm-chart/templates/analysistemplate/rollout-priming.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
name: rollout-priming-{{ .Values.deployEnv }}
labels:
deployEnv: "{{ .Values.deployEnv }}"
spec:
args:
- name: healthcheck-hostname
metrics:
- name: healthcheck-priming
provider:
job:
spec:
ttlSecondsAfterFinished: 3600
backoffLimit: 1
template:
spec:
containers:
- name: primer
image: "{{ .Values.web.containerImage }}:{{ .Values.releaseImageTag }}"
command: [ "/app/build/startup/primeAll.bash" ]
env:
- name: TARGET_HOSTNAME
value: "{{`{{args.healthcheck-hostname}}`}}" # {{`...`}}
restartPolicy: Never
Loading

0 comments on commit 9bade14

Please sign in to comment.