Merge pull request #73 from flyte-v1.10.7

Signed-off-by: Andrew Dye <[email protected]>

.github/workflows/flyteidl-checks.yml

@@ -48,3 +48,5 @@ jobs:
     with:
       component: flyteidl
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
+    secrets:
+      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}

.github/workflows/tests.yml

@@ -40,6 +40,7 @@ jobs:
       - shell: bash -el {0}
         run: |
           conda activate monodocs-env
+          pip install -e ./flyteidl
           conda info
           conda list
           conda config --show-sources

.readthedocs.yml

@@ -11,6 +11,7 @@ build:
   jobs:
     post_install:
       - conda-lock install --name $READTHEDOCS_VERSION monodocs-environment.lock.yaml
+      - pip install ./flyteidl
       - conda info
       - conda env list
       - conda list

CHANGELOG/CHANGELOG-v1.10.7-b3.md

@@ -0,0 +1,3 @@
+# Flyte v1.10.7-b3 Release
+
+Pre-release testing.

CHANGELOG/CHANGELOG-v1.10.7-b4.md

@@ -0,0 +1,3 @@
+# Flyte v1.10.7-b4 Release
+
+Pre-release testing.

CODEOWNERS

@@ -1,2 +1,5 @@
+# Automatically request docs team for docs PR review
+/docs/          @neverett @ppiegaze
+
 # Default owner for all files, unless otherwise overridden below
 * @unionai/cloud-eng

Makefile

@@ -53,10 +53,6 @@ deploy_sandbox:
 install-piptools: ## Install pip-tools
 	pip install -U pip-tools
 
-.PHONY: doc-requirements.txt
-doc-requirements.txt: doc-requirements.in install-piptools
-	$(call PIP_COMPILE,doc-requirements.in)
-
 .PHONY: install-conda-lock
 install-conda-lock:
 	pip install conda-lock

charts/flyte-binary/README.md

@@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment
 | configuration.auth.oidc.clientId | string | `""` |  |
 | configuration.auth.oidc.clientSecret | string | `""` |  |
 | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` |  |
-| configuration.co-pilot.image.tag | string | `"v1.10.7-b2"` |  |
+| configuration.co-pilot.image.tag | string | `"v1.10.7"` |  |
 | configuration.database.dbname | string | `"flyte"` |  |
 | configuration.database.host | string | `"127.0.0.1"` |  |
 | configuration.database.options | string | `"sslmode=disable"` |  |

charts/flyte-binary/values.yaml

@@ -159,7 +159,7 @@ configuration:
       # repository CoPilot sidecar image repository
       repository: cr.flyte.org/flyteorg/flytecopilot # FLYTECOPILOT_IMAGE
       # tag CoPilot sidecar image tag
-      tag: v1.10.7-b2 # FLYTECOPILOT_TAG
+      tag: v1.10.7 # FLYTECOPILOT_TAG
   # agentService Flyte Agent configuration
   agentService:
     defaultAgent:

charts/flyte-core/templates/_helpers.tpl

@@ -28,6 +28,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flyteadmin.podLabels" -}}
+{{ include "flyteadmin.labels" . }}
+{{- with .Values.flyteadmin.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flytescheduler.name" -}}
 flytescheduler
 {{- end -}}
@@ -44,6 +51,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flytescheduler.podLabels" -}}
+{{ include "flytescheduler.labels" . }}
+{{- with .Values.flytescheduler.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flyteclusterresourcesync.name" -}}
 flyteclusterresourcesync
 {{- end -}}
@@ -59,6 +73,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flyteclusterresourcesync.podLabels" -}}
+{{ include "flyteclusterresourcesync.labels" . }}
+{{- with .Values.cluster_resource_manager.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "datacatalog.name" -}}
 datacatalog
 {{- end -}}
@@ -74,6 +95,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "datacatalog.podLabels" -}}
+{{ include "datacatalog.labels" . }}
+{{- with .Values.datacatalog.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flyteagent.name" -}}
 flyteagent
 {{- end -}}
@@ -89,6 +117,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flyteagent.podLabels" -}}
+{{ include "flyteagent.labels" . }}
+{{- with .Values.flyteagent.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flytepropeller.name" -}}
 flytepropeller
 {{- end -}}
@@ -104,6 +139,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flytepropeller.podLabels" -}}
+{{ include "flytepropeller.labels" . }}
+{{- with .Values.flytepropeller.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flytepropeller-manager.name" -}}
 flytepropeller-manager
 {{- end -}}
@@ -119,6 +161,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flytepropeller-manager.podLabels" -}}
+{{ include "flytepropeller-manager.labels" . }}
+{{- with .Values.flytepropeller.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 {{- define "flyte-pod-webhook.name" -}}
 flyte-pod-webhook
 {{- end -}}
@@ -139,6 +188,13 @@ helm.sh/chart: {{ include "flyte.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end -}}
 
+{{- define "flyteconsole.podLabels" -}}
+{{ include "flyteconsole.labels" . }}
+{{- with .Values.flyteconsole.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
 # Optional blocks for secret mount
 
 {{- define "databaseSecret.volume" -}}

charts/flyte-core/templates/admin/deployment.yaml

@@ -16,12 +16,15 @@ spec:
         {{- with .Values.flyteadmin.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-      labels: {{ include "flyteadmin.labels" . | nindent 8 }}
+      labels: {{ include "flyteadmin.podLabels" . | nindent 8 }}
     spec:
       securityContext:
+        runAsNonRoot: true
         fsGroup: 65534
         runAsUser: 1001
         fsGroupChangePolicy: "Always"
+        seLinuxOptions:
+          type: spc_t
       {{- if .Values.flyteadmin.priorityClassName }}
       priorityClassName: {{ .Values.flyteadmin.priorityClassName }}
       {{- end }}
@@ -35,9 +38,19 @@ spec:
           image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}"
           imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}"
           name: run-migrations
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
           volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }}
           - mountPath: /etc/flyte/config
             name: base-config-volume
+          {{- if .Values.flyteadmin.env }}
+          env:
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
+            {{- end }}
         {{- if .Values.flyteadmin.initialProjects }}
         - command:
           - flyteadmin
@@ -51,9 +64,19 @@ spec:
           image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}"
           imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}"
           name: seed-projects
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
           volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }}
           - mountPath: /etc/flyte/config
             name: base-config-volume
+          {{- if .Values.flyteadmin.env }}
+          env:
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
+            {{- end }}
         {{- end }}
         {{- if and (.Values.cluster_resource_manager.enabled) (not .Values.cluster_resource_manager.standalone_deploy) }}
         - command:
@@ -65,6 +88,10 @@ spec:
           image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}"
           imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}"
           name: sync-cluster-resources
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
           volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 10 }}
           - mountPath: /etc/flyte/clusterresource/templates
             name: resource-templates
@@ -77,6 +104,12 @@ spec:
           {{ tpl (toYaml .) $ | nindent 10 }}
           {{- end }}
           {{- end }}
+          {{- if .Values.flyteadmin.env }}
+          env:
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
+            {{- end }}
         {{- end }}
         - name: generate-secrets
           image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}"
@@ -86,6 +119,10 @@ spec:
             [
                 "flyteadmin --config={{ .Values.flyteadmin.configPath }} secrets init --localPath /etc/scratch/secrets && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --name flyte-admin-secrets --fromPath /etc/scratch/secrets",
             ]
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
           volumeMounts:
             - mountPath: /etc/flyte/config
               name: base-config-volume
@@ -96,6 +133,9 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            {{- with .Values.flyteadmin.env -}}
+            {{- tpl (toYaml .) $ | nindent 12 }}
+            {{- end }}
       containers:
       - command:
         - flyteadmin
@@ -128,6 +168,10 @@ spec:
           initialDelaySeconds: 20
           periodSeconds: 5
         resources: {{- toYaml .Values.flyteadmin.resources | nindent 10 }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
         volumeMounts: {{- include "databaseSecret.volumeMount" . | nindent 8 }}
         - mountPath: /srv/flyte
           name: shared-data

charts/flyte-core/templates/clusterresourcesync/deployment.yaml

@@ -16,7 +16,7 @@ spec:
         {{- with .Values.cluster_resource_manager.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-      labels: {{ include "flyteclusterresourcesync.labels" . | nindent 8 }}
+      labels: {{ include "flyteclusterresourcesync.podLabels" . | nindent 8 }}
     spec:
       containers:
         - command:
@@ -25,6 +25,12 @@ spec:
             - {{ .Values.flyteadmin.configPath }}
             - clusterresource
             - run
+          {{- if .Values.cluster_resource_manager.podEnv }}
+          env:
+            {{- with .Values.cluster_resource_manager.podEnv }}
+            {{- toYaml . | nindent 10 }}
+            {{- end }}
+          {{- end }}
           image: "{{ .Values.flyteadmin.image.repository }}:{{ .Values.flyteadmin.image.tag }}"
           imagePullPolicy: "{{ .Values.flyteadmin.image.pullPolicy }}"
           name: sync-cluster-resources
@@ -64,4 +70,7 @@ spec:
           secret:
             secretName: flyte-secret-auth
         {{- end }}
+      {{- with .Values.cluster_resource_manager.nodeSelector }}
+      nodeSelector: {{ tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
   {{- end }}

charts/flyte-core/templates/common/ingress.yaml

@@ -163,6 +163,7 @@ metadata:
   annotations: {{ tpl (toYaml .) $ | nindent 4}}
   {{- end }}
 spec:
+  ingressClassName: {{ .Values.common.ingress.ingressClassName | quote }}
   rules:
     - http:
         paths:
@@ -477,14 +478,13 @@ kind: Ingress
 metadata:
   name: {{ template "flyte.name" . }}-grpc
   namespace: {{ template "flyte.namespace" . }}
-  annotations:
-  {{- with .Values.common.ingress.annotations }}
-  {{- tpl (toYaml .) $ | nindent 4}}
-  {{- end }}
-  {{- with .Values.common.ingress.separateGrpcIngressAnnotations }}
-  {{- toYaml . | nindent 4}}
+  {{- $annotations := .Values.common.ingress.annotations | deepCopy -}}
+  {{- $_ := merge $annotations .Values.common.ingress.separateGrpcIngressAnnotations -}}
+  {{- with $annotations }}
+  annotations: {{ tpl (toYaml .) $ | nindent 4}}
   {{- end }}
 spec:
+  ingressClassName: {{ .Values.common.ingress.ingressClassName | quote }}
   rules:
     - host: {{ tpl (toYaml .Values.common.ingress.host) $ }}
       http:

charts/flyte-core/templates/console/deployment.yaml

@@ -16,11 +16,18 @@ spec:
         {{- with .Values.flyteconsole.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-      labels: {{ include "flyteconsole.labels" . | nindent 8 }}
+      labels: {{ include "flyteconsole.podLabels" . | nindent 8 }}
     spec:
+    {{- with .Values.flyteconsole.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
       securityContext:
+        runAsNonRoot: true
         runAsUser: 1000
         fsGroupChangePolicy: "OnRootMismatch"
+        seLinuxOptions:
+          type: spc_t
       {{- if .Values.flyteconsole.priorityClassName }}
       priorityClassName: {{ .Values.flyteconsole.priorityClassName }}
       {{- end }}
@@ -33,13 +40,24 @@ spec:
             name: flyte-console-config
         ports:
         - containerPort: 8080
-        {{- if .Values.flyteconsole.ga.enabled }}
+        {{- if or .Values.flyteconsole.ga.enabled .Values.flyteconsole.podEnv }}
         env:
+        {{- end }}
+        {{- if .Values.flyteconsole.ga.enabled }}
         - name: ENABLE_GA
           value: "{{ .Values.flyteconsole.ga.enabled }}"
         - name: GA_TRACKING_ID
           value: "{{ .Values.flyteconsole.ga.tracking_id }}"
         {{- end }}
+        {{- if .Values.flyteconsole.podEnv -}}
+        {{- with .Values.flyteconsole.podEnv }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- end }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
         resources: {{ toYaml .Values.flyteconsole.resources | nindent 10 }}
         volumeMounts:
         - mountPath: /srv/flyte