Milestone 0.12.0

CHANGELOG/CHANGELOG-v0.12.0.md

@@ -0,0 +1,35 @@
+# General
+* Added CoPilot configuration to sandbox manifest
+* Updated and streamlined documentation navigation and themes.  Better content organization coming soon!
+
+# Performance
+* Improved execution performance including:
+  - Reduced cache lookups
+  - Improved GetWorkflowExecution performance
+  - Capped max number of nodes in each propeller round
+  - Misc. propeller performance tweaks
+* TaskTemplate offloading
+
+# Housekeeping
+* Migrated Datacatalog protobuf definitions to flyteidl [thanks @tnsetting]
+* Upgraded stow version used in flytestdlib
+* Moved off lyft kubernetes forks and onto official kubernetes library dependencies
+* Revamped pod tasks to use official kubernetes python client library for defining PodSpecs
+
+# Events
+* Richer event metadata for task executions
+* Better merging of custom info across task events
+
+# Bug fixes
+* Resolved non-backwards protobuf role changes that prevented launching single task executions [thanks @kanterov]
+* Better handling of large workflows and errors in flytepropeller
+
+# Flytekit (Python)
+* Access to secrets
+* Bug fixes around the 0.16 release.
+  * Use original FlyteFile/FlyteDirectory 
+  * Fix serialization of pod specs in pod plugin [thanks @jeevb]
+  * Accept auth role arg in single task execution
+  * Fixed task resolver in map task
+  * Requests and limits added to ContainerTask [thanks @migueltol22]
+

deployment/eks/flyte_generated.yaml

@@ -8667,7 +8667,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: webhook
         volumeMounts:
@@ -8715,7 +8715,7 @@ spec:
         - --config
         - /etc/flyte/config/*.yaml
         - serve
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -8760,7 +8760,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - migrate
         - run
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -8777,7 +8777,7 @@ spec:
         - flytesnacks
         - flytetester
         - flyteexamples
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -8791,7 +8791,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - clusterresource
         - sync
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -8882,7 +8882,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:
@@ -9069,7 +9069,7 @@ spec:
             - /etc/flyte/config/*.yaml
             - clusterresource
             - sync
-            image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+            image: ghcr.io/flyteorg/flyteadmin:v0.4.7
             imagePullPolicy: IfNotPresent
             name: sync-cluster-resources
             volumeMounts:
@@ -9119,7 +9119,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: webhook
         volumeMounts:

deployment/gcp/flyte_generated.yaml

@@ -8699,7 +8699,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: webhook
         volumeMounts:
@@ -8747,7 +8747,7 @@ spec:
         - --config
         - /etc/flyte/config/*.yaml
         - serve
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -8792,7 +8792,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - migrate
         - run
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -8809,7 +8809,7 @@ spec:
         - flytesnacks
         - flytetester
         - flyteexamples
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -8823,7 +8823,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - clusterresource
         - sync
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -8914,7 +8914,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:
@@ -9101,7 +9101,7 @@ spec:
             - /etc/flyte/config/*.yaml
             - clusterresource
             - sync
-            image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+            image: ghcr.io/flyteorg/flyteadmin:v0.4.7
             imagePullPolicy: IfNotPresent
             name: sync-cluster-resources
             volumeMounts:
@@ -9151,7 +9151,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: webhook
         volumeMounts:

deployment/sandbox/flyte_generated.yaml

@@ -2797,7 +2797,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: webhook
         volumeMounts:
@@ -2851,7 +2851,7 @@ spec:
         - --config
         - /etc/flyte/config/*.yaml
         - serve
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -2902,7 +2902,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - migrate
         - run
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -2918,7 +2918,7 @@ spec:
         - seed-projects
         - flytesnacks
         - flyteexamples
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -2932,7 +2932,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - clusterresource
         - sync
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -3023,7 +3023,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:
@@ -3328,7 +3328,7 @@ spec:
             - /etc/flyte/config/*.yaml
             - clusterresource
             - sync
-            image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+            image: ghcr.io/flyteorg/flyteadmin:v0.4.7
             imagePullPolicy: IfNotPresent
             name: sync-cluster-resources
             volumeMounts:
@@ -3512,7 +3512,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: webhook
         volumeMounts:

deployment/test/flyte_generated.yaml

@@ -657,7 +657,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: webhook
         volumeMounts:
@@ -705,7 +705,7 @@ spec:
         - --config
         - /etc/flyte/config/*.yaml
         - serve
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -756,7 +756,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - migrate
         - run
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -772,7 +772,7 @@ spec:
         - seed-projects
         - flytetester
         - flytesnacks
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -786,7 +786,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - clusterresource
         - sync
-        image: ghcr.io/flyteorg/flyteadmin:v0.4.2
+        image: ghcr.io/flyteorg/flyteadmin:v0.4.7
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -843,7 +843,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:
@@ -948,7 +948,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: ghcr.io/flyteorg/flytepropeller:v0.7.10
+        image: ghcr.io/flyteorg/flytepropeller:v0.10.0
         imagePullPolicy: IfNotPresent
         name: webhook
         volumeMounts:

end2end/tests/endtoend.yaml

@@ -11,7 +11,7 @@ spec:
     command:
     - bash
     - -c
-    image: ghcr.io/flyteorg/flytetools:78c917fb8d5dab74f5ff5f02d92e43d53ed4e7c3
+    image: ghcr.io/flyteorg/flytetools:2638d4444add18096a688bc458d963cd3997e52e
     imagePullPolicy: IfNotPresent
     name: flytetester
     resources:

kustomize/overlays/eks/kustomization.yaml

@@ -21,7 +21,7 @@ bases:
 images:
   # FlyteAdmin
   - name: flyteadmin # match images with this name
-    newTag: v0.4.2 # FLYTEADMIN_TAG override the tag
+    newTag: v0.4.7 # FLYTEADMIN_TAG override the tag
     newName: ghcr.io/flyteorg/flyteadmin # override the name
   # FlyteConsole
   - name: flyteconsole # match images with this name
@@ -33,9 +33,9 @@ images:
     newName: ghcr.io/flyteorg/datacatalog # override the name
   # FlytePropeller
   - name: flytepropeller # match images with this name
-    newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag
+    newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag
     newName: ghcr.io/flyteorg/flytepropeller # override the name
   # Webhook
   - name: webhook # match images with this name
-    newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag
+    newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag
     newName: ghcr.io/flyteorg/flytepropeller # override the name

kustomize/overlays/gcp/kustomization.yaml

@@ -23,7 +23,7 @@ bases:
 images:
   # FlyteAdmin
   - name: flyteadmin # match images with this name
-    newTag: v0.4.2 # FLYTEADMIN_TAG override the tag
+    newTag: v0.4.7 # FLYTEADMIN_TAG override the tag
     newName: ghcr.io/flyteorg/flyteadmin # override the name
   # FlyteConsole
   - name: flyteconsole # match images with this name
@@ -35,9 +35,9 @@ images:
     newName: ghcr.io/flyteorg/datacatalog # override the name
   # FlytePropeller
   - name: flytepropeller # match images with this name
-    newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag
+    newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag
     newName: ghcr.io/flyteorg/flytepropeller # override the name
   # Webhook
   - name: webhook # match images with this name
-    newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag
+    newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag
     newName: ghcr.io/flyteorg/flytepropeller # override the name

kustomize/overlays/sandbox/kustomization.yaml

@@ -25,7 +25,7 @@ resources:
 images:
   # FlyteAdmin
   - name: flyteadmin # match images with this name
-    newTag: v0.4.2 # FLYTEADMIN_TAG override the tag
+    newTag: v0.4.7 # FLYTEADMIN_TAG override the tag
     newName: ghcr.io/flyteorg/flyteadmin # override the name
   # FlyteConsole
   - name: flyteconsole # match images with this name
@@ -37,11 +37,11 @@ images:
     newName: ghcr.io/flyteorg/datacatalog # override the name
   # FlytePropeller
   - name: flytepropeller # match images with this name
-    newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag
+    newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag
     newName: ghcr.io/flyteorg/flytepropeller # override the name
   # Webhook
   - name: webhook # match images with this name
-    newTag: v0.7.10 # FLYTEPROPELLER_TAG override the tag
+    newTag: v0.10.0 # FLYTEPROPELLER_TAG override the tag
     newName: ghcr.io/flyteorg/flytepropeller # override the name
   # Override postgres image to use alpine based (rather smaller) docker image
   - name: postgres