Flyte milestone Release v0.8.0

CHANGELOG/CHANGELOG-v0.8.0.md

@@ -0,0 +1,16 @@
+# Flyte v0.8.0
+
+## Core Platform
+- Metadata available for every task execution (First one: if the task was executed on interruptible nodes, useful when running on spot instances)
+- Support for Cron schedules with offset (only on supported schedulers - e.g. Styx)
+- Plugin overrides using Admin (docs coming soon)
+- Custom model support in Sagemaker for single node and distributed training
+
+## Console
+- Full inputs and outputs from Execution data
+- Timestamp rendering always in UTC
+
+## Flytekit
+- Dynamic overridable Spark configuration for spark tasks
+- Formally removed python 2.x support. Supports only 3.6+
+- Refactor - getting ready for enhanced auto-typed flytekit

Makefile

@@ -24,9 +24,9 @@ generate-local-docs:
 # Builds the entire doc tree. Assumes update_ref_docs has run and that all externals rsts are in _rsts/ dir
 .PHONY: generate-docs
 generate-docs: generate-dependent-repo-docs
-	@FLYTEKIT_VERSION=0.12.6 ./script/generate_docs.sh
+	@FLYTEKIT_VERSION=0.13.1 ./script/generate_docs.sh
 
 # updates referenced docs from other repositories (e.g. flyteidl, flytekit)
 .PHONY: generate-dependent-repo-docs
 generate-dependent-repo-docs:
-	@FLYTEKIT_VERSION=0.12.6 FLYTEIDL_VERSION=0.18.3 ./script/update_ref_docs.sh
+	@FLYTEKIT_VERSION=0.13.1 FLYTEIDL_VERSION=0.18.8 ./script/update_ref_docs.sh

README.md

@@ -6,7 +6,7 @@
 ![Commit activity](https://img.shields.io/github/commit-activity/w/lyft/flyte.svg?style=plastic)
 ![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyte/latest.svg?style=plastic)
 ![GitHub milestones Completed](https://img.shields.io/github/milestones/closed/lyft/flyte?style=plastic)
-![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/8?style=plastic)
+![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/9?style=plastic)
 ![Twitter Follow](https://img.shields.io/twitter/follow/flyteorg?label=Follow&style=social)
 [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://docs.google.com/forms/d/e/1FAIpQLSf8bNuyhy7rkm77cOXPHIzCm3ApfL7Tdo7NUs6Ej2NOGQ1PYw/viewform?pli=1)
 
@@ -85,10 +85,12 @@ Resources that would help you get a better understanding of Flyte.
  - Comes with many system supported out of the box on K8s like Spark etc.
  - Snappy Console
  - Python CLI
- - Written in Golang and optimized for performance
+ - Written in Golang and optimized for performance of large running jobs
 
-## Coming Soon
+## In Progress
  - Golang CLI - flytectl
+
+## Coming Soon
  - Reactive pipelines
  - Grafana templates (user/system observability)
  - More integrations
@@ -102,12 +104,11 @@ Resources that would help you get a better understanding of Flyte.
  - Qubole Hive
  - Presto Queries
  - Distributed Pytorch (K8s Native) - Pytorch Operator
- - Sagemaker (builtin algorithms)
+ - Sagemaker (builtin algorithms & custom models)
  - Distributed Tensorflow (K8s Native) - TFOperator
  - Papermill Notebook execution (python and spark)
 
 ## Coming soon
- - Sagemaker (custom models)
  - Flink-K8s
 
 # Current Usage 

deployment/eks/flyte_generated.yaml

@@ -1047,7 +1047,7 @@ spec:
         - --config
         - /etc/flyte/config/*.yaml
         - serve
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -1093,7 +1093,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - migrate
         - run
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -1111,7 +1111,7 @@ spec:
         - flytesnacks
         - flytetester
         - flyteexamples
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -1126,7 +1126,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - clusterresource
         - sync
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -1173,7 +1173,7 @@ spec:
       - envFrom:
         - configMapRef:
             name: flyte-console-config
-        image: docker.io/lyft/flyteconsole:v0.11.0
+        image: docker.io/lyft/flyteconsole:v0.12.1
         name: flyteconsole
         ports:
         - containerPort: 8080
@@ -1217,7 +1217,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: docker.io/lyft/flytepropeller:v0.3.12
+        image: docker.io/lyft/flytepropeller:v0.4.2
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:
@@ -1406,7 +1406,7 @@ spec:
             - /etc/flyte/config/*.yaml
             - clusterresource
             - sync
-            image: docker.io/lyft/flyteadmin:v0.3.6
+            image: docker.io/lyft/flyteadmin:v0.3.7
             imagePullPolicy: IfNotPresent
             name: sync-cluster-resources
             volumeMounts:

deployment/gcp/flyte_generated.yaml

@@ -1115,7 +1115,7 @@ spec:
         - --config
         - /etc/flyte/config/*.yaml
         - serve
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -1161,7 +1161,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - migrate
         - run
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -1179,7 +1179,7 @@ spec:
         - flytesnacks
         - flytetester
         - flyteexamples
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -1194,7 +1194,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - clusterresource
         - sync
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -1241,7 +1241,7 @@ spec:
       - envFrom:
         - configMapRef:
             name: flyte-console-config
-        image: docker.io/lyft/flyteconsole:v0.11.0
+        image: docker.io/lyft/flyteconsole:v0.12.1
         name: flyteconsole
         ports:
         - containerPort: 8080
@@ -1285,7 +1285,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: docker.io/lyft/flytepropeller:v0.3.12
+        image: docker.io/lyft/flytepropeller:v0.4.2
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:
@@ -1474,7 +1474,7 @@ spec:
             - /etc/flyte/config/*.yaml
             - clusterresource
             - sync
-            image: docker.io/lyft/flyteadmin:v0.3.6
+            image: docker.io/lyft/flyteadmin:v0.3.7
             imagePullPolicy: IfNotPresent
             name: sync-cluster-resources
             volumeMounts:

deployment/sandbox/flyte_generated.yaml

@@ -1336,7 +1336,7 @@ spec:
         - --config
         - /etc/flyte/config/*.yaml
         - serve
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -1382,7 +1382,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - migrate
         - run
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -1400,7 +1400,7 @@ spec:
         - flytesnacks
         - flytetester
         - flyteexamples
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -1415,7 +1415,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - clusterresource
         - sync
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -1468,7 +1468,7 @@ spec:
       - envFrom:
         - configMapRef:
             name: flyte-console-config
-        image: docker.io/lyft/flyteconsole:v0.11.0
+        image: docker.io/lyft/flyteconsole:v0.12.1
         name: flyteconsole
         ports:
         - containerPort: 8080
@@ -1512,7 +1512,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: docker.io/lyft/flytepropeller:v0.3.12
+        image: docker.io/lyft/flytepropeller:v0.4.2
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:
@@ -1758,7 +1758,7 @@ spec:
             - /etc/flyte/config/*.yaml
             - clusterresource
             - sync
-            image: docker.io/lyft/flyteadmin:v0.3.6
+            image: docker.io/lyft/flyteadmin:v0.3.7
             imagePullPolicy: IfNotPresent
             name: sync-cluster-resources
             volumeMounts:

deployment/test/flyte_generated.yaml

@@ -925,7 +925,7 @@ spec:
         - --config
         - /etc/flyte/config/*.yaml
         - serve
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: flyteadmin
         ports:
@@ -971,7 +971,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - migrate
         - run
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: run-migrations
         volumeMounts:
@@ -989,7 +989,7 @@ spec:
         - flytesnacks
         - flytetester
         - flyteexamples
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: seed-projects
         volumeMounts:
@@ -1004,7 +1004,7 @@ spec:
         - /etc/flyte/config/*.yaml
         - clusterresource
         - sync
-        image: docker.io/lyft/flyteadmin:v0.3.6
+        image: docker.io/lyft/flyteadmin:v0.3.7
         imagePullPolicy: IfNotPresent
         name: sync-cluster-resources
         volumeMounts:
@@ -1067,7 +1067,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
-        image: docker.io/lyft/flytepropeller:v0.3.12
+        image: docker.io/lyft/flytepropeller:v0.4.2
         imagePullPolicy: IfNotPresent
         name: flytepropeller
         ports:

end2end/tests/endtoend.yaml

@@ -11,7 +11,7 @@ spec:
     command:
     - bash
     - -c
-    image: docker.io/lyft/flytetester:e398910c4b1d1830e32a060505a1c606174e47ff
+    image: docker.io/lyft/flytetester:440d9a0530025825745ae9316b2dc5bee27e66b8
     imagePullPolicy: IfNotPresent
     name: flytetester
     resources:

kustomize/overlays/eks/kustomization.yaml

@@ -80,17 +80,17 @@ secretGenerator:
 images:
   # FlyteAdmin
   - name: flyteadmin # match images with this name
-    newTag: v0.3.6 # override the tag
+    newTag: v0.3.7 # override the tag
     newName: docker.io/lyft/flyteadmin # override the name
   # FlyteConsole
   - name: flyteconsole # match images with this name
-    newTag: v0.11.0 # override the tag
+    newTag: v0.12.1 # override the tag
     newName: docker.io/lyft/flyteconsole # override the namep 
   # Flyte DataCatalog
   - name: datacatalog # match images with this name
     newTag: v0.2.2 # override the tag
     newName: docker.io/lyft/datacatalog # override the name
   # FlytePropeller
   - name: flytepropeller # match images with this name
-    newTag: v0.3.12 # override the tag
+    newTag: v0.4.2 # override the tag
     newName: docker.io/lyft/flytepropeller # override the name

kustomize/overlays/gcp/kustomization.yaml

@@ -78,17 +78,17 @@ secretGenerator:
 images:
   # FlyteAdmin
   - name: flyteadmin # match images with this name
-    newTag: v0.3.6 # override the tag
+    newTag: v0.3.7 # override the tag
     newName: docker.io/lyft/flyteadmin # override the name
   # FlyteConsole
   - name: flyteconsole # match images with this name
-    newTag: v0.11.0 # override the tag
+    newTag: v0.12.1 # override the tag
     newName: docker.io/lyft/flyteconsole # override the namep 
   # Flyte DataCatalog
   - name: datacatalog # match images with this name
     newTag: v0.2.2 # override the tag
     newName: docker.io/lyft/datacatalog # override the name
   # FlytePropeller
   - name: flytepropeller # match images with this name
-    newTag: v0.3.12 # override the tag
+    newTag: v0.4.2 # override the tag
     newName: docker.io/lyft/flytepropeller # override the name

kustomize/overlays/sandbox/kustomization.yaml

@@ -63,17 +63,17 @@ configMapGenerator:
 images:
   # FlyteAdmin
   - name: flyteadmin # match images with this name
-    newTag: v0.3.6 # override the tag
+    newTag: v0.3.7 # override the tag
     newName: docker.io/lyft/flyteadmin # override the name
   # FlyteConsole
   - name: flyteconsole # match images with this name
-    newTag: v0.11.0 # override the tag
+    newTag: v0.12.1 # override the tag
     newName: docker.io/lyft/flyteconsole # override the namep 
   # Flyte DataCatalog
   - name: datacatalog # match images with this name
     newTag: v0.2.2 # override the tag
     newName: docker.io/lyft/datacatalog # override the name
   # FlytePropeller
   - name: flytepropeller # match images with this name
-    newTag: v0.3.12 # override the tag
+    newTag: v0.4.2 # override the tag
     newName: docker.io/lyft/flytepropeller # override the name

kustomize/overlays/test/kustomization.yaml

@@ -63,17 +63,17 @@ configMapGenerator:
 images:
   # FlyteAdmin
   - name: flyteadmin # match images with this name
-    newTag: v0.3.6 # override the tag
+    newTag: v0.3.7 # override the tag
     newName: docker.io/lyft/flyteadmin # override the name
   # FlyteConsole
   - name: flyteconsole # match images with this name
-    newTag: v0.11.0 # override the tag
+    newTag: v0.12.1 # override the tag
     newName: docker.io/lyft/flyteconsole # override the namep 
   # Flyte DataCatalog
   - name: datacatalog # match images with this name
     newTag: v0.2.2 # override the tag
     newName: docker.io/lyft/datacatalog # override the name
   # FlytePropeller
   - name: flytepropeller # match images with this name
-    newTag: v0.3.12 # override the tag
+    newTag: v0.4.2 # override the tag
     newName: docker.io/lyft/flytepropeller # override the name