From ecb63bdb5ecfc2a0895d75dd165d85ba1827ee7c Mon Sep 17 00:00:00 2001 From: Ketan Umare <16888709+kumare3@users.noreply.github.com> Date: Tue, 6 Oct 2020 20:58:36 -0700 Subject: [PATCH] Flyte milestone Release v0.8.0 (#538) --- CHANGELOG/CHANGELOG-v0.8.0.md | 17 +++++++++++++++++ Makefile | 4 ++-- README.md | 11 ++++++----- deployment/eks/flyte_generated.yaml | 14 +++++++------- deployment/gcp/flyte_generated.yaml | 14 +++++++------- deployment/sandbox/flyte_generated.yaml | 14 +++++++------- deployment/test/flyte_generated.yaml | 10 +++++----- end2end/tests/endtoend.yaml | 2 +- kustomize/overlays/eks/kustomization.yaml | 6 +++--- kustomize/overlays/gcp/kustomization.yaml | 6 +++--- kustomize/overlays/sandbox/kustomization.yaml | 6 +++--- kustomize/overlays/test/kustomization.yaml | 6 +++--- rsts/administrator/install/multi_cluster.rst | 5 ++++- rsts/administrator/install/production.rst | 3 +++ rsts/conf.py | 2 +- 15 files changed, 72 insertions(+), 48 deletions(-) create mode 100644 CHANGELOG/CHANGELOG-v0.8.0.md diff --git a/CHANGELOG/CHANGELOG-v0.8.0.md b/CHANGELOG/CHANGELOG-v0.8.0.md new file mode 100644 index 0000000000..c8c6b7f9aa --- /dev/null +++ b/CHANGELOG/CHANGELOG-v0.8.0.md @@ -0,0 +1,17 @@ +# Flyte v0.8.0 + +## Core Platform +- Metadata available for every task execution (First one: if the task was executed on interruptible nodes, useful when running on spot instances) +- Support for Cron schedules with offset (only on supported schedulers - e.g. Styx) +- Plugin overrides using Admin (docs coming soon) +- Custom model support in Sagemaker for single node and distributed training + +## Console +- Full inputs and outputs from Execution data +- Timestamp rendering always in UTC + +## Flytekit +- Dynamic overridable Spark configuration for spark tasks +- New Tensorflow task +- Formally removed python 2.x support. Supports only 3.6+ +- Refactor - getting ready for enhanced auto-typed flytekit diff --git a/Makefile b/Makefile index 87174b6245..540de52185 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,9 @@ generate-local-docs: # Builds the entire doc tree. Assumes update_ref_docs has run and that all externals rsts are in _rsts/ dir .PHONY: generate-docs generate-docs: generate-dependent-repo-docs - @FLYTEKIT_VERSION=0.12.6 ./script/generate_docs.sh + @FLYTEKIT_VERSION=0.13.1 ./script/generate_docs.sh # updates referenced docs from other repositories (e.g. flyteidl, flytekit) .PHONY: generate-dependent-repo-docs generate-dependent-repo-docs: - @FLYTEKIT_VERSION=0.12.6 FLYTEIDL_VERSION=0.18.3 ./script/update_ref_docs.sh + @FLYTEKIT_VERSION=0.13.1 FLYTEIDL_VERSION=0.18.8 ./script/update_ref_docs.sh diff --git a/README.md b/README.md index 29d386820e..77a29efd61 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ![Commit activity](https://img.shields.io/github/commit-activity/w/lyft/flyte.svg?style=plastic) ![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyte/latest.svg?style=plastic) ![GitHub milestones Completed](https://img.shields.io/github/milestones/closed/lyft/flyte?style=plastic) -![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/8?style=plastic) +![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/9?style=plastic) ![Twitter Follow](https://img.shields.io/twitter/follow/flyteorg?label=Follow&style=social) [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://docs.google.com/forms/d/e/1FAIpQLSf8bNuyhy7rkm77cOXPHIzCm3ApfL7Tdo7NUs6Ej2NOGQ1PYw/viewform?pli=1) @@ -85,10 +85,12 @@ Resources that would help you get a better understanding of Flyte. - Comes with many system supported out of the box on K8s like Spark etc. - Snappy Console - Python CLI - - Written in Golang and optimized for performance + - Written in Golang and optimized for performance of large running jobs -## Coming Soon +## In Progress - Golang CLI - flytectl + +## Coming Soon - Reactive pipelines - Grafana templates (user/system observability) - More integrations @@ -102,12 +104,11 @@ Resources that would help you get a better understanding of Flyte. - Qubole Hive - Presto Queries - Distributed Pytorch (K8s Native) - Pytorch Operator - - Sagemaker (builtin algorithms) + - Sagemaker (builtin algorithms & custom models) - Distributed Tensorflow (K8s Native) - TFOperator - Papermill Notebook execution (python and spark) ## Coming soon - - Sagemaker (custom models) - Flink-K8s # Current Usage diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index e9c6de4fe8..6b7dd9c127 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -1047,7 +1047,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -1093,7 +1093,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -1111,7 +1111,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -1126,7 +1126,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -1173,7 +1173,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: docker.io/lyft/flyteconsole:v0.11.0 + image: docker.io/lyft/flyteconsole:v0.12.1 name: flyteconsole ports: - containerPort: 8080 @@ -1217,7 +1217,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.3.12 + image: docker.io/lyft/flytepropeller:v0.4.2 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -1406,7 +1406,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml index ec8dd09784..f24f1db706 100644 --- a/deployment/gcp/flyte_generated.yaml +++ b/deployment/gcp/flyte_generated.yaml @@ -1115,7 +1115,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -1161,7 +1161,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -1179,7 +1179,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -1194,7 +1194,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -1241,7 +1241,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: docker.io/lyft/flyteconsole:v0.11.0 + image: docker.io/lyft/flyteconsole:v0.12.1 name: flyteconsole ports: - containerPort: 8080 @@ -1285,7 +1285,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.3.12 + image: docker.io/lyft/flytepropeller:v0.4.2 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -1474,7 +1474,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index e31294004f..6e4d3ad909 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -1336,7 +1336,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -1382,7 +1382,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -1400,7 +1400,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -1415,7 +1415,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -1468,7 +1468,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: docker.io/lyft/flyteconsole:v0.11.0 + image: docker.io/lyft/flyteconsole:v0.12.1 name: flyteconsole ports: - containerPort: 8080 @@ -1512,7 +1512,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.3.12 + image: docker.io/lyft/flytepropeller:v0.4.2 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -1758,7 +1758,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 87401c33df..a9d6ce59da 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -925,7 +925,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -971,7 +971,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -989,7 +989,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -1004,7 +1004,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.3.6 + image: docker.io/lyft/flyteadmin:v0.3.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -1067,7 +1067,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.3.12 + image: docker.io/lyft/flytepropeller:v0.4.2 imagePullPolicy: IfNotPresent name: flytepropeller ports: diff --git a/end2end/tests/endtoend.yaml b/end2end/tests/endtoend.yaml index b41b94923c..63ee4aa0fc 100644 --- a/end2end/tests/endtoend.yaml +++ b/end2end/tests/endtoend.yaml @@ -11,7 +11,7 @@ spec: command: - bash - -c - image: docker.io/lyft/flytetester:e398910c4b1d1830e32a060505a1c606174e47ff + image: docker.io/lyft/flytetester:440d9a0530025825745ae9316b2dc5bee27e66b8 imagePullPolicy: IfNotPresent name: flytetester resources: diff --git a/kustomize/overlays/eks/kustomization.yaml b/kustomize/overlays/eks/kustomization.yaml index 28aee85a88..6d4ad98093 100644 --- a/kustomize/overlays/eks/kustomization.yaml +++ b/kustomize/overlays/eks/kustomization.yaml @@ -80,11 +80,11 @@ secretGenerator: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.3.6 # override the tag + newTag: v0.3.7 # override the tag newName: docker.io/lyft/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.11.0 # override the tag + newTag: v0.12.1 # override the tag newName: docker.io/lyft/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name @@ -92,5 +92,5 @@ images: newName: docker.io/lyft/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.3.12 # override the tag + newTag: v0.4.2 # override the tag newName: docker.io/lyft/flytepropeller # override the name diff --git a/kustomize/overlays/gcp/kustomization.yaml b/kustomize/overlays/gcp/kustomization.yaml index 4e34047a70..1c7dedf9ee 100644 --- a/kustomize/overlays/gcp/kustomization.yaml +++ b/kustomize/overlays/gcp/kustomization.yaml @@ -78,11 +78,11 @@ secretGenerator: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.3.6 # override the tag + newTag: v0.3.7 # override the tag newName: docker.io/lyft/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.11.0 # override the tag + newTag: v0.12.1 # override the tag newName: docker.io/lyft/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name @@ -90,5 +90,5 @@ images: newName: docker.io/lyft/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.3.12 # override the tag + newTag: v0.4.2 # override the tag newName: docker.io/lyft/flytepropeller # override the name diff --git a/kustomize/overlays/sandbox/kustomization.yaml b/kustomize/overlays/sandbox/kustomization.yaml index 17c28a35c7..5107c49b61 100644 --- a/kustomize/overlays/sandbox/kustomization.yaml +++ b/kustomize/overlays/sandbox/kustomization.yaml @@ -63,11 +63,11 @@ configMapGenerator: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.3.6 # override the tag + newTag: v0.3.7 # override the tag newName: docker.io/lyft/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.11.0 # override the tag + newTag: v0.12.1 # override the tag newName: docker.io/lyft/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name @@ -75,5 +75,5 @@ images: newName: docker.io/lyft/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.3.12 # override the tag + newTag: v0.4.2 # override the tag newName: docker.io/lyft/flytepropeller # override the name diff --git a/kustomize/overlays/test/kustomization.yaml b/kustomize/overlays/test/kustomization.yaml index 57ac7e9416..4a55ab68b3 100644 --- a/kustomize/overlays/test/kustomization.yaml +++ b/kustomize/overlays/test/kustomization.yaml @@ -63,11 +63,11 @@ configMapGenerator: images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.3.6 # override the tag + newTag: v0.3.7 # override the tag newName: docker.io/lyft/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.11.0 # override the tag + newTag: v0.12.1 # override the tag newName: docker.io/lyft/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name @@ -75,5 +75,5 @@ images: newName: docker.io/lyft/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.3.12 # override the tag + newTag: v0.4.2 # override the tag newName: docker.io/lyft/flytepropeller # override the name diff --git a/rsts/administrator/install/multi_cluster.rst b/rsts/administrator/install/multi_cluster.rst index edaf120c06..23cfe60f1c 100644 --- a/rsts/administrator/install/multi_cluster.rst +++ b/rsts/administrator/install/multi_cluster.rst @@ -13,11 +13,14 @@ This protects you from a failure in a single kubernetes cluster, and increases s First, you'll need to create additional kubernetes clusters. For this example, we'll assume you have 3 kubernetes clusters, and can access them all with ``kubectl``. We'll call these clusters "cluster1", "cluster2", and "cluster3". -We want to deploy **just** the Data Plane to these clusters. To do this, we'll remove the DataPlane components from the ``flyte`` overlay, and create a new overlay containing **ony** the dataplane resources. +We want to deploy **just** the Data Plane to these clusters. To do this, we'll remove the DataPlane components from the ``flyte`` overlay, and create a new overlay containing **only** the dataplane resources. Data Plane Deployment ********************* +NOTE: + With v0.8.0 and the entire setup overhaul, this section is getting revisited. Keep on the lookout for an update soon + To create the "Data Plane only" overlay, lets make a ``dataplane`` subdirectory inside our main deployment directory (my-flyte-deployment). This directory will contain contain only the dataplane resources. :: mkdir dataplane diff --git a/rsts/administrator/install/production.rst b/rsts/administrator/install/production.rst index 088facfc23..67b9064f53 100644 --- a/rsts/administrator/install/production.rst +++ b/rsts/administrator/install/production.rst @@ -54,6 +54,9 @@ Before you create a custom deployment, you'll need to `install kustomize