diff --git a/CHANGELOG/CHANGELOG-v0.3.0.md b/CHANGELOG/CHANGELOG-v0.3.0.md new file mode 100644 index 0000000000..09502ec2e9 --- /dev/null +++ b/CHANGELOG/CHANGELOG-v0.3.0.md @@ -0,0 +1,26 @@ +# Flyte V0.3.0 + +## Changes since v0.2.0 + +### Core Platform +- Improved differentiation of System and User errors with Deeper visibility +- Support for Presto queries (no-op client to be replaced with custom client) +- Support for optional interruptible Task executions - on Spot Like environments +- Better backoff behavior on Kubernetes resource creation failures +- Publish notifications to GCP pubsub +- Improved filtering logic in Service Layer +- Configurable Scheduler for K8s pod executions +- Spark Operator webhook support +- Performance improvements +- [Experimental] Inclusion of an EKS archetype with Terraform files to make it easy to deploy and test in a cloud + + +### Flytekit (SDK improvements) +- Dynamic Subworkflow and Launch Plan support +- Inclusion of queuing budget and interruptible flag in workflow and task decorators +- Ability to call non-Python based Spark jobs + +### Flyte Console (UI) +- Added support for a custom banner to show live status messages +- Tweaks in execution list page + diff --git a/Makefile b/Makefile index 19ab05f437..4747b76925 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,9 @@ generate-local-docs: # Builds the entire doc tree. Assumes update_ref_docs has run and that all externals rsts are in _rsts/ dir .PHONY: generate-docs generate-docs: generate-dependent-repo-docs - @FLYTEKIT_VERSION=0.6.2 ./script/generate_docs.sh + @FLYTEKIT_VERSION=0.7.0 ./script/generate_docs.sh # updates referenced docs from other repositories (e.g. flyteidl, flytekit) .PHONY: generate-dependent-repo-docs generate-dependent-repo-docs: - @FLYTEKIT_VERSION=0.6.2 FLYTEIDL_VERSION=0.17.27 ./script/update_ref_docs.sh + @FLYTEKIT_VERSION=0.7.0 FLYTEIDL_VERSION=0.17.27 ./script/update_ref_docs.sh diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index ccaf510eab..57c87802a1 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -594,7 +594,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.2.6 + app.kubernetes.io/version: 0.2.7 spec: containers: - command: @@ -603,7 +603,7 @@ spec: - --config - /etc/flyte/config/flyteadmin_config.yaml - serve - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -647,7 +647,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -663,7 +663,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -676,7 +676,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -754,7 +754,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.20 + app.kubernetes.io/version: 0.2.36 spec: containers: - args: @@ -767,7 +767,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.20 + image: docker.io/lyft/flytepropeller:v0.2.36 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -874,7 +874,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 76b9b0caa3..cad7f92ba5 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -992,7 +992,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.2.6 + app.kubernetes.io/version: 0.2.7 spec: containers: - command: @@ -1001,7 +1001,7 @@ spec: - --config - /etc/flyte/config/flyteadmin_config.yaml - serve - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -1052,7 +1052,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -1068,7 +1068,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -1081,7 +1081,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -1159,7 +1159,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.20 + app.kubernetes.io/version: 0.2.36 spec: containers: - args: @@ -1174,7 +1174,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.20 + image: docker.io/lyft/flytepropeller:v0.2.36 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -1489,7 +1489,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index f21b8e183a..8d5940efb2 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -644,7 +644,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.2.6 + app.kubernetes.io/version: 0.2.7 spec: containers: - command: @@ -653,7 +653,7 @@ spec: - --config - /etc/flyte/config/flyteadmin_config.yaml - serve - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -704,7 +704,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -720,7 +720,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -733,7 +733,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -772,7 +772,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.20 + app.kubernetes.io/version: 0.2.36 spec: containers: - args: @@ -785,7 +785,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.20 + image: docker.io/lyft/flytepropeller:v0.2.36 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -1031,7 +1031,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/end2end/tests/endtoend.yaml b/end2end/tests/endtoend.yaml index 6703a1f0da..8a33a39916 100644 --- a/end2end/tests/endtoend.yaml +++ b/end2end/tests/endtoend.yaml @@ -11,7 +11,7 @@ spec: command: - bash - -c - image: docker.io/lyft/flytetester:v0.1.4_flytekitv0.6.0 + image: docker.io/lyft/flytetester:v0.1.5 imagePullPolicy: IfNotPresent name: flytetester resources: diff --git a/kustomize/base/admindeployment/deployment.yaml b/kustomize/base/admindeployment/deployment.yaml index 5372c24e0d..3f47f89628 100644 --- a/kustomize/base/admindeployment/deployment.yaml +++ b/kustomize/base/admindeployment/deployment.yaml @@ -16,7 +16,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.2.6 + app.kubernetes.io/version: 0.2.7 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -31,7 +31,7 @@ spec: name: flyte-admin-config initContainers: - name: run-migrations - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] volumeMounts: @@ -39,7 +39,7 @@ spec: mountPath: /etc/flyte/config containers: - name: flyteadmin - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "serve"] ports: diff --git a/kustomize/base/propeller/deployment.yaml b/kustomize/base/propeller/deployment.yaml index b341e2b18b..eabb4710a9 100644 --- a/kustomize/base/propeller/deployment.yaml +++ b/kustomize/base/propeller/deployment.yaml @@ -15,7 +15,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.20 + app.kubernetes.io/version: 0.2.36 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -31,7 +31,7 @@ spec: name: flyte-plugin-config containers: - name: flytepropeller - image: docker.io/lyft/flytepropeller:v0.2.20 + image: docker.io/lyft/flytepropeller:v0.2.36 command: - flytepropeller args: diff --git a/kustomize/overlays/eks/admindeployment/admindeployment.yaml b/kustomize/overlays/eks/admindeployment/admindeployment.yaml index a01718f823..b3261b9cb1 100644 --- a/kustomize/overlays/eks/admindeployment/admindeployment.yaml +++ b/kustomize/overlays/eks/admindeployment/admindeployment.yaml @@ -12,7 +12,7 @@ spec: name: clusterresource-template initContainers: - name: run-migrations - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] @@ -20,7 +20,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "seed-projects", "flytesnacks", "flytetester", "flyteexamples"] @@ -28,7 +28,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/kustomize/overlays/eks/admindeployment/cron.yaml b/kustomize/overlays/eks/admindeployment/cron.yaml index 435a5bb7a0..6b35cb696b 100644 --- a/kustomize/overlays/eks/admindeployment/cron.yaml +++ b/kustomize/overlays/eks/admindeployment/cron.yaml @@ -12,7 +12,7 @@ spec: serviceAccountName: flyteadmin containers: - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml b/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml index 4a76e86070..d426a6f992 100644 --- a/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml +++ b/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml @@ -17,7 +17,7 @@ spec: 'until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done;'] - name: run-migrations - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] @@ -25,7 +25,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "seed-projects", "flytesnacks", "flytetester", "flyteexamples"] @@ -33,7 +33,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/kustomize/overlays/sandbox/admindeployment/cron.yaml b/kustomize/overlays/sandbox/admindeployment/cron.yaml index 435a5bb7a0..6b35cb696b 100644 --- a/kustomize/overlays/sandbox/admindeployment/cron.yaml +++ b/kustomize/overlays/sandbox/admindeployment/cron.yaml @@ -12,7 +12,7 @@ spec: serviceAccountName: flyteadmin containers: - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.2.6 + image: docker.io/lyft/flyteadmin:v0.2.7 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/rsts/administrator/install/getting_started.rst b/rsts/administrator/install/getting_started.rst index fa56932bdc..e7c3d17fce 100644 --- a/rsts/administrator/install/getting_started.rst +++ b/rsts/administrator/install/getting_started.rst @@ -41,8 +41,19 @@ For local deployments, this endpoint is typically http://localhost:30081/console (for Minikube deployment, you need to run ``minikube tunnel`` and use the ip that Minikube tunnel outputs) WARNING: - The sandbox deployment is not well suited for production use. - Most importantly, Flyte needs access to an object store, and a PostgreSQL database. - In the sandbox deployment, the object store and PostgreSQL database are each installed as a single kubernetes pod. - These pods are sufficient for testing and playground purposes, but they not designed to handle production load. - Read on to learn how to configure Flyte for production. + - The sandbox deployment is not well suited for production use. + - Most importantly, Flyte needs access to an object store, and a PostgreSQL database. + - In the sandbox deployment, the object store and PostgreSQL database are each installed as a single kubernetes pod. + - These pods are sufficient for testing and playground purposes, but they not designed to handle production load. + - Read on to learn how to configure Flyte for production. + +SPECIAL NOTE FOR MINIKUBE: + - Minikube runs in a Virtual Machine on your host + - So if you try to access the flyte console on localhost, that will not work, because the Virtual Machine has a different IP address. + - Flyte runs within Kubernetes (minikube), thus to access FlyteConsole, you cannot just use https://localhost:30081/console, you need to use the ``IP address`` of the minikube VM instead of ``localhost`` + - Refer to https://kubernetes.io/docs/tutorials/hello-minikube/ to understand how to access a + - also to register workflows, tasks etc or use the CLI to query Flyte service, you have to use the IP address. + - If you are building an image locally and want to execute on Minikube hosted Flyte environment, please push the image to docker registry running on the Minikube VM. + - Another alternative is to change the docker host, to build the docker image on the Minikube hosted docker daemon. https://minikube.sigs.k8s.io/docs/handbook/pushing/ provides more + detailed information about this process. As a TL;DR, Flyte can only run images that are accessible to Kubernetes. To make an image accessible, you could either push it to a remote registry or to + a regisry that is available to Kuberentes. In case on minikube this registry is the one thats running on the VM. diff --git a/rsts/conf.py b/rsts/conf.py index 6b23aba6bf..d5c58eccda 100644 --- a/rsts/conf.py +++ b/rsts/conf.py @@ -26,7 +26,7 @@ # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'0.0.1' +release = u'0.3.0' # -- General configuration --------------------------------------------------- diff --git a/rsts/introduction/docs_overview.rst b/rsts/introduction/docs_overview.rst index b128dbc42a..770047a6a2 100644 --- a/rsts/introduction/docs_overview.rst +++ b/rsts/introduction/docs_overview.rst @@ -12,7 +12,6 @@ personas. User: I want to write Flyte Workflows ===================================== A user refers to anyone who wants to: - - Explore how Flyte works and try it out before installing, operating, or using it - Use a hosted Flyte deployment available at her organization @@ -23,11 +22,10 @@ A user refers to anyone who wants to: Administrator: I want to manage a Flyte installation at my company ================================================================== -An administrator is someone who wants to deploy, manage, and scale a Flyte installation for -his or her organization. The administrator is not interested in altering or modifying any code, -only using the system off the shelf, and configuring the various available knobs and settings. This -section also talks about the typical installation structure and scalability -primitives available in the system. +An administrator is someone who wants to: + - deploy, manage, and scale a Flyte installation for their organization. + - The administrator is not interested in altering or modifying any code, only using the system off the shelf, and configuring the various available knobs and settings. + - This section also talks about the typical installation structure and scalability primitives available in the system. :ref:`Jump to Administrator Docs ` diff --git a/rsts/user/getting_started/create_first.rst b/rsts/user/getting_started/create_first.rst index 8b504ad7ff..e6acbc7107 100644 --- a/rsts/user/getting_started/create_first.rst +++ b/rsts/user/getting_started/create_first.rst @@ -33,7 +33,7 @@ Lets create a new project called ``myflyteproject``. Use the project creation en Writing a Task ***************** -The most basic Flyte primitive is a "task". Flyte Tasks are units of work that can be composed in a workflow. The simplest way to write a Flyte task is using the FlyteSDK. +The most basic Flyte primitive is a "task". Flyte Tasks are units of work that can be composed in a workflow. The simplest way to write a Flyte task is using the Flyte Python SDK - flytekit. Start by creating a new file :: @@ -109,7 +109,7 @@ Flyte fulfills tasks using docker images. You'll need to build a docker image fr If you have the flyte sandbox installed on your local machine, the image will be accessible to to your Flyte system. If you're running a remote Flyte instance, you'll need to upload this image to a remote registry such as Dockerhub, Amazon ECR, or Google Container Registry, so that it can be used by the Flyte system. -To upload to a remote registry, use :: +To upload to a remote registry (or even local registry), use :: DOCKER_REGISTRY_USERNAME={username} DOCKER_REGISTRY_PASSWORD={pass} REGISTRY=docker.io make docker_build