From 3f7dd687647ac132465e8acbe56ce62dfa8b60a6 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Mon, 30 Mar 2020 10:46:01 -0700 Subject: [PATCH 01/19] Tracker branch for Milestone v0.2.0 --- kustomize/base/propeller/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kustomize/base/propeller/deployment.yaml b/kustomize/base/propeller/deployment.yaml index 8a6323d341..b341e2b18b 100644 --- a/kustomize/base/propeller/deployment.yaml +++ b/kustomize/base/propeller/deployment.yaml @@ -15,7 +15,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.13 + app.kubernetes.io/version: 0.2.20 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -31,7 +31,7 @@ spec: name: flyte-plugin-config containers: - name: flytepropeller - image: docker.io/lyft/flytepropeller:v0.2.13 + image: docker.io/lyft/flytepropeller:v0.2.20 command: - flytepropeller args: From d6554162636d60f6e244edd3864531faa0093420 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 30 Mar 2020 13:49:07 -0700 Subject: [PATCH 02/19] updating flytetester image to use the new flytekit v0.6.0 --- end2end/tests/endtoend.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/end2end/tests/endtoend.yaml b/end2end/tests/endtoend.yaml index 6e73aa4c13..6703a1f0da 100644 --- a/end2end/tests/endtoend.yaml +++ b/end2end/tests/endtoend.yaml @@ -11,7 +11,7 @@ spec: command: - bash - -c - image: docker.io/lyft/flytetester:v0.1.2_flytekitv0.6.0b1 + image: docker.io/lyft/flytetester:v0.1.4_flytekitv0.6.0 imagePullPolicy: IfNotPresent name: flytetester resources: From fe93129e78cde20a71ec985d1991ecf2c0a04e9b Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 30 Mar 2020 15:57:37 -0700 Subject: [PATCH 03/19] bump admin to latest --- kustomize/base/admindeployment/deployment.yaml | 4 ++-- .../overlays/sandbox/admindeployment/admindeployment.yaml | 6 +++--- kustomize/overlays/sandbox/admindeployment/cron.yaml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kustomize/base/admindeployment/deployment.yaml b/kustomize/base/admindeployment/deployment.yaml index fffae8bba9..bfd853949a 100644 --- a/kustomize/base/admindeployment/deployment.yaml +++ b/kustomize/base/admindeployment/deployment.yaml @@ -29,7 +29,7 @@ spec: name: flyte-admin-config initContainers: - name: run-migrations - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] volumeMounts: @@ -37,7 +37,7 @@ spec: mountPath: /etc/flyte/config containers: - name: flyteadmin - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "serve"] ports: diff --git a/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml b/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml index 0515b5c63f..f53ae6b651 100644 --- a/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml +++ b/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml @@ -17,7 +17,7 @@ spec: 'until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done;'] - name: run-migrations - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] @@ -25,7 +25,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "seed-projects", "flytesnacks", "flytetester", "flyteexamples"] @@ -33,7 +33,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/kustomize/overlays/sandbox/admindeployment/cron.yaml b/kustomize/overlays/sandbox/admindeployment/cron.yaml index d498aa5a29..a2ab5cb2de 100644 --- a/kustomize/overlays/sandbox/admindeployment/cron.yaml +++ b/kustomize/overlays/sandbox/admindeployment/cron.yaml @@ -12,7 +12,7 @@ spec: serviceAccountName: flyteadmin containers: - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.2.1 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: From bd42c06d79c7e9919af410f218e8b58a2997f538 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 30 Mar 2020 15:58:00 -0700 Subject: [PATCH 04/19] make kustomize --- deployment/sandbox/flyte_generated.yaml | 14 +++++++------- deployment/test/flyte_generated.yaml | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 0b85c69e97..08a46bf453 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -965,7 +965,7 @@ spec: - --config - /etc/flyte/config/flyteadmin_config.yaml - serve - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -1016,7 +1016,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -1032,7 +1032,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -1045,7 +1045,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -1121,7 +1121,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.13 + app.kubernetes.io/version: 0.2.20 spec: containers: - args: @@ -1136,7 +1136,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.13 + image: docker.io/lyft/flytepropeller:v0.2.20 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -1441,7 +1441,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.1 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 4d7d9b0642..003cd60341 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -634,7 +634,7 @@ spec: - --config - /etc/flyte/config/flyteadmin_config.yaml - serve - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -685,7 +685,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -701,7 +701,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -714,7 +714,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.3 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -753,7 +753,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.2.13 + app.kubernetes.io/version: 0.2.20 spec: containers: - args: @@ -766,7 +766,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: docker.io/lyft/flytepropeller:v0.2.13 + image: docker.io/lyft/flytepropeller:v0.2.20 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -1012,7 +1012,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.2.1 + image: docker.io/lyft/flyteadmin:v0.2.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: From affeb2b86fd9d94883acba923ab136c2402ed790 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Mon, 30 Mar 2020 16:48:33 -0700 Subject: [PATCH 05/19] updated versions --- deployment/sandbox/flyte_generated.yaml | 10 +++++++--- kustomize/base/console/deployment.yaml | 4 +++- kustomize/base/datacatalog/deployment.yaml | 6 ++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 08a46bf453..fdc867371f 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -883,6 +883,8 @@ spec: prometheus.io/scrape: "true" labels: app: datacatalog + app.kubernetes.io/name: datacatalog + app.kubernetes.io/version: 0.2.1 spec: containers: - command: @@ -891,7 +893,7 @@ spec: - --config - /etc/datacatalog/config/datacatalog_config.yaml - serve - image: docker.io/lyft/datacatalog:v0.1.2 + image: docker.io/lyft/datacatalog:v0.2.1 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -923,7 +925,7 @@ spec: - /etc/datacatalog/config/datacatalog_config.yaml - migrate - run - image: docker.io/lyft/datacatalog:v0.1.2 + image: docker.io/lyft/datacatalog:v0.2.1 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -1076,6 +1078,8 @@ spec: selector: matchLabels: app: flyteconsole + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/version: 0.4.0 template: metadata: labels: @@ -1085,7 +1089,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: docker.io/lyft/flyteconsole:v0.3.2 + image: docker.io/lyft/flyteconsole:v0.4.0 name: flyteconsole ports: - containerPort: 8080 diff --git a/kustomize/base/console/deployment.yaml b/kustomize/base/console/deployment.yaml index 64c350d498..df016b359f 100644 --- a/kustomize/base/console/deployment.yaml +++ b/kustomize/base/console/deployment.yaml @@ -11,6 +11,8 @@ spec: selector: matchLabels: app: flyteconsole + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/version: 0.4.0 template: metadata: labels: @@ -21,7 +23,7 @@ spec: emptyDir: {} containers: - name: flyteconsole - image: docker.io/lyft/flyteconsole:v0.3.2 + image: docker.io/lyft/flyteconsole:v0.4.0 # args: [] ports: - containerPort: 8080 diff --git a/kustomize/base/datacatalog/deployment.yaml b/kustomize/base/datacatalog/deployment.yaml index 4bffb41609..72d4dcd0bc 100644 --- a/kustomize/base/datacatalog/deployment.yaml +++ b/kustomize/base/datacatalog/deployment.yaml @@ -14,6 +14,8 @@ spec: metadata: labels: app: datacatalog + app.kubernetes.io/name: datacatalog + app.kubernetes.io/version: 0.2.1 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -28,7 +30,7 @@ spec: name: datacatalog-config initContainers: - name: run-migrations - image: docker.io/lyft/datacatalog:v0.1.2 + image: docker.io/lyft/datacatalog:v0.2.1 imagePullPolicy: IfNotPresent command: ["datacatalog", "--logtostderr", "--config", "/etc/datacatalog/config/datacatalog_config.yaml", "migrate", "run"] volumeMounts: @@ -36,7 +38,7 @@ spec: mountPath: /etc/datacatalog/config containers: - name: datacatalog - image: docker.io/lyft/datacatalog:v0.1.2 + image: docker.io/lyft/datacatalog:v0.2.1 imagePullPolicy: IfNotPresent command: ["datacatalog", "--logtostderr", "--config", "/etc/datacatalog/config/datacatalog_config.yaml", "serve"] ports: From db4860630cc0e03527867eb3659b8d67c974cb10 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Mon, 30 Mar 2020 21:04:52 -0700 Subject: [PATCH 06/19] Fixed deployments --- deployment/sandbox/flyte_generated.yaml | 6 ++++-- deployment/test/flyte_generated.yaml | 2 ++ kustomize/base/admindeployment/deployment.yaml | 2 ++ kustomize/base/console/deployment.yaml | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index fdc867371f..4ac50dc7bc 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -959,6 +959,8 @@ spec: prometheus.io/scrape: "true" labels: app: flyteadmin + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/version: 0.2.4 spec: containers: - command: @@ -1078,12 +1080,12 @@ spec: selector: matchLabels: app: flyteconsole - app.kubernetes.io/name: flyteconsole - app.kubernetes.io/version: 0.4.0 template: metadata: labels: app: flyteconsole + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/version: 0.4.0 spec: containers: - envFrom: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 003cd60341..e544cdeda2 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -626,6 +626,8 @@ spec: prometheus.io/scrape: "true" labels: app: flyteadmin + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/version: 0.2.4 spec: containers: - command: diff --git a/kustomize/base/admindeployment/deployment.yaml b/kustomize/base/admindeployment/deployment.yaml index bfd853949a..1823270dd9 100644 --- a/kustomize/base/admindeployment/deployment.yaml +++ b/kustomize/base/admindeployment/deployment.yaml @@ -15,6 +15,8 @@ spec: metadata: labels: app: flyteadmin + app.kubernetes.io/name: flyteadmin + app.kubernetes.io/version: 0.2.4 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" diff --git a/kustomize/base/console/deployment.yaml b/kustomize/base/console/deployment.yaml index df016b359f..021e5a4d58 100644 --- a/kustomize/base/console/deployment.yaml +++ b/kustomize/base/console/deployment.yaml @@ -11,12 +11,12 @@ spec: selector: matchLabels: app: flyteconsole - app.kubernetes.io/name: flyteconsole - app.kubernetes.io/version: 0.4.0 template: metadata: labels: app: flyteconsole + app.kubernetes.io/name: flyteconsole + app.kubernetes.io/version: 0.4.0 spec: volumes: - name: shared-data From 0646171fef5978331408fcab3ee2f3c87bd3913f Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 11:36:23 -0700 Subject: [PATCH 07/19] Try generating release notes --- .github/workflows/release-notes.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/workflows/release-notes.yml diff --git a/.github/workflows/release-notes.yml b/.github/workflows/release-notes.yml new file mode 100644 index 0000000000..09fa5be5aa --- /dev/null +++ b/.github/workflows/release-notes.yml @@ -0,0 +1,4 @@ +name: release-notes-generator-action +uses: Decathlon/release-notes-generator-action@v2.0.0 +on: push + From 3b3e5563cb6969d711387c6e716a4226b17da194 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 11:49:30 -0700 Subject: [PATCH 08/19] try --- .github/workflows/tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 37464e907f..380df750fc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,4 +23,14 @@ jobs: go-version: ${{ matrix.go-version }} - name: Run end-to-end tests run: make end2end + create-release-notes: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Create Release Notes + uses: docker://decathlon/release-notes-generator-action:2.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OUTPUT_FOLDER: temp_release_notes + USE_MILESTONE_TITLE: "true" From 53081247cee15b7d143d1207af2c30dbb91a9747 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 13:49:59 -0700 Subject: [PATCH 09/19] updated --- .github/workflows/release-notes.yml | 4 ---- .github/workflows/tests.yml | 11 ----------- 2 files changed, 15 deletions(-) delete mode 100644 .github/workflows/release-notes.yml diff --git a/.github/workflows/release-notes.yml b/.github/workflows/release-notes.yml deleted file mode 100644 index 09fa5be5aa..0000000000 --- a/.github/workflows/release-notes.yml +++ /dev/null @@ -1,4 +0,0 @@ -name: release-notes-generator-action -uses: Decathlon/release-notes-generator-action@v2.0.0 -on: push - diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 380df750fc..f7be948dc4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,14 +23,3 @@ jobs: go-version: ${{ matrix.go-version }} - name: Run end-to-end tests run: make end2end - create-release-notes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Create Release Notes - uses: docker://decathlon/release-notes-generator-action:2.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OUTPUT_FOLDER: temp_release_notes - USE_MILESTONE_TITLE: "true" - From 07755b3a66ed64547c98f9d3c6f65ea9d3216eec Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 16:50:32 -0700 Subject: [PATCH 10/19] README updated --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 5665f674aa..27197c15ed 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,12 @@ # Flyte +[![Current Release](https://img.shields.io/github/release/lyft/flyte.svg)](https://github.com/lyft/flyte/releases/latest) +[![Build Status](https://travis-ci.org/lyft/flyte.svg?branch=master)](https://travis-ci.org/lyft/flyte) +[![GoDoc](https://godoc.org/github.com/lyft/flyte?status.svg)](https://godoc.org/github.com/lyft/flyte) +[![License](https://img.shields.io/badge/LICENSE-Apache2.0-ff69b4.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) +[![CodeCoverage](https://img.shields.io/codecov/c/github/lyft/flyte.svg)](https://codecov.io/gh/lyft/flyte) +[![Go Report Card](https://goreportcard.com/badge/github.com/lyft/flyte)](https://goreportcard.com/report/github.com/lyft/flyte) +![Commit activity](https://img.shields.io/github/commit-activity/w/lyft/flyte.svg?style=plastic) +![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyte/latest.svg?style=plastic) Flyte is an open source, K8s-native extensible orchestration engine that manages the core machine learning pipelines at Lyft: ETAs, pricing, incentives, mapping, vision, and more. From 055d7102228967319947d268e4738ce45b740a1d Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 17:04:01 -0700 Subject: [PATCH 11/19] more shields --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 27197c15ed..ebb2a84ddd 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,11 @@ # Flyte [![Current Release](https://img.shields.io/github/release/lyft/flyte.svg)](https://github.com/lyft/flyte/releases/latest) [![Build Status](https://travis-ci.org/lyft/flyte.svg?branch=master)](https://travis-ci.org/lyft/flyte) -[![GoDoc](https://godoc.org/github.com/lyft/flyte?status.svg)](https://godoc.org/github.com/lyft/flyte) [![License](https://img.shields.io/badge/LICENSE-Apache2.0-ff69b4.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) -[![CodeCoverage](https://img.shields.io/codecov/c/github/lyft/flyte.svg)](https://codecov.io/gh/lyft/flyte) -[![Go Report Card](https://goreportcard.com/badge/github.com/lyft/flyte)](https://goreportcard.com/report/github.com/lyft/flyte) ![Commit activity](https://img.shields.io/github/commit-activity/w/lyft/flyte.svg?style=plastic) ![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyte/latest.svg?style=plastic) +![GitHub milestones Completed](https://img.shields.io/github/milestones/closed/lyft/flyte?style=plastic) +![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/2?style=plastic) Flyte is an open source, K8s-native extensible orchestration engine that manages the core machine learning pipelines at Lyft: ETAs, pricing, incentives, mapping, vision, and more. From 34f6d2edf681e644c7fb4f20dc29a520ff465ba9 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 17:55:49 -0700 Subject: [PATCH 12/19] Updated description etc --- README.md | 55 ++++++++++++++++++---- images/flyte_lockup_gradient_on_light.png | Bin 0 -> 17200 bytes 2 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 images/flyte_lockup_gradient_on_light.png diff --git a/README.md b/README.md index ebb2a84ddd..37d803bf76 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# Flyte +![alt text](https://github.com/lyft/flyte/raw/master/images/flyte_lockup_gradient_on_light.png "Flyte Logo") + [![Current Release](https://img.shields.io/github/release/lyft/flyte.svg)](https://github.com/lyft/flyte/releases/latest) [![Build Status](https://travis-ci.org/lyft/flyte.svg?branch=master)](https://travis-ci.org/lyft/flyte) [![License](https://img.shields.io/badge/LICENSE-Apache2.0-ff69b4.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) @@ -6,23 +7,59 @@ ![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyte/latest.svg?style=plastic) ![GitHub milestones Completed](https://img.shields.io/github/milestones/closed/lyft/flyte?style=plastic) ![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/2?style=plastic) +![Twitter Follow](https://img.shields.io/twitter/follow/flyteorg?label=Follow&style=social) +[![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://docs.google.com/forms/d/e/1FAIpQLSf8bNuyhy7rkm77cOXPHIzCm3ApfL7Tdo7NUs6Ej2NOGQ1PYw/viewform?pli=1) -Flyte is an open source, K8s-native extensible orchestration engine that manages the core machine learning pipelines at Lyft: ETAs, pricing, incentives, mapping, vision, and more. +Flyte is a container-native, type-safe workflow and pipelines platform optimized for large scale processing and machine learning written in Golang. Workflows can be written in any language, with out of the box support for Python. -# Community -Home: https://flyte.org +# Homepage +https://flyte.org +Docs: https://lyft.github.io/flyte -Docs: https://lyft.github.io/flyte +# Introduction +Flyte is a fabric that connects disparate computation backends using a type safe data dependency graph. It records all changes to a pipeline, making it possible to rewind time. It also stores +a history of all executions and provides an intuitive UI, CLI and REST/gRPC API to interact with the computation. -Slack: [https://flyte-org.slack.com](https://docs.google.com/forms/d/e/1FAIpQLSf8bNuyhy7rkm77cOXPHIzCm3ApfL7Tdo7NUs6Ej2NOGQ1PYw/viewform) +Flyte is more than a workflow engine, it provides workflows as a core concepts, but it also provides a single unit of execution - tasks, as a top level concept. Multiple tasks arranged in a data +producer-consumer order creates a workflow. Flyte workflows are pure specification and can be created using any language. Every task can also by any language. We do provide first class support for +python, making it perfect for modern Machine Learning and Data processing pipelines. -Twitter: https://twitter.com/flyteorg +# Features + - Used at Scale in production by 500+ users at Lyft with more than 400k workflows a month and more than 20+ million container executions per month + - Centralized Inventory of Tasks, Workflows and Executions + - gRPC / REST interface to define and executes tasks and workflows + - Type safe construction of pipelines, each task has an interface which is characterized by its input and outputs. Thus illegal construction of pipelines fails during declaration rather than at + runtime + - Types that help in creating machine learning and data processing pipelines like - Blobs (images, arbitrary files), Directories, Schema (columnar structured data), collections, maps etc + - Memoization and Lineage tracking + - Workflows features + * Multiple Schedules for every workflow + * Parallel step execution + * Extensible Backend to add customized plugin experiences + * Arbitrary container execution + * Branching + * Inline Subworkflows (a workflow can be embeded within one node of the top level workflow) + * Distributed Remote Child workflows (a remote workflow can be triggered and statically verified at compile time) + * Array Tasks (map some function over a large dataset, controlled execution of 1000's of containers) + * Dynamic Workflow creation and execution - with runtime type safety + * Container side plugins with first class support in python + - Multi Cloud support (AWS, GCP and others) + - Deep observability + - Multi K8s cluster support + - Comes with many system supported out of the box on K8s like Spark etc. + - Snappy Console + - Python CLI + - Written in Golang and optimized for performance -# Repos +# Current Usage +- Lyft Rideshare +- Lyft L5 autonomous +- Juno +# Component Repos Repo | Language | Purpose --- | --- | --- -[flyte](https://github.com/lyft/flyte) | RST | home, documentation, issues +[flyte](https://github.com/lyft/flyte) | Kustomize,RST | deployment, documentation, issues [flyteidl](https://github.com/lyft/flyteidl) | Protobuf | interface definitions [flytepropeller](https://github.com/lyft/flytepropeller) | Go | execution engine [flyteadmin](https://github.com/lyft/flyteadmin) | Go | control plane diff --git a/images/flyte_lockup_gradient_on_light.png b/images/flyte_lockup_gradient_on_light.png new file mode 100644 index 0000000000000000000000000000000000000000..1f986f45adae87bbf319c33f6cca89bba7a6eb7b GIT binary patch literal 17200 zcmeJF^;eYN_Xdm)f^;L@O83wqttc>n4&5+xBPlsZD7*j#85$`Wx>H&d>Fx%l8|j*v z=l1>m{teGs&sv`!;GUVaW}kEKz3+3beeLT^%q#5|B!movAP|T|{pB+#2!t&P0%7^% z;{uYI#W;P(gO-(vH~Ki6nFL;mrV~{7?>zf$|JKH)4n@~KfN>QU zm!;8P?v)nw5{3F3+u=S0)27|vNw4;nD9BGP>rb0c>ld|XU+v$p5u_jOR#qNCa{S>6 z+e{@0N?>xJwa=-4U9o@{o+QfuCLQLD58MF=l%<;R@6$D=aN>V&x5;h)U8Fy-VFQ=% zGC1`A-j1;M{kwRQ{@+*s&rSa?r9@j+QF-hAC|S;dp$6$L9uk4pdW5*51;4osa2T=s zYXv`TDAL&K7}SVj-Xt$ob5e*EncC3kn;#4{_he=C(gw08;y9tll&QS(eDvbro*Z=fJhnpBkML%1BWo zW~j=!K5pSe5RXKJ`J3Ct5rnaP2j+fHw=}R5t=u3rmksWfbQ79{3$^Wjt>U|LFV_t5 zJ-5T>kL#^^sikF(&pdR!pv{Wd0hN1K;OUrKzF=P!p^lO*_PUyPnu!RpWoecOAZYBeAT!fHd6?j|$^aSLDp zR)IZ>XY{rVxPxeApUvY~jee)9@F(5p&4z7{{UdhpDpBSQKk{=rm37${$Myo8HI?6#vj!r}`=rnIfK2gbGa8!D zgP^Nq4O<`r-;RonjEeLHZ}KvWN)o(pBCed$xgK$>J}kcoLM?{ z)0K5OC-rGi+wTi|Y1RW`7;X1PwD43QS`t0}t`be{Mg-|ze|lKUVWA`-P<1$4(z|W= zC6T-`e2Pfq2jtHsiP_hO>3%A4ED6~GX!xzex6 zV9q?g`{(cULsT$WinN7O5d!f!y*FQ@|USK#sd+@3950RcT|LdFF#|v!S*j=ocC$p7o%Utt9xB{EXK3K?wcs8 z7?SkX_UCCCE^60-9R5pTEASGn9?EwH+ruk0RzN>ef$W5}|FY`1T#`tm4UEuWKi|Oo zIJtaB(Ed54xDv|6cV-9qzDuO=KFp?W3Yu~)bF!>+DQE}SlBBSOrf6syKd{Dz`fFRw zv2|9Y)Gq7~6L4NUCbmdRXUG(eJD@w+vNtngNE6HJA(CAE_BWlK%2^F^9TR%3e5Ipl zymqCJVFiJ5nT6NOk>x)H?V6<@t~^EL_R=!d?(wnoAI-wGQzCwZLoFoK-dd~0>;j4X zowG!^#V+I~-+W(__SmFqam)apy^)8LyFH%eIqJqe*y8i)s{1|pdS|?=$-sTnqHK48 z?+UPJ9JBWoR8@Q#n#2d^DR=pMr0{}w6yGP!A6-x#Hy=*@3369#u;h7@0)^!?joZX+bDq?OIYU~*N7cXA+*Hz{+i=>sLO$hf z1pv9$N2$zZr-c`yRSJ#~XG}0Sg4=YA8kcGc;+c6ki6|_4wv)7IoSj@cWV}x|IbQa5 zzVSGc_C5DFda6HKCZmO4qlFJkAerBjf5q640Z-`| zO*mv=ru_*1e6`{!G5~O0-tW6Qgt4Zv&otHqU@Y z&wB)ghebT4DIx(6HTwL}=A}mt8wO%e+LaE!&8g;j65tj}blwa^wT72OV^>=T4iGUZgCHBC;;NBrWugJ?1HxfTk$b*$@R; zFsMR??eqY6t7M8V>K9pQMkBowOAq^FU?VyzA2?lGZDV@d>Y=pg*?!U$%$R~ZNc)jw zh?jjETLp3vD#^oH#QN>hza$E{6*zh$>fgVZPSl5__s+D0C+0OV^{FQAMGTXbx!b{4 zpLFn1O%DL)6D#-Nat>Her#)LB2H)1sf54O_{HCWn33*;X1kO7(wQ74)z2tmS zzBn{s*?H-&+Mx+OO$^e>xa(`@j)$KUag*?}k5FnHOeTL#aWg%1zRc(^MhbxVQ?Ni` zjlhCXTt9fN3R%k))&>G0P$p`Py;VS#JO3|@9^kESzF*8B?2JbyNaSLVDO$J*59x&* z#BwJqULup!gi)vJ08$}BojH}h6TfbI-(l}um8z6+24irKa0UtjM@HqIaNU)i3bRI| zpuu+$9mFOBPy&iK51BI+c_Ph(ZNFz7yxj|rYx)Ua17Clv(D{&lS5?buU#idnU!Lma zp3&c)bS-mlo9|DlefC2YvKav6jU$zE&u3(p*M!c`=cnXJ17YI%y~ijm)Y9iI?+PF}rS(E4dM$6y9W78t3 zm8Y8c;CZ@_rt01nkdqsCblbDH{S^OVy=XB~Qui#NwT~>eTG_JO8S?l7doT}z`%!}! zxi~TPFR9wcy56qYNDhjSH4z%C<&gh5nXSA>rChCs3`8FH2;6mt?E5?~c{_TSJff<* ziGbd(sQt8RvsnB7=a1>^=);PP%}!R!Z%gpinN{vvacL6M&d@*SGkDqp8O`E04OI== zO^D~>)4SflTDAm$fZ|YSK%6rqGb4(Qv)f9Ito6_?TQ`eVw|(URGcPmdx_w>JlR!Pb zQgy$&x}Bdda=W$=HI6r%(kNf=_lmdXIX-y375WUTDn0CM`96Sva0aUh3}4jIcD!r@ zvZel6LXCQpFagGDQ4w?=vbSmgtn`p@$Y*B=-xa?z-n3{nL*2c+*MPt4cuwkYlHt?L zvddzARtvc|n?GHA2j`77`uf+-n3>8ar7;w$aiJDzs(@L_Cw)fZo+CMs=jKj)w$)j{ zPRO9lUQ615h;o#*3LSMTGFZSY5T|Oy3{&O2ZEV9(joOQ1cuIAcjZ8cCzGvh^|A@1f zGP+5Ufvv03*Y;ly%gy(DRuAe1cX}n{MJA;3+^YoUG8RD6a+u)^apQzVSWsRLrplXMpKR;6 z(`VQE4L@-?cXp5+LbwO@zBVrz$0&~FXHai0{UQP1HoaLr1Sz3+$m@Piyr!@XLyVc& z&pSoU;x?wv&$V|>bMMpdjw)-WM=x}ougDjrl(la6l7LBiXNqY3uP2bB@z+NjFBDK; zrm1G-TUu&Gl$O^?Lx;sfEEaw`1Hgoh8Yi8&cp#foee2Pb6L?!AU z(;&~E=B>>_{`>ch{zZ68evxDs4hVEfZY!GWBc}y?eiK?Qh+9vvjdSC!bFC7G#uz2_ zCyR{gOgTeb?mAD#x3kj5LVsQv>MULBlwI7H2)@3Q*C2<%6#oT$7_f6PAl2D;;MS2g z)0QS#3f>A-m1I5qq;R$ve*6~d&Kt`!6L#}&`z``@CRQD#XeTdMl21-&?W_Urdn=tG zLWcYqQO$qx%DCKbyGr*g#LuipH6NS>g*JwFc>WLBL)6-IP)RRYVCS36jCw^KLGZT{ z_NV!#Gfk5nQ{wkH%MOt?j5BBUXzM}NS`)BP4Nw(i%%SH>H3ri~1RbzG$@3W9jcP8R zLA^GafJsY{+t&UWPD)%Uz~sKO9(0C0kWlVA3NwkxtHyGxT_hnPjAvKlFrs#OywjMP zoqFl^N(F)GxW+4+uC70nuGhem3mLLeiTM@DZVx`?^8zeVP}qu|mNvEd zir)&mSj3lDOZ4%u-$ooMzLDT+NssmWm9f#ef%4C(WkX2Qowte0H`1^V*8jX9U?+u# zp0Rp$;wx25AUC+$yM6oCU4kvRuAX*jqk&`&!AXzf zMU}v51nt;E?~{wl8&Y}oOZp1Np)>FGxRQc2af*PKS)_=~(Kbv#(bllu;cXLDp+D-5 zK(%nW+`Vr0e@O1L;{UItteIT(E3^pr<2tv{&7hs+;F-(fo=jb6OQTWFv;N+eR?krP zEanZ7pq=lm*yw&3^v{X>anMRhfLMb&>WWH(t8FAwug3l;mjPh+1laq0!kQkr83C{% zTDeOj;pTV8Y9Z9z@kw`dx2Ear^_YCokOWDF^`EG;g~3bAa!`)H-`BonMFnej0O5?? z;h7glnic#rRjZP)8z#0Ax>Zl<#pPPP=q|za*pxaQ4XmbuTt;3pV_reVV>b8%#_C5p z(ZhEuXTbsYpA}{6Rv}*(L%qVYEeoWkvc$Ka8QRQ&s6Zei)r8$Fq)r(sB9LFuZhGH1 zJv9%(L6K#YqSJ~EzD&nWPymyUq zPhRARr1QOa^Do)kvbIaaMxJ8gcDXxI7$2g^o&r2_F`hXSL4Ne%?WbX>J`W1mmhei$ zioxAy_p629ch0Ea+@xUN-wmR@J7X)cXz9j7Mwc&cgA1yhg;eLGpDkpLhgu(i-r|dX z(bv;>ddN7o;KW;|!Ibr2*BpiYvBIfO!^eNc;}Z8SaH6vmC%rl$`Q(xjfS}GEu-@fk zw2d+^_!OXg?4GDU16Lqm+LWN&qA`;h<*N3zxc4D;<&6lz5KL$24IElEz{80baXhK5 znLak1W~ysaBJe$B^h-H*cR@o+?TpV_R_nfszv(b=qppwpd@M@d2!AY8!5w_98rS?W zK0p+^qY4LKzVUnSBPQ68ow0!F0VYxoCvqsYa=Yj>3!t=wMqe&Ib`Pw1LL_K>J z?Xj9)6+IkJKa!}#oF39Lz461=0?27`3R|x=ofq1cj*Vt+bzNRVDYbicPNS`k7QdbR zT;F}-W$PBOl=S;su`g!Yu=C_9Tf731_E#RaRnX2Ft5OHQ2GdCN5wd*w@oS?&4sc_h z50tsM(dR!Vi+tEn>QblKYg!Q)$vEKjdci9W4tdi@2ci`t_|z~1&d%SEZA ziszP3lz+3;(y;dsr+k>0Be)OpSCHoj#=acWcqNV|uPfHOX37@`KG5XWy8rnhhyQtr z{mrPqv!ES%kgw@b-@!d#za~T@dO}Ol=oeQ8D6EnLIc3h)&{_JPfkv#9d3{J9&WjYy zbRfFJ7Y!&Va0kmVNf%R7l!C_#By{X~E>c80CpcT{GS^r&`ae^`Ix*mBLmSHF=lA5g zrtcQ{5OnhaM>T&gdku^&vl&33%mM_;{#|}J0Ux8ylKGl1^1O#$tR=H6@{1Gl5i3ac zKei-Q2lLa6Q!29iD}{D=^a+Vs5&)Ki_f~_EH9INTuLoMMG6&DI!Oe5zrxU1sE;O(m zEyK3-?ZQ*Uk)Y#s#&PNK3GGYoHht+Z@(0GExPbsww ztT{$Ss$FpaSrmC})<6LZcUSE+<_mD_?OE5A8<80quvgMf!ueA>xW-@Gucm? zW0(c+FFZM!{2XuI&B*l5+9p-@!o*M#mIUaCI`4d>>^8or%;olRh?cPjC>_Y?!&xeQ zyN;mVNtA}i#N4K!U3}XzdeW6w(@Ws-f#1G`*iVh<=0gwJqm4D(9cavDk+U`1i$l#f zb1dS1EezPIhE|ZU02BP0dDrVKsTQwBOATYc^Hp6Pw!V!eOz5i1IoZm5^;ZZv?DJ@n z`ZY>R1;S!FkYa|WuHy3+IYg!`mh^Pzu5w(i_+0)yjdx((#uFV2kc^1yV|z4bb(ThZ zagVoZm?*=$SzPkn)o7uA)5}x@i2qkW1Uy= zPWfp~DYa^R>icRy%^U9i_3|!cTDV4;=grRh&S2|5@1;7vliB8~4n_Wnpg9idMk=SK zZ+Lhg^*!R(^-am{t}0_Jj1u@noo3J$vN#08=_3e_!?w6W8}8H8jjz3P37V#RGS5WM z`5_#4(6@I>E*k)?NzBXFqWL;YP3|KucSIqu|pDr`ondg}sY&8p4-N zNY`_qCY8I2nWw(WkwzLCx#-8MbHtOcSM}g5y`Spx!AQW-u$v6Upu~q*VXkv5_|319 zYXagRr^g_%PX$X4D{L#s?PcTG*3+b1XU^<48Y2bm8n74q@XE(KMx}~LiNS#uadnFL z2(qjuvB~_ zRPM_z!ecuJO^JlA94!dofaV;0C*UL8T$2Vf&jUre_1x^R!baHoqz5t|S*9}`6EKRr z7`yrz(iy+3S`q0a=t2WCF|;}VJn<{m!`HzAl2$>MWmF2`(c*m;4KW=EFST9hk{jfA z><0-90uvT86c$UmcVl5px)-y}9xY)Z2J7D$XUGKM(7!-Uf3R$@l8iSaW=c>i>(Q&C zEeqL^Yu(1pMCB5)=;4v3#0rhX60FO!DsOl8pHTuU>|3T7)wm{)#7xFLtjb#BWcrca z0!*U)F(+>>LsEs)pj)_58fIBA0#&(HOnutIN9P{CpF5%dsn5}asa9G;VDLw>eaWZr zv}GXYztt(8^0hn9H135pYyIW&;RipWfraC)W98iHShTr>7@}1>Pk%Suz8St2iheu{ zEF2X*dXNbyu5H;Y7b=s6DHGdL1)^4fc#?93N}&OjxPZ~qX5{d8?qA=z;C{KbJ0 zvNE9|h!54X69up^rTvTGRogRs_MX;*JAZk5LC7bd7WkhQfcb_*yJVoq(XAe_8$kPH zS){o$T_?o*FpE<3Z3o0CTHPdGJ@=fWWI9J4 z_D{;v>>}7EIVFLLf~XKmxvy1iPtM5O7wj^d<=+l~5}J#pj9CAQ;~1&bGSc$#3gf$A zT5)_?5921p^qikVz(f9f&=`%o(%^pItKQ zo{L!D#AxXs&`_FK{+R#xW@w^QcBtlPv8Uf`* z1@AlC_X&ptyyiuku8oMYiU{4h&ws=!6}@b4TV%DW;x$7LoaT+9?Vs@m6z_)3U`pUq zGqoxW&g4HEWBr?3z2TG^F%)@`7Q^MjhbgtTcw3pYKgmql@Xm$Kp#$o%NkuQ@jB7-^ z`CLQ)%elH@K%Gene<5&dqR`~kq!(&vb5xo9hQ9jUD*m{xXslS~^7YxEpQ%|QVV*fo zx={TtRaF*x+#$3~v*Ip-#atK_q`5^TX>(D@#Vhqw%}w+tzGO;ES8tU4_Lx{t|9UMnp$GJLcgUUS(dwu=wXbp zAmLYj9wY}=VPbQ|XJgU8fz!PJM!EarihT;(mZ?m|oBv{Zk8TcgU%_6)4b%AQbDgth zN-1)MM(^I}+*aMpWb7^qN!LE-X;w_0?tk`!To>N{dh3@@RDRiD$lfLSDs51fdtxGV z#Pq&9s6Im9i0h_x^pc7DH7a3PAj~={pn^DE$;SI;M;jfq9|+vF-w|~c56W}Z!>Si;VeJdeZ9z)>{uIS1rV9?(V-vwQQ@@13fFYwa}|&V+-Z}RypG8uYR-R zim_KFfv=d;xox;UECzo$JriJTTwQeFcRSeCx6T;%b}Tqw*mR3Ze>$YOZvweI_VSp& z{V!iirqVZhJYsVzCyvh;AFW_1p>|BI`_0kNU$~BG*7sAT5T9K(QDTUuF}8ejPAacQvVTp3f?PLSjvTXX(D2bzY{M*wi}zrN;Q3B zw$_v{pwa2v-I1*TP4rcPmRi#9*l5%z8Epj>y zRQ?KAx?s=`AjAd&6V%b^)yDcfo7Q&1$SK6UqnH~uJ=}9lEbU-bb>*^TlBUCdds4O- zB4z9@BGM;t%$gJuoqoMhhf;~jdyjRO3x$bzQa9W-bKB*0tNipGNL_8J{2Fn?h;h33 zA?P7bb8S}8G3B)x7%SVi@+OUX0Mt8|b#6HXyLwZX&ysY%?7-i*bEB0A;R3RqNC*iq0_EH@SA$(1x7mR(qleD8nH&$Wby(QZI23ct~%*%N(OQXhW=Hj~s zKQsM!!LLssR-$KP@wj^<8R4~XsurdV>%|8x5H*-{oYV0{Z zZ9KD|z0^zBqVhMAUYW9dVqASMTaRXl)6{6TvX0qRuPX*YpFJWJIhb2}K|PYj}OdduKaj(q&8$&yl4LoN5bUrMWZaF&Z+N_Fp41q{w) z3vf{&Z2mTfC3xn$mwx6Hc`WZQy;P{dd*Uq%&PBcw^~_0^GzE5)o6dCt72-ANOzL=RW=R3< z+QxPhDrGLl#&Afs$$;}bzNobz>SY+)|{tEo6_2hq0@UShi6?w|(A7KE)J$yVLe+f=Hr8;;@jllJH*H zAj86V;mydKkamK+wnZo7c!FHRLdNFx80xkz?B$XvJVg0oJ{4_HU1lw=CO1Q>7=zdXUrA# zhCJ)DYE}RBgklv{umYX&Y!5sBbWu@=~MTj>QBT^kK3$0yY8{ej$|zlT`JKaQ*lh^$%)qJj&oP623$6ct^2x)wvHhw z3=;})rCrl$VT+`s##?v=?oZ_O_?QgkBw2C*-~1gv+-723xEbXi+zLJ>3PND@N;Ymg z@T=j2m9D_vMy{kwZjq#!XEvv@P5F%T3z$e6en4B&_1S+}`kNKd@W8ozS2EFNVeQ&| zK&1;++1neO)ZoklZ8L5-tU^SR#(T ztkwgAWdVXpB>_pi<0tk}HF__3V^d0B=+`IP^%^Af2i?`huiJdUAjVDJ%l=%%qXAOm zg3v}+qY12sJ7UMVv9iXex!G!?Z$o*q>V7F`rydoV*3z5n!}tXx@|BTU-%#9;9lXG}%7KHX z^Mn_Z_@dCjpvd{9`($TGqpQ{{2N@XPH05stD?C4^yMigZB7#i@!aJ&jZ1qhB+LA|x zaFkK~b31N|GVdfsbEUM>MQpyhxkQk%^Okei1$|abLe~9K`?#AokR%#TtitO-8#+(^ zbfK(=W3$uT|9?6R(e5E0&i0eq{Z$>Kz@HsQ7s5N7l~+X69ROcPD?ZRwut}`5_S}s9vO>RmmP-6u+q3Y)wf%BBgc> zl;EamT#L8Xx_mQWMk}MAVL=}d_KF0o5u@PkdImYcvjPBj%@=u>Ar6f7>cibUhNK$t z)D2u$Heh%~kX~i_b4}yN9g8krt{wYPk-6Wd)BZEol~130Wu~0|64NQM|c&_;CDPN-amYyOc$AI<*FbhqJIzv9a`U zp#Q?d$eG)Pso+HIBbztzX`m)<5*h$|M|;tdhWs{2HIM=hedJ_?-D3+0u0%wt$>pkA zq%$PQXVXq))W3Nb@9;w^A2Uo3c{)M+iWgQ*m^*M;fa&|KtYKU&-4EQDfmA`bjC!BM zzZkO*qyrk2giGikK|HX?knoeu{MFbes|1o&6swv#2 zXO8Ki{ZEkV7#fc?)(6ZdMbUpK($h4Vy=w&w`ay0xhJdQ*Kv^Au(uWNw*dm08h>ic2 zDxKL=UUd6@DrB`NZ6p8M9g49jUQ9r+Vdo|et79^eh_4i-07n9@YvEhF;|t5?#utsw zEk`h2eu(7)^g-=FsksQXaBlv+9~)TPK?%DDAgc|EL>v}9jv0SKnyU@C_Qm7!Xa}*r zSMgZn|FK0F_)uHDSoY0$_>e-rn*(4<0EHHxl#4sKr?Js8-%JCHseURg$}Z(K73!kW;0u1 zvlvOvUm0LK{Bnj}ZI_(tny$D96x~yAF>EAj&ImCgu<`D<~>PWSiv$!Iw?DB(dW-Iz=M75=0TAdKETXe+yftMbG1k z!h3&HL;N0+Yw$|2BkLu06-D~e$ZgiqI7#>3zXtA*(9*TL$K+JXxE1=PSV4;ly1c>- zOKKAKP`Jqr(8)NwF156Xehu@!UMDK=sWzbx<2nDpVJetaqQGnS`UCQPl~{kL)b;j2 zp;D6or9t8&Z`-knpsOrd2gxvEQbLvJ!s)a~Kqhmh5(7qd{|9vNP1{a?kGxM~#O%Gr zkic(Ln&SagleQ^f*p(E$uM&&!8iVmK!jv3Uw&xZCbLW@%*(mu$qo3{S*dpfElr+ox)L6|f|b0!~|v=v3m ziyT%*i%IC`XxCn_LjeHcUj0U)t=X`n>H(PCmb~}vnw8vUN-chAP$#iF>NW-mutRAd zq}pYrzJ#g%N-y@p6_|oNZmsUM;#lZ2f%9uS7hQsLV8M zD6AQD6$+P7!%Le0sJFCzDOqM%ic|{%-Md)8FuAs7UWanU#bc{{qgj?pASP8v%XMRV z7Y5NVex)HZEm>|zn>4XMKzCxEHl)<|-&-XPJzMhmyJLmtRTh#|z=QtSGokYljy;0X zwDKFKTNzJf=vga!4_v+!4EiHhVcaOmSWgXmmR$-?SzI*x!U)@!cUp3-Tu45DOfTE! zx4*tG9!?B zIM)7fF9l?7DDK~B$eNLeHd^S0EH6NLkW~*N&b66@ z3RfBTFE2P-P1BeJMKGE&*jZcY+57}cZ4p3SiGBuLRMy+JGTj0z7iXbBx)+$uf3j03 zN@{oOVBIZ^i?!~#JgZ;*H4sN2!m0Z!b`|y!kdu)wemIhOZpcQ8OV(}EfEeCazfaYw zWNdTw`l74&%`eM4SdZY163=+i9*bqYGlZyo-ngM71D?jpUG2#946N%(mFRC*InOzrj6Pu3dAvR%w*qMJ zh{i{X-lz2w{4LkG2SC-Waxlqg92yKW->vQU)z-2Xrv7-9X%I1Iw@$}ye|eO1wRkXy zJEqE%b`BT$kaX`dUpa7|E|N4~YGBWzF;D#6X0Vv=$xyRQF8A>MK;rQP>DXwJQ%F!E zD2WV*zFsH&3;JtR8hze|Q|ghW)a_rC`$Ib55mwb8X#X_weu12_fq@Deh@I2m*=TYb zW@r%Bkp%`5-s7&~m5HXiiHMAN5DtWXcciHzJQiqhnsjk9Pthw*tYcB=iLb z#!T21R!Y{mI9Pg~mEJ6Fcr;OATo^9mkn%v=&LPy>;_^nQ+NF*l*p5n|T;FNvw4XD? zvJfTI;Qhge!Zty#L|vlCOv8_S*lBon&>Z$(%>@Ut%2 z-mpko86262P)8?9C#?*F+8ntC_pg$$xj=J{@A!O+-QGkH(CB2o5_2EXHSH>FIqM)0 zn%rt*maMR03|Q40pyaS>tu~GXIpP0 zJ5MA)GU-HLWafA@ zaI6AhA~O9IyMw5%uIqk*k8fKY19ll1H|r}boIXpwkx{+H7k#&X4oG?o2_H?^Uk_1GNq1P8X+;d5zQdT3ipi^gOq-l zHoZuvS3`e(fVtL|>&i=stZH^UvfKvZ7~V}(m|}6NE_#QSi45$aU?7@wv%vSWi5_-7 z`$fS6l$B6K80|v6GvloiG+SH}4od&}IHFs@>sY-dGIOwDX3ZH9(29yg z0v(%rC-6FpE2!Nmf0@%?H|0(p9S>pDFFBM+GA{>6>5R?jL@tmer`fA zK`EFj2;L7UX8blCC*T%_uU!TX`o$&>WZN}(k=sWFjSCFq)XKQ>ZrjcMCjF(9k!j8@ zEFfEUGC^}8VOP<;JHn@h1C3F2?*xFNDBI`E6vCfQ&bY7oq7V>K-I0C7l&;!6|vI`tsfxLg_5CR*l6KHngjbfeu74 z&#~3H@Qv9cE5nUTVzsET(i8UltL!C52MR7bC=o0kEDCZ_^8i} zlf?xE62d)ZJ-BHg*+T3j^q1@n*7pMt7yJ2R_Bt{hmP_zJIH~(r^C^vVQ$5?UWWV-9 z$Je~CD@nj;a!R0reijm)$AHa7R(mfq)u4Fh%qN!ALG(H(+b2P$Y_Lr1_UAi6jS_Yh zh$Oz~4qsEYe?5FQ;6>5U#HZx!C2@QLrW|UzN{KT7k67Ou>VN@%VqdkGhm*)xZ&fG)p#=A^M-sMTURTF9yBRPqgN z7HF7Y=Q(;dppBTru31=B#asb)KGgXcevSG+ou|q+vv|aD=|;xu zzgdmfQ)Jtj^x=yzKTCKw>6dd{@v=}+Yr*JF8k$K<^_CH+P`b9R42?sECjDK(u{To5^heL+!| zUU*17vNf#XJSh~VPDR>wrNO%yZTu{6ERXuPz4Hib9%oSjRQ=W=%=Mq(1XU>S;$oHd zvIIEvBKac%GIli+j2Y_Sz9VWg1Dt}Wb)QE z=u1NNt!%s0YGHhWa834nIDIT1BS2-gq@3_h%)NE3FZz|2S;+ud)q+T##hz<4MMW7n z^HP3(6dDKedahyIc=So{1_FJ)zO*OINmvZgwEAUwRL-8+{-x8X|Lp5w4(1@C)4L zhV3se+9X9eyJwK(e~7qw>k*t&kGlm3IgLw*m003Uo)%_**&t(+!=c|8Vg6|mMlaz} z8`EV09rhow?A8d;Ct^Y*ccj5qS3MzE-C@gt4n|Yb1L`Z`mHdt~CTP(k z7Ngw^=znGEn(YdJatbMsxWlftdbQJGc~xUV7G?%MgGmC;cn$zA0RZW|He@yXx_WM( z!J?X>DcEEhALDMnT6H%EB2{wV@e}3UL~I6t7o;_Xuz_qlmJ4tY1VNl{E;1xc* zdIVPVOZd*;eD|x*q9`&Y2(?<%5etoq>xnsg2=HLqe$*dKU~Dq?wAJ8|DgVLu_{IY1 z>b07L0@x4o7y^1Z=gL+`u0Q*G{q@k&ig+Iv$rh%DrzM>!3YW^jO_kR;)1RuUx6|AI zu~U{KhizTpz*paalzc+$Nn@oxJ=c)&P|<3>Q#8054hee%UOrupz!PR!|dtETfgCqn> z8ztPdClO}NN8{6zDls)58xX&mqR6q?R730uHsfXR!9VJ&v6Nnr+a}1(!aW+$1*cZ! z>w8OWbSWmdV`V)EXmA2{t_{9uA2;WJ_DAYU-|&ljIL7Rw)9rwyVJS2n&;W$yK7jk1 zeE|MLct_qCafH{j1f9WM{%T=WVh56U9v2V*ftUd@%YFGeLVt4%y+0z@P%d>Oa1d0R zLynzm7+@6=y4-clXNO+y>)?hPa9K=Lk7Uh1ViA`r7m z!aFLzd&lIX-kHqG;YQ!Kew85(c3dU0N%q1%B9PJwz$nK36VC?bx4(iZ_(>Fo$abIL zlI9w^v3G;|_``{_L?!FIly~hElbZt5fL%R~wD7u9E__I|6KN0V3niJYIqW4!yL~wqlh4|L=x$LEMLlk!f>exwN}v9{^83 zL{2ePfj|K-65iFh-=rqnAvY3U!DcM&oXQx!E>4du+-U*>W{WwMisw$h%Yx-Izu@{HHxJ^u{~@Lb-v97{$FyW!Bc&u{H4 zckWoO#o#1R^KK@Ps|fRmK8~d8JE`1`BEQ|<#QFU&Z1CS^nS+j{Rg0nwdh$N)Ip?!o z0Q`SC;{%PUb4@*h6wf>Ue&sVAxDZLpHwtQbpp0w2y)fHeygdvMa+2Z%fHFku?@`XA z1}+;MQa#>@tM#2?>o`P^)Bumh+&o*X)8{QOKeAXFdq){@F Date: Tue, 31 Mar 2020 18:00:24 -0700 Subject: [PATCH 13/19] Updated readme --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 37d803bf76..2265a4a5bd 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,12 @@ python, making it perfect for modern Machine Learning and Data processing pipeli * Array Tasks (map some function over a large dataset, controlled execution of 1000's of containers) * Dynamic Workflow creation and execution - with runtime type safety * Container side plugins with first class support in python + - Maintain an inventory of tasks and workflows + - Record history of all executions and executions (as long as they follow convention) are completely repeatable - Multi Cloud support (AWS, GCP and others) + - Extensible core + - Modularized + - Automated notifications to Slack, Email, Pagerduty - Deep observability - Multi K8s cluster support - Comes with many system supported out of the box on K8s like Spark etc. @@ -51,6 +56,11 @@ python, making it perfect for modern Machine Learning and Data processing pipeli - Python CLI - Written in Golang and optimized for performance +## Coming Soon + - Single Task Execution support + - Reactive pipelines + - More integrations + # Current Usage - Lyft Rideshare - Lyft L5 autonomous From b7a21b9c708f79b575177d12bd61bc6daf9d57b2 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 18:10:42 -0700 Subject: [PATCH 14/19] updated Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2265a4a5bd..3665899378 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![alt text](https://github.com/lyft/flyte/raw/master/images/flyte_lockup_gradient_on_light.png "Flyte Logo") +![Flyte Logo](images/flyte_lockup_gradient_on_light.png "Flyte Logo") [![Current Release](https://img.shields.io/github/release/lyft/flyte.svg)](https://github.com/lyft/flyte/releases/latest) [![Build Status](https://travis-ci.org/lyft/flyte.svg?branch=master)](https://travis-ci.org/lyft/flyte) From 27570722bca1c6c9dfbf921913072965b4628199 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 18:14:36 -0700 Subject: [PATCH 15/19] added available plugins --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 3665899378..516f2208d0 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,16 @@ python, making it perfect for modern Machine Learning and Data processing pipeli - Reactive pipelines - More integrations + +# Available Plugins + - Containers + - K8s Pods + - AWS Batch Arrays + - K8s Pod arrays + - K8s Spark (native pyspark and java/scala) + - Qubole Hive + - Presto Queries + # Current Usage - Lyft Rideshare - Lyft L5 autonomous From dd6021691abc5fbdc891e5e7c679dd35006c10c1 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 19:18:43 -0700 Subject: [PATCH 16/19] Added changelog --- CHANGELOG/CHANGELOG-v0.2.0.md | 32 ++++++++++++++++++++++++++++++++ CHANGELOG/README.md | 3 +++ README.md | 2 +- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG/CHANGELOG-v0.2.0.md create mode 100644 CHANGELOG/README.md diff --git a/CHANGELOG/CHANGELOG-v0.2.0.md b/CHANGELOG/CHANGELOG-v0.2.0.md new file mode 100644 index 0000000000..636c6d65b7 --- /dev/null +++ b/CHANGELOG/CHANGELOG-v0.2.0.md @@ -0,0 +1,32 @@ +#Flyte V0.2.0 + +Changes since v0.1.0 + +- Go Mod support +- Go 1.13 upgrade +- OAuth Support (CLI, UI, Admin) +- Standardized pull request templates & OSS process standardization +- PluginMachinery and flexible statemachine for Arbitrary plugins +- Improved Engine performance, reliability and write requirements to etcD (scalability) +- Backoff support +- Timeouts +- Dynamic workflow support +- Resource manager per project/domain +- RawOutputDirectories created in FlytePropeller +- Improve visibility and observability +- User/System error differentiation +- Optional interruptible tasks (lets use spot instances, reduce cost) +- Caps on queue time for workflows +- Multi cluster improvements +- Visibility into execution cluster for the execution +- Add descriptions to Identifiers (Projects, Workflows etc) +- Configuration improvements throughout - You can configure Flyte for your environment +- Multi Cluster placement manager + +Flytekit (SDK improvements) +- Subworkflow support +- Collaboration of Workflows support (Workflow fetch) +- Three-legged OAuth and basic authentication support +- Fetch the latest version of a task instead of having to specify a version + +And ofcourse Bug fixes throughout! diff --git a/CHANGELOG/README.md b/CHANGELOG/README.md new file mode 100644 index 0000000000..9c46c5bfb7 --- /dev/null +++ b/CHANGELOG/README.md @@ -0,0 +1,3 @@ +Change Logs + +[Changes between v0.1.0 -> v0.2.0](CHANGELOG-v0.2.0.md) diff --git a/README.md b/README.md index 516f2208d0..dc5e3b9239 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ![Commit activity](https://img.shields.io/github/commit-activity/w/lyft/flyte.svg?style=plastic) ![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyte/latest.svg?style=plastic) ![GitHub milestones Completed](https://img.shields.io/github/milestones/closed/lyft/flyte?style=plastic) -![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/2?style=plastic) +![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/3?style=plastic) ![Twitter Follow](https://img.shields.io/twitter/follow/flyteorg?label=Follow&style=social) [![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://docs.google.com/forms/d/e/1FAIpQLSf8bNuyhy7rkm77cOXPHIzCm3ApfL7Tdo7NUs6Ej2NOGQ1PYw/viewform?pli=1) From 11bbe25b99010cbc64f6ffec905dd3173c6209fc Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 19:19:44 -0700 Subject: [PATCH 17/19] Changelog link --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index dc5e3b9239..84f9d31dd6 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,9 @@ python, making it perfect for modern Machine Learning and Data processing pipeli - Lyft L5 autonomous - Juno +# Changelogs +[Changelogs](CHANGELOG/README.md) + # Component Repos Repo | Language | Purpose --- | --- | --- From 271a81413ddec438441ec2de1de265d7f7bb0be3 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Tue, 31 Mar 2020 19:20:10 -0700 Subject: [PATCH 18/19] Update CHANGELOG/CHANGELOG-v0.2.0.md --- CHANGELOG/CHANGELOG-v0.2.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG/CHANGELOG-v0.2.0.md b/CHANGELOG/CHANGELOG-v0.2.0.md index 636c6d65b7..cb6de99876 100644 --- a/CHANGELOG/CHANGELOG-v0.2.0.md +++ b/CHANGELOG/CHANGELOG-v0.2.0.md @@ -29,4 +29,4 @@ Flytekit (SDK improvements) - Three-legged OAuth and basic authentication support - Fetch the latest version of a task instead of having to specify a version -And ofcourse Bug fixes throughout! +And of course Bug fixes throughout! From 995d6d0a9d1a2f4a447e9ef8790666a976470d9d Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 31 Mar 2020 19:24:09 -0700 Subject: [PATCH 19/19] updated --- CHANGELOG/CHANGELOG-v0.2.0.md | 9 +++++---- CHANGELOG/README.md | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG/CHANGELOG-v0.2.0.md b/CHANGELOG/CHANGELOG-v0.2.0.md index cb6de99876..4c16bb0742 100644 --- a/CHANGELOG/CHANGELOG-v0.2.0.md +++ b/CHANGELOG/CHANGELOG-v0.2.0.md @@ -1,7 +1,8 @@ -#Flyte V0.2.0 +# Flyte V0.2.0 -Changes since v0.1.0 +## Changes since v0.1.0 +### Core Platform - Go Mod support - Go 1.13 upgrade - OAuth Support (CLI, UI, Admin) @@ -23,10 +24,10 @@ Changes since v0.1.0 - Configuration improvements throughout - You can configure Flyte for your environment - Multi Cluster placement manager -Flytekit (SDK improvements) +### Flytekit (SDK improvements) - Subworkflow support - Collaboration of Workflows support (Workflow fetch) - Three-legged OAuth and basic authentication support - Fetch the latest version of a task instead of having to specify a version -And of course Bug fixes throughout! +### And of course Bug fixes throughout! diff --git a/CHANGELOG/README.md b/CHANGELOG/README.md index 9c46c5bfb7..1a4d040710 100644 --- a/CHANGELOG/README.md +++ b/CHANGELOG/README.md @@ -1,3 +1,3 @@ -Change Logs +# ChangeLogs -[Changes between v0.1.0 -> v0.2.0](CHANGELOG-v0.2.0.md) +- [Changes between v0.1.0 -> v0.2.0](CHANGELOG-v0.2.0.md)