From 9de00b49d61770eeb1866de59b89ffd2eb99fd61 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Fri, 13 May 2022 12:52:34 -0700 Subject: [PATCH] Bump plugin pins to 1.1-> 1.2 (#1001) Signed-off-by: Yee Hing Tong --- plugins/README.md | 10 ++++++++++ plugins/flytekit-aws-athena/setup.py | 2 +- plugins/flytekit-aws-batch/setup.py | 2 +- plugins/flytekit-aws-sagemaker/setup.py | 2 +- plugins/flytekit-bigquery/setup.py | 2 +- plugins/flytekit-data-fsspec/setup.py | 2 +- plugins/flytekit-deck-standard/setup.py | 2 +- plugins/flytekit-dolt/setup.py | 2 +- plugins/flytekit-greatexpectations/setup.py | 2 +- plugins/flytekit-hive/setup.py | 2 +- plugins/flytekit-k8s-pod/setup.py | 2 +- plugins/flytekit-kf-mpi/setup.py | 2 +- plugins/flytekit-kf-pytorch/setup.py | 2 +- plugins/flytekit-kf-tensorflow/setup.py | 2 +- plugins/flytekit-modin/setup.py | 2 +- plugins/flytekit-pandera/setup.py | 2 +- plugins/flytekit-papermill/setup.py | 2 +- plugins/flytekit-snowflake/setup.py | 2 +- plugins/flytekit-spark/setup.py | 2 +- plugins/flytekit-sqlalchemy/setup.py | 2 +- 20 files changed, 29 insertions(+), 19 deletions(-) diff --git a/plugins/README.md b/plugins/README.md index f2bcb73068..4986e3445f 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -125,6 +125,16 @@ In such a case, you can add a special directive in the `setup.py` which will ins ``` +### Flytekit Version Pinning +Currently we advocate pinning to minor releases of flytekit. To bump the pins across the board, `cd plugins/` and then +update the command below with the appropriate range and run + +```bash +for f in $(ls **/setup.py); do sed -i "s/flytekit>.*,<1.1/flytekit>=1.1.0b0,<1.2/" $f; done +``` + +Try using `gsed` instead of `sed` if you are on a Mac. Also this only works of course for setup files that start with the version in your sed command. There may be plugins that have different pins to start out with. + ## References 📚 - Example of a simple Python task that allows adding only Python side functionality: [flytekit-greatexpectations](./flytekit-greatexpectations/) - Example of a TypeTransformer or a Type Plugin: [flytekit-pandera](./flytekit-pandera/). These plugins add new types to Flyte and tell Flyte how to transform them and add additional features through types. Flyte is a multi-lang system, and type transformers allow marshaling between Flytekit and backend and other languages. diff --git a/plugins/flytekit-aws-athena/setup.py b/plugins/flytekit-aws-athena/setup.py index 402b05741f..1164b99d00 100644 --- a/plugins/flytekit-aws-athena/setup.py +++ b/plugins/flytekit-aws-athena/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-aws-batch/setup.py b/plugins/flytekit-aws-batch/setup.py index 5086f7defe..e176e35aae 100644 --- a/plugins/flytekit-aws-batch/setup.py +++ b/plugins/flytekit-aws-batch/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-aws-sagemaker/setup.py b/plugins/flytekit-aws-sagemaker/setup.py index 5ba7ff50f1..2781192f41 100644 --- a/plugins/flytekit-aws-sagemaker/setup.py +++ b/plugins/flytekit-aws-sagemaker/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0", "sagemaker-training>=3.6.2,<4.0.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0", "sagemaker-training>=3.6.2,<4.0.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-bigquery/setup.py b/plugins/flytekit-bigquery/setup.py index 073f1ef261..e84cd6ce2b 100644 --- a/plugins/flytekit-bigquery/setup.py +++ b/plugins/flytekit-bigquery/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0", "google-cloud-bigquery"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0", "google-cloud-bigquery"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-data-fsspec/setup.py b/plugins/flytekit-data-fsspec/setup.py index 1db30f9423..c43d7c1481 100644 --- a/plugins/flytekit-data-fsspec/setup.py +++ b/plugins/flytekit-data-fsspec/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-data-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0", "fsspec>=2021.7.0", "botocore>=1.7.48"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0", "fsspec>=2021.7.0", "botocore>=1.7.48"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-deck-standard/setup.py b/plugins/flytekit-deck-standard/setup.py index 0bc220e556..fe04ab5434 100644 --- a/plugins/flytekit-deck-standard/setup.py +++ b/plugins/flytekit-deck-standard/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}-standard" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0", "markdown", "plotly", "pandas_profiling"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0", "markdown", "plotly", "pandas_profiling"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-dolt/setup.py b/plugins/flytekit-dolt/setup.py index 5b1ba7824b..bb8b572bc7 100644 --- a/plugins/flytekit-dolt/setup.py +++ b/plugins/flytekit-dolt/setup.py @@ -6,7 +6,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0", "dolt_integrations>=0.1.5"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0", "dolt_integrations>=0.1.5"] dev_requires = ["pytest-mock>=3.6.1"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-greatexpectations/setup.py b/plugins/flytekit-greatexpectations/setup.py index fdde19ba47..e50f707624 100644 --- a/plugins/flytekit-greatexpectations/setup.py +++ b/plugins/flytekit-greatexpectations/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0", "great-expectations>=0.13.30", "sqlalchemy>=1.4.23"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0", "great-expectations>=0.13.30", "sqlalchemy>=1.4.23"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-hive/setup.py b/plugins/flytekit-hive/setup.py index 13c3bdf67b..f9602500f8 100644 --- a/plugins/flytekit-hive/setup.py +++ b/plugins/flytekit-hive/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-k8s-pod/setup.py b/plugins/flytekit-k8s-pod/setup.py index 2b7312d86f..01704e1a6a 100644 --- a/plugins/flytekit-k8s-pod/setup.py +++ b/plugins/flytekit-k8s-pod/setup.py @@ -5,7 +5,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" plugin_requires = [ - "flytekit>=1.0.0b0,<1.1.0", + "flytekit>=1.1.0b0,<1.2.0", "kubernetes>=12.0.1", ] diff --git a/plugins/flytekit-kf-mpi/setup.py b/plugins/flytekit-kf-mpi/setup.py index de794808f8..18f168af18 100644 --- a/plugins/flytekit-kf-mpi/setup.py +++ b/plugins/flytekit-kf-mpi/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0", "flyteidl>=0.21.4"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0", "flyteidl>=0.21.4"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-kf-pytorch/setup.py b/plugins/flytekit-kf-pytorch/setup.py index b3fb831586..2e0b57a7f8 100644 --- a/plugins/flytekit-kf-pytorch/setup.py +++ b/plugins/flytekit-kf-pytorch/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-kf-tensorflow/setup.py b/plugins/flytekit-kf-tensorflow/setup.py index 06df557bcd..45d8fe6b2e 100644 --- a/plugins/flytekit-kf-tensorflow/setup.py +++ b/plugins/flytekit-kf-tensorflow/setup.py @@ -5,7 +5,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" # TODO: Requirements are missing, add them back in later. -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-modin/setup.py b/plugins/flytekit-modin/setup.py index 0f08010f00..777a19db47 100644 --- a/plugins/flytekit-modin/setup.py +++ b/plugins/flytekit-modin/setup.py @@ -5,7 +5,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" plugin_requires = [ - "flytekit>=1.0.0b3,<1.1.0", + "flytekit>=1.1.0b0,<1.2.0", "modin>=0.13.0", "fsspec", "ray", diff --git a/plugins/flytekit-pandera/setup.py b/plugins/flytekit-pandera/setup.py index 994e5a90c3..cbe9bf4061 100644 --- a/plugins/flytekit-pandera/setup.py +++ b/plugins/flytekit-pandera/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0", "pandera>=0.7.1"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0", "pandera>=0.7.1"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-papermill/setup.py b/plugins/flytekit-papermill/setup.py index 3fdbac5b4c..26a3f1b705 100644 --- a/plugins/flytekit-papermill/setup.py +++ b/plugins/flytekit-papermill/setup.py @@ -5,7 +5,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" plugin_requires = [ - "flytekit>=1.0.0b3,<1.1.0", + "flytekit>=1.1.0b0,<1.2.0", "papermill>=1.2.0", "nbconvert>=6.0.7", "ipykernel>=5.0.0", diff --git a/plugins/flytekit-snowflake/setup.py b/plugins/flytekit-snowflake/setup.py index d1ce4b195d..ebea48f304 100644 --- a/plugins/flytekit-snowflake/setup.py +++ b/plugins/flytekit-snowflake/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-spark/setup.py b/plugins/flytekit-spark/setup.py index 5a606f10a6..9fdffe6c22 100644 --- a/plugins/flytekit-spark/setup.py +++ b/plugins/flytekit-spark/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0", "pyspark>=3.0.0"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0", "pyspark>=3.0.0"] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-sqlalchemy/setup.py b/plugins/flytekit-sqlalchemy/setup.py index 34fbe76318..6bf0a8e1ab 100644 --- a/plugins/flytekit-sqlalchemy/setup.py +++ b/plugins/flytekit-sqlalchemy/setup.py @@ -4,7 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.0.0b3,<1.1.0", "sqlalchemy>=1.4.7"] +plugin_requires = ["flytekit>=1.1.0b0,<1.2.0", "sqlalchemy>=1.4.7"] __version__ = "0.0.0+develop"