diff --git a/dev-requirements.txt b/dev-requirements.txt index 2409f24d1a..4af063b029 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -235,7 +235,7 @@ py==1.11.0 # -c requirements.txt # pytest # retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via # -c requirements.txt # flytekit diff --git a/doc-requirements.txt b/doc-requirements.txt index 0e94b20be9..cded630951 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -250,7 +250,7 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit pycparser==2.21 # via cffi diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt index c00d38146f..6f88de4253 100644 --- a/plugins/flytekit-aws-athena/requirements.txt +++ b/plugins/flytekit-aws-athena/requirements.txt @@ -69,7 +69,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit python-dateutil==2.8.1 # via diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt index 523164a193..6e8050eac6 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.txt +++ b/plugins/flytekit-aws-sagemaker/requirements.txt @@ -101,7 +101,7 @@ psutil==5.8.0 # via sagemaker-training py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit pycparser==2.20 # via cffi diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt index bd0ebdf4ad..5187609526 100644 --- a/plugins/flytekit-data-fsspec/requirements.txt +++ b/plugins/flytekit-data-fsspec/requirements.txt @@ -71,7 +71,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit python-dateutil==2.8.1 # via diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt index 045e6f90b2..589a6e9e10 100644 --- a/plugins/flytekit-dolt/requirements.txt +++ b/plugins/flytekit-dolt/requirements.txt @@ -77,7 +77,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit python-dateutil==2.8.1 # via diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index 6c639b47a3..103b5ea465 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -204,7 +204,7 @@ ptyprocess==0.7.0 # terminado py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit pycparser==2.20 # via cffi diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt index 3d0828e637..7daca31692 100644 --- a/plugins/flytekit-hive/requirements.txt +++ b/plugins/flytekit-hive/requirements.txt @@ -69,7 +69,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit python-dateutil==2.8.1 # via diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt index d2a8907347..0489ee45db 100644 --- a/plugins/flytekit-k8s-pod/requirements.txt +++ b/plugins/flytekit-k8s-pod/requirements.txt @@ -79,7 +79,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit pyasn1==0.4.8 # via diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt index eae5b2b7a1..62533710fa 100644 --- a/plugins/flytekit-kf-pytorch/requirements.txt +++ b/plugins/flytekit-kf-pytorch/requirements.txt @@ -69,7 +69,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit python-dateutil==2.8.1 # via diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt index 30733deb05..d8f6760e3d 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.txt +++ b/plugins/flytekit-kf-tensorflow/requirements.txt @@ -69,7 +69,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit python-dateutil==2.8.1 # via diff --git a/plugins/flytekit-modin/requirements.txt b/plugins/flytekit-modin/requirements.txt index a8eacdb29d..acdd7a26b1 100644 --- a/plugins/flytekit-modin/requirements.txt +++ b/plugins/flytekit-modin/requirements.txt @@ -98,7 +98,7 @@ protobuf==3.18.0 # ray py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit pycparser==2.20 # via cffi diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt index 36e3d67510..b31d2e2879 100644 --- a/plugins/flytekit-pandera/requirements.txt +++ b/plugins/flytekit-pandera/requirements.txt @@ -76,7 +76,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via # flytekit # pandera diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index 119628099c..900a633f76 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -173,7 +173,7 @@ py==1.10.0 # via retry py4j==0.10.9 # via pyspark -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit pygments==2.10.0 # via diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt index ac1305316a..a36ccc60f8 100644 --- a/plugins/flytekit-snowflake/requirements.txt +++ b/plugins/flytekit-snowflake/requirements.txt @@ -69,7 +69,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit python-dateutil==2.8.1 # via diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt index 699562c845..8bb92b3bb5 100644 --- a/plugins/flytekit-spark/requirements.txt +++ b/plugins/flytekit-spark/requirements.txt @@ -71,7 +71,7 @@ py==1.10.0 # via retry py4j==0.10.9 # via pyspark -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit pyspark==3.1.2 # via flytekitplugins-spark diff --git a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt index e1e06c4a88..10e49666a6 100644 --- a/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/flytekitplugins/sqlalchemy/requirements.txt @@ -67,7 +67,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit python-dateutil==2.8.1 # via diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt index 515dd88e68..dde4317a78 100644 --- a/plugins/flytekit-sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/requirements.txt @@ -71,7 +71,7 @@ protobuf==3.17.3 # flytekit py==1.10.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit python-dateutil==2.8.1 # via diff --git a/requirements-spark2.txt b/requirements-spark2.txt index da84f0b8c7..3811a19548 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -224,7 +224,7 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit pycparser==2.21 # via cffi diff --git a/requirements.txt b/requirements.txt index 1f1f07d4f5..4d4a893bb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -222,7 +222,7 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit pycparser==2.21 # via cffi diff --git a/setup.py b/setup.py index a31089c48d..305e3d3700 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ spark = ["pyspark>=2.4.0,<3.0.0"] spark3 = ["pyspark>=3.0.0"] sidecar = ["k8s-proto>=0.0.3,<1.0.0"] -schema = ["numpy>=1.14.0,<2.0.0", "pandas>=0.22.0,<2.0.0", "pyarrow>2.0.0"] +schema = ["numpy>=1.14.0,<2.0.0", "pandas>=0.22.0,<2.0.0", "pyarrow>=6.0.0"] hive_sensor = ["hmsclient>=0.0.1,<1.0.0"] notebook = ["papermill>=1.2.0", "nbconvert>=6.0.7", "ipykernel>=5.0.0,<6.0.0"] sagemaker = ["sagemaker-training>=3.6.2,<4.0.0"] @@ -67,7 +67,7 @@ "flyteidl>=0.21.4", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", - "pyarrow>=2.0.0,<4.0.0", + "pyarrow>=6.0.0,<7.0.0", "click>=6.6,<8.0", "croniter>=0.3.20,<4.0.0", "deprecated>=1.0,<2.0", diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index e7ce990324..21be38841a 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -89,7 +89,7 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==3.0.0 +pyarrow==6.0.0 # via flytekit pycparser==2.21 # via cffi