From e55990e4dc487610e0ef76f7b2a54d5e94384d80 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Thu, 4 Aug 2022 17:20:14 -0700 Subject: [PATCH 1/5] cherry pick 97b454b1 Signed-off-by: Yee Hing Tong --- flytekit/core/local_cache.py | 21 +++++++++--- setup.py | 1 + tests/flytekit/unit/core/test_local_cache.py | 34 ++++++++++++++++++-- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/flytekit/core/local_cache.py b/flytekit/core/local_cache.py index 4afab73955..c1f42a36a4 100644 --- a/flytekit/core/local_cache.py +++ b/flytekit/core/local_cache.py @@ -1,8 +1,8 @@ -import base64 from typing import Optional -import cloudpickle +import joblib from diskcache import Cache +from google.protobuf.struct_pb2 import Struct from flytekit.models.literals import Literal, LiteralCollection, LiteralMap @@ -28,15 +28,26 @@ def _recursive_hash_placement(literal: Literal) -> Literal: return literal +class ProtoJoblibHasher(joblib.hashing.NumpyHasher): + def save(self, obj): + if isinstance(obj, Struct): + obj = dict( + rewrite_rule="google.protobuf.struct_pb2.Struct", + cls=obj.__class__, + obj=dict(sorted(obj.fields.items())), + ) + return obj + + def _calculate_cache_key(task_name: str, cache_version: str, input_literal_map: LiteralMap) -> str: # Traverse the literals and replace the literal with a new literal that only contains the hash literal_map_overridden = {} for key, literal in input_literal_map.literals.items(): literal_map_overridden[key] = _recursive_hash_placement(literal) - # Pickle the literal map and use base64 encoding to generate a representation of it - b64_encoded = base64.b64encode(cloudpickle.dumps(LiteralMap(literal_map_overridden))) - return f"{task_name}-{cache_version}-{b64_encoded}" + # Generate a hash key of inputs with joblib + hashed_inputs = ProtoJoblibHasher().hash(literal_map_overridden) + return f"{task_name}-{cache_version}-{hashed_inputs}" class LocalTaskCache(object): diff --git a/setup.py b/setup.py index 85af8691b6..56e2dd5624 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ "grpcio-status>=1.43,!=1.45.0", "importlib-metadata", "pyopenssl", + "joblib", "protobuf>=3.6.1,<4", "python-json-logger>=2.0.0", "pytimeparse>=1.1.8,<2.0.0", diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py index 3f3e56de88..91c29ec2d0 100644 --- a/tests/flytekit/unit/core/test_local_cache.py +++ b/tests/flytekit/unit/core/test_local_cache.py @@ -1,19 +1,24 @@ import datetime import typing from dataclasses import dataclass -from typing import List +from typing import Dict, List import pandas from dataclasses_json import dataclass_json from pytest import fixture from typing_extensions import Annotated -from flytekit import SQLTask, dynamic, kwtypes +from flytekit.core.base_sql_task import SQLTask +from flytekit.core.base_task import kwtypes +from flytekit.core.context_manager import FlyteContextManager +from flytekit.core.dynamic_workflow_task import dynamic from flytekit.core.hash import HashMethod -from flytekit.core.local_cache import LocalTaskCache +from flytekit.core.local_cache import LocalTaskCache, _calculate_cache_key from flytekit.core.task import TaskMetadata, task from flytekit.core.testing import task_mock +from flytekit.core.type_engine import TypeEngine from flytekit.core.workflow import workflow +from flytekit.models.literals import LiteralMap from flytekit.types.schema import FlyteSchema # Global counter used to validate number of calls to cache @@ -383,3 +388,26 @@ def my_workflow(): # Confirm that we see a cache hit in the case of annotated dataframes. my_workflow() assert n_cached_task_calls == 1 + + +def test_cache_key_repetition(): + pt = Dict + lt = TypeEngine.to_literal_type(pt) + ctx = FlyteContextManager.current_context() + kwargs = { + "a": 0.41083513079747874, + "b": 0.7773927872515183, + "c": 17, + } + keys = set() + for i in range(0, 100): + lit = TypeEngine.to_literal(ctx, kwargs, Dict, lt) + lm = LiteralMap( + literals={ + "d": lit, + } + ) + key = _calculate_cache_key("t1", "007", lm) + keys.add(key) + + assert len(keys) == 1 From fbdc50a73c5e5f26c6306cfca3819d492ee5e67a Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Thu, 4 Aug 2022 17:24:01 -0700 Subject: [PATCH 2/5] requirements Signed-off-by: Yee Hing Tong --- dev-requirements.txt | 87 +++++-------- doc-requirements.txt | 122 +++++++++--------- requirements-spark2.txt | 34 ++--- requirements.txt | 34 ++--- .../workflows/requirements.txt | 45 +++---- 5 files changed, 130 insertions(+), 192 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 645ffbeb18..dad926c664 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make dev-requirements.txt @@ -8,10 +8,14 @@ # via # -c requirements.txt # pytest-flyte +appnope==0.1.3 + # via ipython arrow==1.2.2 # via # -c requirements.txt # jinja2-time +asttokens==2.0.5 + # via stack-data attrs==20.3.0 # via # -c requirements.txt @@ -26,8 +30,6 @@ binaryornot==0.4.4 # via # -c requirements.txt # cookiecutter -cached-property==1.5.2 - # via docker-compose cachetools==5.2.0 # via google-auth certifi==2022.6.15 @@ -65,7 +67,7 @@ cookiecutter==2.1.1 # via # -c requirements.txt # flytekit -coverage[toml]==6.4.1 +coverage[toml]==6.4.2 # via # -r dev-requirements.in # pytest-cov @@ -78,7 +80,6 @@ cryptography==37.0.4 # -c requirements.txt # paramiko # pyopenssl - # secretstorage dataclasses-json==0.5.7 # via # -c requirements.txt @@ -96,7 +97,7 @@ diskcache==5.4.0 # via # -c requirements.txt # flytekit -distlib==0.3.4 +distlib==0.3.5 # via virtualenv distro==1.7.0 # via docker-compose @@ -106,9 +107,7 @@ docker[ssh]==5.0.3 # docker-compose # flytekit docker-compose==1.29.2 - # via - # pytest-docker - # pytest-flyte + # via pytest-flyte docker-image-py==0.1.12 # via # -c requirements.txt @@ -121,9 +120,11 @@ docstring-parser==0.14.1 # via # -c requirements.txt # flytekit +executing==0.9.1 + # via stack-data filelock==3.7.1 # via virtualenv -flyteidl==1.1.8 +flyteidl==1.1.9 # via # -c requirements.txt # flytekit @@ -132,23 +133,23 @@ google-api-core[grpc]==2.8.2 # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-core -google-auth==2.9.0 +google-auth==2.9.1 # via # google-api-core # google-cloud-core -google-cloud-bigquery==3.2.0 +google-cloud-bigquery==3.3.0 # via -r dev-requirements.in -google-cloud-bigquery-storage==2.14.0 +google-cloud-bigquery-storage==2.14.1 # via # -r dev-requirements.in # google-cloud-bigquery -google-cloud-core==2.3.1 +google-cloud-core==2.3.2 # via google-cloud-bigquery google-crc32c==1.3.0 # via google-resumable-media google-resumable-media==2.3.3 # via google-cloud-bigquery -googleapis-common-protos==1.56.3 +googleapis-common-protos==1.56.4 # via # -c requirements.txt # flyteidl @@ -166,7 +167,7 @@ grpcio-status==1.47.0 # -c requirements.txt # flytekit # google-api-core -identify==2.5.1 +identify==2.5.3 # via pre-commit idna==3.3 # via @@ -175,25 +176,14 @@ idna==3.3 importlib-metadata==4.12.0 # via # -c requirements.txt - # click # flytekit - # jsonschema # keyring - # pluggy - # pre-commit - # pytest - # virtualenv iniconfig==1.1.1 # via pytest -ipython==7.34.0 +ipython==8.4.0 # via -r dev-requirements.in jedi==0.18.1 # via ipython -jeepney==0.8.0 - # via - # -c requirements.txt - # keyring - # secretstorage jinja2==3.1.2 # via # -c requirements.txt @@ -205,12 +195,15 @@ jinja2-time==0.2.0 # -c requirements.txt # cookiecutter joblib==1.1.0 - # via -r dev-requirements.in + # via + # -c requirements.txt + # -r dev-requirements.in + # flytekit jsonschema==3.2.0 # via # -c requirements.txt # docker-compose -keyring==23.6.0 +keyring==23.7.0 # via # -c requirements.txt # flytekit @@ -236,7 +229,7 @@ matplotlib-inline==0.1.3 # via ipython mock==4.0.3 # via -r dev-requirements.in -mypy==0.961 +mypy==0.971 # via -r dev-requirements.in mypy-extensions==0.4.3 # via @@ -252,7 +245,6 @@ nodeenv==1.7.0 numpy==1.21.6 # via # -c requirements.txt - # flytekit # pandas # pyarrow packaging==21.3 @@ -303,6 +295,8 @@ protoc-gen-swagger==0.1.0 # flyteidl ptyprocess==0.7.0 # via pexpect +pure-eval==0.2.2 + # via stack-data py==1.11.0 # via # -c requirements.txt @@ -347,7 +341,7 @@ pytest==7.1.2 # pytest-flyte pytest-cov==3.0.0 # via -r dev-requirements.in -pytest-docker==0.12.0 +pytest-docker==1.0.0 # via pytest-flyte pytest-flyte @ git+https://github.com/flyteorg/pytest-flyte@main # via -r dev-requirements.in @@ -385,7 +379,7 @@ pyyaml==5.4.1 # docker-compose # flytekit # pre-commit -regex==2022.7.9 +regex==2022.7.25 # via # -c requirements.txt # docker-image-py @@ -407,31 +401,25 @@ retry==0.9.2 # via # -c requirements.txt # flytekit -rsa==4.8 +rsa==4.9 # via google-auth -secretstorage==3.3.2 - # via - # -c requirements.txt - # keyring -singledispatchmethod==1.0 - # via - # -c requirements.txt - # flytekit six==1.16.0 # via # -c requirements.txt + # asttokens # dockerpty # google-auth # grpcio # jsonschema # paramiko # python-dateutil - # virtualenv # websocket-client sortedcontainers==2.4.0 # via # -c requirements.txt # flytekit +stack-data==0.3.0 + # via ipython statsd==3.3.0 # via # -c requirements.txt @@ -455,29 +443,24 @@ traitlets==5.3.0 # via # ipython # matplotlib-inline -typed-ast==1.5.4 - # via mypy typing-extensions==4.3.0 # via # -c requirements.txt - # arrow # flytekit - # importlib-metadata # mypy - # responses # torch # typing-inspect typing-inspect==0.7.1 # via # -c requirements.txt # dataclasses-json -urllib3==1.26.10 +urllib3==1.26.11 # via # -c requirements.txt # flytekit # requests # responses -virtualenv==20.15.1 +virtualenv==20.16.3 # via pre-commit wcwidth==0.2.5 # via prompt-toolkit @@ -495,7 +478,7 @@ wrapt==1.14.1 # -c requirements.txt # deprecated # flytekit -zipp==3.8.0 +zipp==3.8.1 # via # -c requirements.txt # importlib-metadata diff --git a/doc-requirements.txt b/doc-requirements.txt index e9f16c89a0..07c9f9e770 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make doc-requirements.txt @@ -12,15 +12,21 @@ altair==4.2.0 # via great-expectations ansiwrap==0.8.4 # via papermill +appnope==0.1.3 + # via + # ipykernel + # ipython argon2-cffi==21.3.0 # via notebook argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.2.2 # via jinja2-time -astroid==2.11.6 +astroid==2.12.2 # via sphinx-autoapi -attrs==21.4.0 +asttokens==2.0.5 + # via stack-data +attrs==22.1.0 # via # jsonschema # visions @@ -42,7 +48,7 @@ binaryornot==0.4.4 # via cookiecutter bleach==5.0.1 # via nbconvert -botocore==1.27.22 +botocore==1.27.46 # via -r doc-requirements.in cachetools==5.2.0 # via google-auth @@ -77,7 +83,6 @@ cryptography==37.0.4 # -r doc-requirements.in # great-expectations # pyopenssl - # secretstorage css-html-js-minify==2.5.5 # via sphinx-material cycler==0.11.0 @@ -86,7 +91,7 @@ dataclasses-json==0.5.7 # via # dolt-integrations # flytekit -debugpy==1.6.0 +debugpy==1.6.2 # via ipykernel decorator==5.1.1 # via @@ -118,13 +123,15 @@ entrypoints==0.4 # jupyter-client # nbconvert # papermill -fastjsonschema==2.15.3 +executing==0.9.1 + # via stack-data +fastjsonschema==2.16.1 # via nbformat -flyteidl==1.1.8 +flyteidl==1.1.9 # via flytekit -fonttools==4.33.3 +fonttools==4.34.4 # via matplotlib -fsspec==2022.5.0 +fsspec==2022.7.1 # via # -r doc-requirements.in # modin @@ -135,29 +142,29 @@ google-api-core[grpc]==2.8.2 # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-core -google-auth==2.9.0 +google-auth==2.9.1 # via # google-api-core # google-cloud-core # kubernetes google-cloud==0.34.0 # via -r doc-requirements.in -google-cloud-bigquery==3.2.0 +google-cloud-bigquery==3.3.0 # via -r doc-requirements.in -google-cloud-bigquery-storage==2.13.2 +google-cloud-bigquery-storage==2.14.1 # via google-cloud-bigquery -google-cloud-core==2.3.1 +google-cloud-core==2.3.2 # via google-cloud-bigquery google-crc32c==1.3.0 # via google-resumable-media google-resumable-media==2.3.3 # via google-cloud-bigquery -googleapis-common-protos==1.56.3 +googleapis-common-protos==1.56.4 # via # flyteidl # google-api-core # grpcio-status -great-expectations==0.15.12 +great-expectations==0.15.17 # via -r doc-requirements.in greenlet==1.1.2 # via sqlalchemy @@ -182,24 +189,21 @@ imagesize==1.4.1 # via sphinx importlib-metadata==4.12.0 # via - # click # flytekit # great-expectations - # jsonschema # keyring # markdown # sphinx - # sqlalchemy -importlib-resources==5.8.0 +importlib-resources==5.9.0 # via jsonschema -ipykernel==6.15.0 +ipykernel==6.15.1 # via # ipywidgets # jupyter # jupyter-console # notebook # qtconsole -ipython==7.34.0 +ipython==8.4.0 # via # great-expectations # ipykernel @@ -211,13 +215,11 @@ ipython-genutils==0.2.0 # notebook # qtconsole ipywidgets==7.7.1 - # via jupyter + # via + # great-expectations + # jupyter jedi==0.18.1 # via ipython -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.2 # via # altair @@ -235,13 +237,14 @@ jmespath==1.0.1 # via botocore joblib==1.1.0 # via + # flytekit # pandas-profiling # phik jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.6.1 +jsonschema==4.9.1 # via # altair # great-expectations @@ -257,7 +260,7 @@ jupyter-client==7.3.4 # qtconsole jupyter-console==6.4.4 # via jupyter -jupyter-core==4.10.0 +jupyter-core==4.11.1 # via # jupyter-client # nbconvert @@ -268,9 +271,9 @@ jupyterlab-pygments==0.2.2 # via nbconvert jupyterlab-widgets==1.1.1 # via ipywidgets -keyring==23.6.0 +keyring==23.7.0 # via flytekit -kiwisolver==1.4.3 +kiwisolver==1.4.4 # via matplotlib kubernetes==24.2.0 # via -r doc-requirements.in @@ -278,7 +281,7 @@ lazy-object-proxy==1.7.1 # via astroid lxml==4.9.1 # via sphinx-material -markdown==3.3.7 +markdown==3.4.1 # via -r doc-requirements.in markupsafe==2.1.1 # via @@ -310,7 +313,7 @@ mistune==0.8.4 # via # great-expectations # nbconvert -modin==0.12.1 +modin==0.15.2 # via -r doc-requirements.in multimethod==1.8 # via @@ -341,17 +344,16 @@ nest-asyncio==1.5.5 # jupyter-client # nbclient # notebook -networkx==2.6.3 +networkx==2.8.5 # via visions notebook==6.4.12 # via # great-expectations # jupyter # widgetsnbextension -numpy==1.21.6 +numpy==1.23.1 # via # altair - # flytekit # great-expectations # imagehash # matplotlib @@ -380,7 +382,7 @@ packaging==21.3 # pandera # qtpy # sphinx -pandas==1.3.5 +pandas==1.4.3 # via # altair # dolt-integrations @@ -394,7 +396,7 @@ pandas==1.3.5 # visions pandas-profiling==3.2.0 # via -r doc-requirements.in -pandera==0.9.0 +pandera==0.11.0 # via -r doc-requirements.in pandocfilters==1.5.0 # via nbconvert @@ -413,6 +415,8 @@ pillow==9.2.0 # imagehash # matplotlib # visions +pkgutil-resolve-name==1.3.10 + # via jsonschema plotly==5.9.0 # via -r doc-requirements.in prometheus-client==0.14.1 @@ -439,11 +443,15 @@ protobuf==3.20.1 protoc-gen-swagger==0.1.0 # via flyteidl psutil==5.9.1 - # via ipykernel + # via + # ipykernel + # modin ptyprocess==0.7.0 # via # pexpect # terminado +pure-eval==0.2.2 + # via stack-data py==1.11.0 # via retry py4j==0.10.9.5 @@ -497,7 +505,7 @@ python-dateutil==2.8.2 # kubernetes # matplotlib # pandas -python-json-logger==2.0.2 +python-json-logger==2.0.4 # via flytekit python-slugify[unidecode]==6.1.2 # via @@ -533,7 +541,7 @@ qtconsole==5.3.1 # via jupyter qtpy==2.1.0 # via qtconsole -regex==2022.6.2 +regex==2022.7.25 # via docker-image-py requests==2.28.1 # via @@ -555,13 +563,13 @@ responses==0.21.0 # via flytekit retry==0.9.2 # via flytekit -rsa==4.8 +rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.6 # via ruamel-yaml -scipy==1.7.3 +scipy==1.9.0 # via # great-expectations # imagehash @@ -573,14 +581,11 @@ seaborn==0.11.2 # via # missingno # pandas-profiling -secretstorage==3.3.2 - # via keyring send2trash==1.8.0 # via notebook -singledispatchmethod==1.0 - # via flytekit six==1.16.0 # via + # asttokens # bleach # google-auth # grpcio @@ -608,7 +613,7 @@ sphinx==4.5.0 # sphinx-panels # sphinx-prompt # sphinxcontrib-yt -sphinx-autoapi==1.8.4 +sphinx-autoapi==1.9.0 # via -r doc-requirements.in sphinx-basic-ng==0.0.1a12 # via furo @@ -618,7 +623,7 @@ sphinx-copybutton==0.5.0 # via -r doc-requirements.in sphinx-fontawesome==0.0.6 # via -r doc-requirements.in -sphinx-gallery==0.10.1 +sphinx-gallery==0.11.0 # via -r doc-requirements.in sphinx-material==0.0.35 # via -r doc-requirements.in @@ -642,6 +647,8 @@ sphinxcontrib-yt==0.2.2 # via -r doc-requirements.in sqlalchemy==1.4.39 # via -r doc-requirements.in +stack-data==0.3.0 + # via ipython statsd==3.3.0 # via flytekit tangled-up-in-unicode==0.2.0 @@ -662,9 +669,9 @@ textwrap3==0.9.2 # via ansiwrap tinycss2==1.1.1 # via nbconvert -toolz==0.11.2 +toolz==0.12.0 # via altair -torch==1.11.0 +torch==1.12.0 # via -r doc-requirements.in tornado==6.2 # via @@ -690,21 +697,12 @@ traitlets==5.3.0 # nbformat # notebook # qtconsole -typed-ast==1.5.4 - # via astroid typing-extensions==4.3.0 # via - # argon2-cffi - # arrow # astroid # flytekit # great-expectations - # importlib-metadata - # jsonschema - # kiwisolver - # pandera # pydantic - # responses # torch # typing-inspect typing-inspect==0.7.1 @@ -719,7 +717,7 @@ unidecode==1.3.4 # via # python-slugify # sphinx-autoapi -urllib3==1.26.9 +urllib3==1.26.11 # via # botocore # flytekit @@ -749,7 +747,7 @@ wrapt==1.14.1 # deprecated # flytekit # pandera -zipp==3.8.0 +zipp==3.8.1 # via # importlib-metadata # importlib-resources diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 8398d30d1e..012c993dab 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make requirements-spark2.txt @@ -35,9 +35,7 @@ cookiecutter==2.1.1 croniter==1.3.5 # via flytekit cryptography==37.0.4 - # via - # pyopenssl - # secretstorage + # via pyopenssl dataclasses-json==0.5.7 # via flytekit decorator==5.1.1 @@ -52,9 +50,9 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.14.1 # via flytekit -flyteidl==1.1.8 +flyteidl==1.1.9 # via flytekit -googleapis-common-protos==1.56.3 +googleapis-common-protos==1.56.4 # via # flyteidl # grpcio-status @@ -68,23 +66,19 @@ idna==3.3 # via requests importlib-metadata==4.12.0 # via - # click # flytekit - # jsonschema # keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.2 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter +joblib==1.1.0 + # via flytekit jsonschema==3.2.0 # via -r requirements.in -keyring==23.6.0 +keyring==23.7.0 # via flytekit markupsafe==2.1.1 # via jinja2 @@ -104,7 +98,6 @@ natsort==8.1.0 numpy==1.21.6 # via # -r requirements.in - # flytekit # pandas # pyarrow packaging==21.3 @@ -155,7 +148,7 @@ pyyaml==5.4.1 # -r requirements.in # cookiecutter # flytekit -regex==2022.7.9 +regex==2022.7.25 # via docker-image-py requests==2.28.1 # via @@ -167,10 +160,6 @@ responses==0.21.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.2 - # via keyring -singledispatchmethod==1.0 - # via flytekit six==1.16.0 # via # grpcio @@ -185,14 +174,11 @@ text-unidecode==1.3 # via python-slugify typing-extensions==4.3.0 # via - # arrow # flytekit - # importlib-metadata - # responses # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.10 +urllib3==1.26.11 # via # flytekit # requests @@ -207,7 +193,7 @@ wrapt==1.14.1 # via # deprecated # flytekit -zipp==3.8.0 +zipp==3.8.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements.txt b/requirements.txt index 153da8b4d2..cff0dd1a97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make requirements.txt @@ -33,9 +33,7 @@ cookiecutter==2.1.1 croniter==1.3.5 # via flytekit cryptography==37.0.4 - # via - # pyopenssl - # secretstorage + # via pyopenssl dataclasses-json==0.5.7 # via flytekit decorator==5.1.1 @@ -50,9 +48,9 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.14.1 # via flytekit -flyteidl==1.1.8 +flyteidl==1.1.9 # via flytekit -googleapis-common-protos==1.56.3 +googleapis-common-protos==1.56.4 # via # flyteidl # grpcio-status @@ -66,23 +64,19 @@ idna==3.3 # via requests importlib-metadata==4.12.0 # via - # click # flytekit - # jsonschema # keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.2 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter +joblib==1.1.0 + # via flytekit jsonschema==3.2.0 # via -r requirements.in -keyring==23.6.0 +keyring==23.7.0 # via flytekit markupsafe==2.1.1 # via jinja2 @@ -102,7 +96,6 @@ natsort==8.1.0 numpy==1.21.6 # via # -r requirements.in - # flytekit # pandas # pyarrow packaging==21.3 @@ -153,7 +146,7 @@ pyyaml==5.4.1 # -r requirements.in # cookiecutter # flytekit -regex==2022.7.9 +regex==2022.7.25 # via docker-image-py requests==2.28.1 # via @@ -165,10 +158,6 @@ responses==0.21.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.2 - # via keyring -singledispatchmethod==1.0 - # via flytekit six==1.16.0 # via # grpcio @@ -183,14 +172,11 @@ text-unidecode==1.3 # via python-slugify typing-extensions==4.3.0 # via - # arrow # flytekit - # importlib-metadata - # responses # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.10 +urllib3==1.26.11 # via # flytekit # requests @@ -205,7 +191,7 @@ wrapt==1.14.1 # via # deprecated # flytekit -zipp==3.8.0 +zipp==3.8.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index c93c56435c..99442c4f57 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.7 +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # make tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -27,9 +27,7 @@ cookiecutter==2.1.1 croniter==1.3.5 # via flytekit cryptography==37.0.4 - # via - # pyopenssl - # secretstorage + # via pyopenssl cycler==0.11.0 # via matplotlib dataclasses-json==0.5.7 @@ -46,13 +44,13 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.14.1 # via flytekit -flyteidl==1.1.8 +flyteidl==1.1.9 # via flytekit -flytekit==1.1.0 +flytekit==1.1.1 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -fonttools==4.33.3 +fonttools==4.34.4 # via matplotlib -googleapis-common-protos==1.56.3 +googleapis-common-protos==1.56.4 # via # flyteidl # grpcio-status @@ -66,13 +64,8 @@ idna==3.3 # via requests importlib-metadata==4.12.0 # via - # click # flytekit # keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.2 # via # cookiecutter @@ -81,9 +74,9 @@ jinja2-time==0.2.0 # via cookiecutter joblib==1.1.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -keyring==23.6.0 +keyring==23.7.0 # via flytekit -kiwisolver==1.4.3 +kiwisolver==1.4.4 # via matplotlib markupsafe==2.1.1 # via jinja2 @@ -102,9 +95,8 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.21.6 +numpy==1.23.1 # via - # flytekit # matplotlib # opencv-python # pandas @@ -115,7 +107,7 @@ packaging==21.3 # via # marshmallow # matplotlib -pandas==1.3.5 +pandas==1.4.3 # via flytekit pillow==9.2.0 # via matplotlib @@ -142,11 +134,12 @@ pyparsing==3.0.9 # packaging python-dateutil==2.8.2 # via + # arrow # croniter # flytekit # matplotlib # pandas -python-json-logger==2.0.2 +python-json-logger==2.0.4 # via flytekit python-slugify==6.1.2 # via cookiecutter @@ -160,7 +153,7 @@ pyyaml==6.0 # via # cookiecutter # flytekit -regex==2022.6.2 +regex==2022.7.25 # via docker-image-py requests==2.28.1 # via @@ -172,10 +165,6 @@ responses==0.21.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.2 - # via keyring -singledispatchmethod==1.0 - # via flytekit six==1.16.0 # via # grpcio @@ -188,15 +177,11 @@ text-unidecode==1.3 # via python-slugify typing-extensions==4.3.0 # via - # arrow # flytekit - # importlib-metadata - # kiwisolver - # responses # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.9 +urllib3==1.26.11 # via # flytekit # requests @@ -211,5 +196,5 @@ wrapt==1.14.1 # via # deprecated # flytekit -zipp==3.8.0 +zipp==3.8.1 # via importlib-metadata From 62d97b074d641d4071f09f1bef37d88f73eafa85 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 16 Aug 2022 16:09:05 -0700 Subject: [PATCH 3/5] Fix usage of save in ProtoJoblibHasher Signed-off-by: Eduardo Apolinario --- flytekit/core/local_cache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flytekit/core/local_cache.py b/flytekit/core/local_cache.py index c1f42a36a4..48b6f9c7da 100644 --- a/flytekit/core/local_cache.py +++ b/flytekit/core/local_cache.py @@ -1,8 +1,8 @@ from typing import Optional -import joblib from diskcache import Cache from google.protobuf.struct_pb2 import Struct +from joblib.hashing import NumpyHasher from flytekit.models.literals import Literal, LiteralCollection, LiteralMap @@ -28,7 +28,7 @@ def _recursive_hash_placement(literal: Literal) -> Literal: return literal -class ProtoJoblibHasher(joblib.hashing.NumpyHasher): +class ProtoJoblibHasher(NumpyHasher): def save(self, obj): if isinstance(obj, Struct): obj = dict( @@ -36,7 +36,7 @@ def save(self, obj): cls=obj.__class__, obj=dict(sorted(obj.fields.items())), ) - return obj + NumpyHasher.save(self, obj) def _calculate_cache_key(task_name: str, cache_version: str, input_literal_map: LiteralMap) -> str: From d35c7d21625c260089488c66a1e2bad3001d5882 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 16 Aug 2022 16:46:43 -0700 Subject: [PATCH 4/5] Regenerate requirements using python 3.7 Signed-off-by: Eduardo Apolinario --- dev-requirements.txt | 67 +++++++---- doc-requirements.txt | 109 ++++++++++-------- requirements-spark2.txt | 26 ++++- requirements.txt | 26 ++++- .../workflows/requirements.txt | 34 ++++-- 5 files changed, 169 insertions(+), 93 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index dad926c664..b477f2553b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # make dev-requirements.txt @@ -8,14 +8,10 @@ # via # -c requirements.txt # pytest-flyte -appnope==0.1.3 - # via ipython arrow==1.2.2 # via # -c requirements.txt # jinja2-time -asttokens==2.0.5 - # via stack-data attrs==20.3.0 # via # -c requirements.txt @@ -30,6 +26,8 @@ binaryornot==0.4.4 # via # -c requirements.txt # cookiecutter +cached-property==1.5.2 + # via docker-compose cachetools==5.2.0 # via google-auth certifi==2022.6.15 @@ -67,7 +65,7 @@ cookiecutter==2.1.1 # via # -c requirements.txt # flytekit -coverage[toml]==6.4.2 +coverage[toml]==6.4.4 # via # -r dev-requirements.in # pytest-cov @@ -80,6 +78,7 @@ cryptography==37.0.4 # -c requirements.txt # paramiko # pyopenssl + # secretstorage dataclasses-json==0.5.7 # via # -c requirements.txt @@ -120,11 +119,9 @@ docstring-parser==0.14.1 # via # -c requirements.txt # flytekit -executing==0.9.1 - # via stack-data -filelock==3.7.1 +filelock==3.8.0 # via virtualenv -flyteidl==1.1.9 +flyteidl==1.1.12 # via # -c requirements.txt # flytekit @@ -133,13 +130,13 @@ google-api-core[grpc]==2.8.2 # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-core -google-auth==2.9.1 +google-auth==2.10.0 # via # google-api-core # google-cloud-core -google-cloud-bigquery==3.3.0 +google-cloud-bigquery==3.3.1 # via -r dev-requirements.in -google-cloud-bigquery-storage==2.14.1 +google-cloud-bigquery-storage==2.14.2 # via # -r dev-requirements.in # google-cloud-bigquery @@ -176,14 +173,25 @@ idna==3.3 importlib-metadata==4.12.0 # via # -c requirements.txt + # click # flytekit + # jsonschema # keyring + # pluggy + # pre-commit + # pytest + # virtualenv iniconfig==1.1.1 # via pytest -ipython==8.4.0 +ipython==7.34.0 # via -r dev-requirements.in jedi==0.18.1 # via ipython +jeepney==0.8.0 + # via + # -c requirements.txt + # keyring + # secretstorage jinja2==3.1.2 # via # -c requirements.txt @@ -203,7 +211,7 @@ jsonschema==3.2.0 # via # -c requirements.txt # docker-compose -keyring==23.7.0 +keyring==23.8.2 # via # -c requirements.txt # flytekit @@ -225,7 +233,7 @@ marshmallow-jsonschema==0.13.0 # via # -c requirements.txt # flytekit -matplotlib-inline==0.1.3 +matplotlib-inline==0.1.5 # via ipython mock==4.0.3 # via -r dev-requirements.in @@ -245,6 +253,7 @@ nodeenv==1.7.0 numpy==1.21.6 # via # -c requirements.txt + # flytekit # pandas # pyarrow packaging==21.3 @@ -273,7 +282,7 @@ pre-commit==2.20.0 # via -r dev-requirements.in prompt-toolkit==3.0.30 # via ipython -proto-plus==1.20.6 +proto-plus==1.22.0 # via # google-cloud-bigquery # google-cloud-bigquery-storage @@ -295,8 +304,6 @@ protoc-gen-swagger==0.1.0 # flyteidl ptyprocess==0.7.0 # via pexpect -pure-eval==0.2.2 - # via stack-data py==1.11.0 # via # -c requirements.txt @@ -317,7 +324,7 @@ pycparser==2.21 # via # -c requirements.txt # cffi -pygments==2.12.0 +pygments==2.13.0 # via ipython pynacl==1.5.0 # via paramiko @@ -367,7 +374,7 @@ pytimeparse==1.1.8 # via # -c requirements.txt # flytekit -pytz==2022.1 +pytz==2022.2.1 # via # -c requirements.txt # flytekit @@ -403,10 +410,17 @@ retry==0.9.2 # flytekit rsa==4.9 # via google-auth +secretstorage==3.3.3 + # via + # -c requirements.txt + # keyring +singledispatchmethod==1.0 + # via + # -c requirements.txt + # flytekit six==1.16.0 # via # -c requirements.txt - # asttokens # dockerpty # google-auth # grpcio @@ -418,8 +432,6 @@ sortedcontainers==2.4.0 # via # -c requirements.txt # flytekit -stack-data==0.3.0 - # via ipython statsd==3.3.0 # via # -c requirements.txt @@ -437,17 +449,22 @@ tomli==2.0.1 # coverage # mypy # pytest -torch==1.12.0 +torch==1.12.1 # via -r dev-requirements.in traitlets==5.3.0 # via # ipython # matplotlib-inline +typed-ast==1.5.4 + # via mypy typing-extensions==4.3.0 # via # -c requirements.txt + # arrow # flytekit + # importlib-metadata # mypy + # responses # torch # typing-inspect typing-inspect==0.7.1 diff --git a/doc-requirements.txt b/doc-requirements.txt index 07c9f9e770..5bfb8c5d31 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # make doc-requirements.txt @@ -12,10 +12,6 @@ altair==4.2.0 # via great-expectations ansiwrap==0.8.4 # via papermill -appnope==0.1.3 - # via - # ipykernel - # ipython argon2-cffi==21.3.0 # via notebook argon2-cffi-bindings==21.2.0 @@ -24,8 +20,6 @@ arrow==1.2.2 # via jinja2-time astroid==2.12.2 # via sphinx-autoapi -asttokens==2.0.5 - # via stack-data attrs==22.1.0 # via # jsonschema @@ -48,7 +42,7 @@ binaryornot==0.4.4 # via cookiecutter bleach==5.0.1 # via nbconvert -botocore==1.27.46 +botocore==1.27.53 # via -r doc-requirements.in cachetools==5.2.0 # via google-auth @@ -83,6 +77,7 @@ cryptography==37.0.4 # -r doc-requirements.in # great-expectations # pyopenssl + # secretstorage css-html-js-minify==2.5.5 # via sphinx-material cycler==0.11.0 @@ -91,7 +86,7 @@ dataclasses-json==0.5.7 # via # dolt-integrations # flytekit -debugpy==1.6.2 +debugpy==1.6.3 # via ipykernel decorator==5.1.1 # via @@ -123,13 +118,11 @@ entrypoints==0.4 # jupyter-client # nbconvert # papermill -executing==0.9.1 - # via stack-data fastjsonschema==2.16.1 # via nbformat -flyteidl==1.1.9 +flyteidl==1.1.12 # via flytekit -fonttools==4.34.4 +fonttools==4.35.0 # via matplotlib fsspec==2022.7.1 # via @@ -142,16 +135,16 @@ google-api-core[grpc]==2.8.2 # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-core -google-auth==2.9.1 +google-auth==2.10.0 # via # google-api-core # google-cloud-core # kubernetes google-cloud==0.34.0 # via -r doc-requirements.in -google-cloud-bigquery==3.3.0 +google-cloud-bigquery==3.3.1 # via -r doc-requirements.in -google-cloud-bigquery-storage==2.14.1 +google-cloud-bigquery-storage==2.14.2 # via google-cloud-bigquery google-cloud-core==2.3.2 # via google-cloud-bigquery @@ -164,7 +157,7 @@ googleapis-common-protos==1.56.4 # flyteidl # google-api-core # grpcio-status -great-expectations==0.15.17 +great-expectations==0.15.18 # via -r doc-requirements.in greenlet==1.1.2 # via sqlalchemy @@ -189,11 +182,14 @@ imagesize==1.4.1 # via sphinx importlib-metadata==4.12.0 # via + # click # flytekit # great-expectations + # jsonschema # keyring # markdown # sphinx + # sqlalchemy importlib-resources==5.9.0 # via jsonschema ipykernel==6.15.1 @@ -203,7 +199,7 @@ ipykernel==6.15.1 # jupyter-console # notebook # qtconsole -ipython==8.4.0 +ipython==7.34.0 # via # great-expectations # ipykernel @@ -220,6 +216,10 @@ ipywidgets==7.7.1 # jupyter jedi==0.18.1 # via ipython +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.2 # via # altair @@ -244,7 +244,7 @@ jsonpatch==1.32 # via great-expectations jsonpointer==2.3 # via jsonpatch -jsonschema==4.9.1 +jsonschema==4.10.0 # via # altair # great-expectations @@ -271,7 +271,7 @@ jupyterlab-pygments==0.2.2 # via nbconvert jupyterlab-widgets==1.1.1 # via ipywidgets -keyring==23.7.0 +keyring==23.8.2 # via flytekit kiwisolver==1.4.4 # via matplotlib @@ -280,7 +280,11 @@ kubernetes==24.2.0 lazy-object-proxy==1.7.1 # via astroid lxml==4.9.1 - # via sphinx-material + # via + # nbconvert + # sphinx-material +makefun==1.14.0 + # via great-expectations markdown==3.4.1 # via -r doc-requirements.in markupsafe==2.1.1 @@ -297,13 +301,13 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib==3.5.2 +matplotlib==3.5.3 # via # missingno # pandas-profiling # phik # seaborn -matplotlib-inline==0.1.3 +matplotlib-inline==0.1.5 # via # ipykernel # ipython @@ -313,7 +317,7 @@ mistune==0.8.4 # via # great-expectations # nbconvert -modin==0.15.2 +modin==0.12.1 # via -r doc-requirements.in multimethod==1.8 # via @@ -327,7 +331,7 @@ nbclient==0.6.6 # via # nbconvert # papermill -nbconvert==6.5.0 +nbconvert==6.5.3 # via # jupyter # notebook @@ -344,16 +348,17 @@ nest-asyncio==1.5.5 # jupyter-client # nbclient # notebook -networkx==2.8.5 +networkx==2.6.3 # via visions notebook==6.4.12 # via # great-expectations # jupyter # widgetsnbextension -numpy==1.23.1 +numpy==1.21.6 # via # altair + # flytekit # great-expectations # imagehash # matplotlib @@ -382,7 +387,7 @@ packaging==21.3 # pandera # qtpy # sphinx -pandas==1.4.3 +pandas==1.3.5 # via # altair # dolt-integrations @@ -396,11 +401,11 @@ pandas==1.4.3 # visions pandas-profiling==3.2.0 # via -r doc-requirements.in -pandera==0.11.0 +pandera==0.9.0 # via -r doc-requirements.in pandocfilters==1.5.0 # via nbconvert -papermill==2.3.4 +papermill==2.4.0 # via -r doc-requirements.in parso==0.8.3 # via jedi @@ -417,7 +422,7 @@ pillow==9.2.0 # visions pkgutil-resolve-name==1.3.10 # via jsonschema -plotly==5.9.0 +plotly==5.10.0 # via -r doc-requirements.in prometheus-client==0.14.1 # via notebook @@ -425,7 +430,7 @@ prompt-toolkit==3.0.30 # via # ipython # jupyter-console -proto-plus==1.20.6 +proto-plus==1.22.0 # via # google-cloud-bigquery # google-cloud-bigquery-storage @@ -443,15 +448,11 @@ protobuf==3.20.1 protoc-gen-swagger==0.1.0 # via flyteidl psutil==5.9.1 - # via - # ipykernel - # modin + # via ipykernel ptyprocess==0.7.0 # via # pexpect # terminado -pure-eval==0.2.2 - # via stack-data py==1.11.0 # via retry py4j==0.10.9.5 @@ -469,11 +470,11 @@ pyasn1-modules==0.2.8 # via google-auth pycparser==2.21 # via cffi -pydantic==1.9.1 +pydantic==1.9.2 # via # pandas-profiling # pandera -pygments==2.12.0 +pygments==2.13.0 # via # furo # ipython @@ -513,7 +514,7 @@ python-slugify[unidecode]==6.1.2 # sphinx-material pytimeparse==1.1.8 # via flytekit -pytz==2022.1 +pytz==2022.2.1 # via # babel # flytekit @@ -531,7 +532,7 @@ pyyaml==6.0 # pandas-profiling # papermill # sphinx-autoapi -pyzmq==23.2.0 +pyzmq==23.2.1 # via # ipykernel # jupyter-client @@ -539,7 +540,7 @@ pyzmq==23.2.0 # qtconsole qtconsole==5.3.1 # via jupyter -qtpy==2.1.0 +qtpy==2.2.0 # via qtconsole regex==2022.7.25 # via docker-image-py @@ -569,7 +570,7 @@ ruamel-yaml==0.17.17 # via great-expectations ruamel-yaml-clib==0.2.6 # via ruamel-yaml -scipy==1.9.0 +scipy==1.7.3 # via # great-expectations # imagehash @@ -581,11 +582,14 @@ seaborn==0.11.2 # via # missingno # pandas-profiling +secretstorage==3.3.3 + # via keyring send2trash==1.8.0 # via notebook +singledispatchmethod==1.0 + # via flytekit six==1.16.0 # via - # asttokens # bleach # google-auth # grpcio @@ -645,10 +649,8 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-yt==0.2.2 # via -r doc-requirements.in -sqlalchemy==1.4.39 +sqlalchemy==1.4.40 # via -r doc-requirements.in -stack-data==0.3.0 - # via ipython statsd==3.3.0 # via flytekit tangled-up-in-unicode==0.2.0 @@ -671,7 +673,7 @@ tinycss2==1.1.1 # via nbconvert toolz==0.12.0 # via altair -torch==1.12.0 +torch==1.12.1 # via -r doc-requirements.in tornado==6.2 # via @@ -697,19 +699,28 @@ traitlets==5.3.0 # nbformat # notebook # qtconsole +typed-ast==1.5.4 + # via astroid typing-extensions==4.3.0 # via + # argon2-cffi + # arrow # astroid # flytekit # great-expectations + # importlib-metadata + # jsonschema + # kiwisolver + # pandera # pydantic + # responses # torch # typing-inspect typing-inspect==0.7.1 # via # dataclasses-json # pandera -tzdata==2022.1 +tzdata==2022.2 # via pytz-deprecation-shim tzlocal==4.2 # via great-expectations diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 012c993dab..ded88f1bb2 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # make requirements-spark2.txt @@ -35,7 +35,9 @@ cookiecutter==2.1.1 croniter==1.3.5 # via flytekit cryptography==37.0.4 - # via pyopenssl + # via + # pyopenssl + # secretstorage dataclasses-json==0.5.7 # via flytekit decorator==5.1.1 @@ -50,7 +52,7 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.14.1 # via flytekit -flyteidl==1.1.9 +flyteidl==1.1.12 # via flytekit googleapis-common-protos==1.56.4 # via @@ -66,8 +68,14 @@ idna==3.3 # via requests importlib-metadata==4.12.0 # via + # click # flytekit + # jsonschema + # keyring +jeepney==0.8.0 + # via # keyring + # secretstorage jinja2==3.1.2 # via # cookiecutter @@ -78,7 +86,7 @@ joblib==1.1.0 # via flytekit jsonschema==3.2.0 # via -r requirements.in -keyring==23.7.0 +keyring==23.8.2 # via flytekit markupsafe==2.1.1 # via jinja2 @@ -98,6 +106,7 @@ natsort==8.1.0 numpy==1.21.6 # via # -r requirements.in + # flytekit # pandas # pyarrow packaging==21.3 @@ -139,7 +148,7 @@ python-slugify==6.1.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.1 +pytz==2022.2.1 # via # flytekit # pandas @@ -160,6 +169,10 @@ responses==0.21.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.3 + # via keyring +singledispatchmethod==1.0 + # via flytekit six==1.16.0 # via # grpcio @@ -174,7 +187,10 @@ text-unidecode==1.3 # via python-slugify typing-extensions==4.3.0 # via + # arrow # flytekit + # importlib-metadata + # responses # typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/requirements.txt b/requirements.txt index cff0dd1a97..17a6487f7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # make requirements.txt @@ -33,7 +33,9 @@ cookiecutter==2.1.1 croniter==1.3.5 # via flytekit cryptography==37.0.4 - # via pyopenssl + # via + # pyopenssl + # secretstorage dataclasses-json==0.5.7 # via flytekit decorator==5.1.1 @@ -48,7 +50,7 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.14.1 # via flytekit -flyteidl==1.1.9 +flyteidl==1.1.12 # via flytekit googleapis-common-protos==1.56.4 # via @@ -64,8 +66,14 @@ idna==3.3 # via requests importlib-metadata==4.12.0 # via + # click # flytekit + # jsonschema + # keyring +jeepney==0.8.0 + # via # keyring + # secretstorage jinja2==3.1.2 # via # cookiecutter @@ -76,7 +84,7 @@ joblib==1.1.0 # via flytekit jsonschema==3.2.0 # via -r requirements.in -keyring==23.7.0 +keyring==23.8.2 # via flytekit markupsafe==2.1.1 # via jinja2 @@ -96,6 +104,7 @@ natsort==8.1.0 numpy==1.21.6 # via # -r requirements.in + # flytekit # pandas # pyarrow packaging==21.3 @@ -137,7 +146,7 @@ python-slugify==6.1.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.1 +pytz==2022.2.1 # via # flytekit # pandas @@ -158,6 +167,10 @@ responses==0.21.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.3 + # via keyring +singledispatchmethod==1.0 + # via flytekit six==1.16.0 # via # grpcio @@ -172,7 +185,10 @@ text-unidecode==1.3 # via python-slugify typing-extensions==4.3.0 # via + # arrow # flytekit + # importlib-metadata + # responses # typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 99442c4f57..0fc659135f 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # make tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -27,7 +27,9 @@ cookiecutter==2.1.1 croniter==1.3.5 # via flytekit cryptography==37.0.4 - # via pyopenssl + # via + # pyopenssl + # secretstorage cycler==0.11.0 # via matplotlib dataclasses-json==0.5.7 @@ -44,11 +46,11 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.14.1 # via flytekit -flyteidl==1.1.9 +flyteidl==1.1.12 # via flytekit flytekit==1.1.1 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -fonttools==4.34.4 +fonttools==4.35.0 # via matplotlib googleapis-common-protos==1.56.4 # via @@ -64,8 +66,13 @@ idna==3.3 # via requests importlib-metadata==4.12.0 # via + # click # flytekit # keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.2 # via # cookiecutter @@ -74,7 +81,7 @@ jinja2-time==0.2.0 # via cookiecutter joblib==1.1.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -keyring==23.7.0 +keyring==23.8.2 # via flytekit kiwisolver==1.4.4 # via matplotlib @@ -89,14 +96,15 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib==3.5.2 +matplotlib==3.5.3 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.23.1 +numpy==1.21.6 # via + # flytekit # matplotlib # opencv-python # pandas @@ -107,7 +115,7 @@ packaging==21.3 # via # marshmallow # matplotlib -pandas==1.4.3 +pandas==1.3.5 # via flytekit pillow==9.2.0 # via matplotlib @@ -145,7 +153,7 @@ python-slugify==6.1.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2022.1 +pytz==2022.2.1 # via # flytekit # pandas @@ -165,6 +173,10 @@ responses==0.21.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.3 + # via keyring +singledispatchmethod==1.0 + # via flytekit six==1.16.0 # via # grpcio @@ -177,7 +189,11 @@ text-unidecode==1.3 # via python-slugify typing-extensions==4.3.0 # via + # arrow # flytekit + # importlib-metadata + # kiwisolver + # responses # typing-inspect typing-inspect==0.7.1 # via dataclasses-json From d1bbf289f366b8963668f54197e6dde5d83aad32 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 17 Aug 2022 15:14:14 -0700 Subject: [PATCH 5/5] Add test_stable_cache_key Signed-off-by: Eduardo Apolinario --- tests/flytekit/unit/core/test_local_cache.py | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py index 5a6d2d3853..674f6176e1 100644 --- a/tests/flytekit/unit/core/test_local_cache.py +++ b/tests/flytekit/unit/core/test_local_cache.py @@ -19,6 +19,7 @@ from flytekit.core.type_engine import TypeEngine from flytekit.core.workflow import workflow from flytekit.models.literals import LiteralMap +from flytekit.models.types import LiteralType, SimpleType from flytekit.types.schema import FlyteSchema # Global counter used to validate number of calls to cache @@ -411,3 +412,29 @@ def test_cache_key_repetition(): keys.add(key) assert len(keys) == 1 + + +def test_stable_cache_key(): + """ + The intent of this test is to ensure cache keys are stable across releases and python versions. + """ + pt = Dict + lt = TypeEngine.to_literal_type(pt) + ctx = FlyteContextManager.current_context() + kwargs = { + "a": 42, + "b": "abcd", + "c": 0.12349, + "d": [1, 2, 3], + } + lit = TypeEngine.to_literal(ctx, kwargs, Dict, lt) + lm = LiteralMap( + literals={ + "lit_1": lit, + "lit_2": TypeEngine.to_literal(ctx, 99, int, LiteralType(simple=SimpleType.INTEGER)), + "lit_3": TypeEngine.to_literal(ctx, 3.14, float, LiteralType(simple=SimpleType.FLOAT)), + "lit_4": TypeEngine.to_literal(ctx, True, bool, LiteralType(simple=SimpleType.BOOLEAN)), + } + ) + key = _calculate_cache_key("task_name_1", "31415", lm) + assert key == "task_name_1-31415-a291dc6fe0be387c1cfd67b4c6b78259"