From fd5dea3bc37c229ba52a5701d286e1edd45ccc3e Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 27 Oct 2021 19:08:19 -0700 Subject: [PATCH 01/20] Add cookiecutter to requirements and pyflyte init Signed-off-by: Eduardo Apolinario --- dev-requirements.txt | 22 ++++++---- flytekit/clis/sdk_in_container/init.py | 9 ++++ flytekit/clis/sdk_in_container/pyflyte.py | 2 + requirements.in | 1 + requirements.txt | 51 +++++++++++++++++------ 5 files changed, 63 insertions(+), 22 deletions(-) create mode 100644 flytekit/clis/sdk_in_container/init.py diff --git a/dev-requirements.txt b/dev-requirements.txt index 9911ffbc22..796acb43c2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -44,6 +44,10 @@ click==7.1.2 # via # -c requirements.txt # flytekit +cloudpickle==2.0.0 + # via + # -c requirements.txt + # flytekit codespell==2.1.0 # via -r dev-requirements.in coverage[toml]==6.0.2 @@ -97,15 +101,15 @@ docstring-parser==0.12 # flytekit filelock==3.3.1 # via virtualenv -flyteidl==0.21.6 +flyteidl==0.21.7 # via # -c requirements.txt # flytekit -grpcio==1.41.0 +grpcio==1.41.1 # via # -c requirements.txt # flytekit -identify==2.3.0 +identify==2.3.1 # via pre-commit idna==3.3 # via @@ -150,7 +154,7 @@ marshmallow-enum==1.5.1 # via # -c requirements.txt # dataclasses-json -marshmallow-jsonschema==0.12.0 +marshmallow-jsonschema==0.13.0 # via # -c requirements.txt # flytekit @@ -216,7 +220,7 @@ pynacl==1.4.0 # via # -c requirements.txt # paramiko -pyparsing==2.4.7 +pyparsing==3.0.3 # via # -c requirements.txt # packaging @@ -259,7 +263,7 @@ pyyaml==5.4.1 # -c requirements.txt # docker-compose # pre-commit -regex==2021.10.21 +regex==2021.10.23 # via # -c requirements.txt # docker-image-py @@ -270,7 +274,7 @@ requests==2.26.0 # docker-compose # flytekit # responses -responses==0.14.0 +responses==0.15.0 # via # -c requirements.txt # flytekit @@ -310,7 +314,7 @@ toml==0.10.2 # mypy # pre-commit # pytest -tomli==1.2.1 +tomli==1.2.2 # via # -c requirements.txt # coverage @@ -329,7 +333,7 @@ urllib3==1.26.7 # flytekit # requests # responses -virtualenv==20.8.1 +virtualenv==20.9.0 # via pre-commit websocket-client==0.59.0 # via diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py new file mode 100644 index 0000000000..87834092ca --- /dev/null +++ b/flytekit/clis/sdk_in_container/init.py @@ -0,0 +1,9 @@ +import click + + +@click.command("init") +def init(): + """ + TODO + """ + print("bla") diff --git a/flytekit/clis/sdk_in_container/pyflyte.py b/flytekit/clis/sdk_in_container/pyflyte.py index 676b1bc675..b23fa5d121 100644 --- a/flytekit/clis/sdk_in_container/pyflyte.py +++ b/flytekit/clis/sdk_in_container/pyflyte.py @@ -6,6 +6,7 @@ from flytekit.clis.sdk_in_container.constants import CTX_PACKAGES from flytekit.clis.sdk_in_container.fast_register import fast_register +from flytekit.clis.sdk_in_container.init import init from flytekit.clis.sdk_in_container.launch_plan import launch_plans from flytekit.clis.sdk_in_container.local_cache import local_cache from flytekit.clis.sdk_in_container.package import package @@ -112,6 +113,7 @@ def update_configuration_file(config_file_path): main.add_command(launch_plans) main.add_command(package) main.add_command(local_cache) +main.add_command(init) if __name__ == "__main__": main() diff --git a/requirements.in b/requirements.in index 408185fca8..3cba59729d 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,6 @@ .[all] -e file:.#egg=flytekit +cookiecutter attrs<21 # We need to restrict constrain the versions of both jsonschema and pyyaml because of docker-compose (which is # used to run integration tests) pins those 2 libraries. We are in the process of removing docker-compose in diff --git a/requirements.txt b/requirements.txt index 929bd0c36e..22ef535bb3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,8 @@ # via -r requirements.in ansiwrap==0.8.4 # via papermill +arrow==1.2.1 + # via jinja2-time attrs==20.3.0 # via # -r requirements.in @@ -16,13 +18,15 @@ backcall==0.2.0 # via ipython bcrypt==3.2.0 # via paramiko +binaryornot==0.4.4 + # via cookiecutter black==21.9b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.0 +boto3==1.19.5 # via sagemaker-training -botocore==1.22.0 +botocore==1.22.5 # via # boto3 # s3transfer @@ -33,6 +37,8 @@ cffi==1.15.0 # bcrypt # cryptography # pynacl +chardet==4.0.0 + # via binaryornot charset-normalizer==2.0.7 # via requests checksumdir==1.2.0 @@ -40,9 +46,14 @@ checksumdir==1.2.0 click==7.1.2 # via # black + # cookiecutter # flytekit # hmsclient # papermill +cloudpickle==2.0.0 + # via flytekit +cookiecutter==1.7.3 + # via -r requirements.in croniter==1.0.15 # via flytekit cryptography==35.0.0 @@ -70,13 +81,13 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.6 +flyteidl==0.21.7 # via flytekit gevent==21.8.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.41.0 +grpcio==1.41.1 # via flytekit hmsclient==0.1.1 # via flytekit @@ -101,7 +112,12 @@ jeepney==0.7.1 # keyring # secretstorage jinja2==3.0.2 - # via nbconvert + # via + # cookiecutter + # jinja2-time + # nbconvert +jinja2-time==0.2.0 + # via cookiecutter jmespath==0.10.0 # via # boto3 @@ -114,7 +130,7 @@ jupyter-client==7.0.6 # via # ipykernel # nbclient -jupyter-core==4.8.1 +jupyter-core==4.9.0 # via # jupyter-client # nbconvert @@ -134,7 +150,7 @@ marshmallow==3.14.0 # marshmallow-jsonschema marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow-jsonschema==0.12.0 +marshmallow-jsonschema==0.13.0 # via flytekit matplotlib-inline==0.1.3 # via ipython @@ -188,7 +204,9 @@ pickleshare==0.7.5 # via ipython platformdirs==2.4.0 # via black -prompt-toolkit==3.0.20 +poyo==0.5.0 + # via cookiecutter +prompt-toolkit==3.0.21 # via ipython protobuf==3.19.0 # via @@ -215,7 +233,7 @@ pygments==2.10.0 # nbconvert pynacl==1.4.0 # via paramiko -pyparsing==2.4.7 +pyparsing==3.0.3 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -223,6 +241,7 @@ pyspark==3.2.0 # via flytekit python-dateutil==2.8.1 # via + # arrow # botocore # croniter # flytekit @@ -230,6 +249,8 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit +python-slugify==5.0.2 + # via cookiecutter pytimeparse==1.1.8 # via flytekit pytz==2018.4 @@ -242,16 +263,17 @@ pyyaml==5.4.1 # papermill pyzmq==22.3.0 # via jupyter-client -regex==2021.10.21 +regex==2021.10.23 # via # black # docker-image-py requests==2.26.0 # via + # cookiecutter # flytekit # papermill # responses -responses==0.14.0 +responses==0.15.0 # via flytekit retry==0.9.2 # via flytekit @@ -269,6 +291,7 @@ six==1.16.0 # via # bcrypt # bleach + # cookiecutter # flytekit # grpcio # jsonschema @@ -286,11 +309,13 @@ tenacity==8.0.1 # via papermill testpath==0.5.0 # via nbconvert +text-unidecode==1.3 + # via python-slugify textwrap3==0.9.2 # via ansiwrap thrift==0.15.0 # via hmsclient -tomli==1.2.1 +tomli==1.2.2 # via black tornado==6.1 # via @@ -298,7 +323,7 @@ tornado==6.1 # jupyter-client tqdm==4.62.3 # via papermill -traitlets==5.1.0 +traitlets==5.1.1 # via # ipykernel # ipython From 79da4241c0ccfe9fd6d387ac5c9729b411ad2245 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 28 Oct 2021 16:08:03 -0700 Subject: [PATCH 02/20] Invoke cookiecutter with the overridden configuration. Signed-off-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/init.py | 28 ++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index 87834092ca..39947b40c2 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -1,9 +1,33 @@ import click +from cookiecutter.main import cookiecutter +from cookiecutter.prompt import read_user_variable @click.command("init") def init(): """ - TODO + Create a flyte-ready project. """ - print("bla") + # TODO: Add `interactive` boolean flag to allow for programmatic access to this subcommand. + + print("What is the name of your project? This is the name of the directory that will be created.") + project_name = read_user_variable("project_name", "flyte_example") + print("What should we call your application? This serves as the top level package where your workflows will live.") + app = read_user_variable("app", "myapp") + print("What should be the name of your example workflow?") + workflow_name = read_user_variable("workflow", "workflow_example") + + config = { + "project_name": project_name, + "app": app, + "workflow": workflow_name, + } + cookiecutter( + # TODO move this to the main repo. Possibly maintain a branch before cutting over the documentation. + "https://github.com/eapolinario/flytekit-python-template.git", + no_input=True, + extra_context=config, + # By specifying directory we can have multiple templates in the same repository, + # as described in https://cookiecutter.readthedocs.io/en/1.7.2/advanced/directories.html. + directory="simple-example", + ) From 38b0d881429f74b9a79773b7460bf666ffccf31a Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 28 Oct 2021 16:16:12 -0700 Subject: [PATCH 03/20] Comment the use of directory in the call to cookiecutter. Signed-off-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/init.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index 39947b40c2..0bee8fdf33 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -29,5 +29,7 @@ def init(): extra_context=config, # By specifying directory we can have multiple templates in the same repository, # as described in https://cookiecutter.readthedocs.io/en/1.7.2/advanced/directories.html. + # The idea is to extend the number of templates, each in their own subdirectory, for example + # a tensorflow-based example. directory="simple-example", ) From 688f214ab67887f1c384ce211f3d6e34aec38178 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 4 Nov 2021 15:18:19 -0700 Subject: [PATCH 04/20] Use the original repo and do not clobber existing files+directories Signed-off-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/init.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index 0bee8fdf33..dcf1ef8c3c 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -24,8 +24,12 @@ def init(): } cookiecutter( # TODO move this to the main repo. Possibly maintain a branch before cutting over the documentation. - "https://github.com/eapolinario/flytekit-python-template.git", + "https://github.com/flyteorg/flytekit-python-template.git", + # TODO: remove this once we make the transition to cookie-cutter official. + checkout="cookie-cutter", no_input=True, + # We do not want to clobber existing files/directories. + overwrite_if_exists=False, extra_context=config, # By specifying directory we can have multiple templates in the same repository, # as described in https://cookiecutter.readthedocs.io/en/1.7.2/advanced/directories.html. From d2130cc3a7cb4fd8ddc823c07f7ea98e539cc3af Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 4 Nov 2021 15:34:21 -0700 Subject: [PATCH 05/20] Add example sub-command Signed-off-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/init.py | 27 +++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index dcf1ef8c3c..a17da0a547 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -3,13 +3,32 @@ from cookiecutter.prompt import read_user_variable -@click.command("init") +@click.group("init") def init(): """ - Create a flyte-ready project. + Create flyte-ready projects. """ # TODO: Add `interactive` boolean flag to allow for programmatic access to this subcommand. + pass + +@click.command("example") +def generate_simple_example(): + """ + This command creates a directory containing the minimal code necessary to start a Flyte-ready project. + The generated directory structure for the default values resemble: + flyte_example + ├── Dockerfile + ├── flyte.config + ├── LICENSE + ├── myapp + │   ├── __init__.py + │   └── workflows + │   ├── example.py + │   └── __init__.py + ├── README.md + └── requirements.txt + """ print("What is the name of your project? This is the name of the directory that will be created.") project_name = read_user_variable("project_name", "flyte_example") print("What should we call your application? This serves as the top level package where your workflows will live.") @@ -23,7 +42,6 @@ def init(): "workflow": workflow_name, } cookiecutter( - # TODO move this to the main repo. Possibly maintain a branch before cutting over the documentation. "https://github.com/flyteorg/flytekit-python-template.git", # TODO: remove this once we make the transition to cookie-cutter official. checkout="cookie-cutter", @@ -37,3 +55,6 @@ def init(): # a tensorflow-based example. directory="simple-example", ) + + +init.add_command(generate_simple_example) From 79e9c1d99d7b3fc194f74574cfae3e0600ba6fc1 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 4 Nov 2021 16:54:52 -0700 Subject: [PATCH 06/20] make requirements Signed-off-by: Eduardo Apolinario --- dev-requirements.txt | 24 ++++----- doc-requirements.txt | 52 +++++++++++-------- requirements-spark2.txt | 38 +++++++------- requirements.txt | 28 +++++----- .../workflows/requirements.txt | 25 +++++---- 5 files changed, 87 insertions(+), 80 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 796acb43c2..a9092b933d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -50,7 +50,7 @@ cloudpickle==2.0.0 # flytekit codespell==2.1.0 # via -r dev-requirements.in -coverage[toml]==6.0.2 +coverage[toml]==6.1.1 # via -r dev-requirements.in croniter==1.0.15 # via @@ -99,9 +99,9 @@ docstring-parser==0.12 # via # -c requirements.txt # flytekit -filelock==3.3.1 +filelock==3.3.2 # via virtualenv -flyteidl==0.21.7 +flyteidl==0.21.8 # via # -c requirements.txt # flytekit @@ -109,7 +109,7 @@ grpcio==1.41.1 # via # -c requirements.txt # flytekit -identify==2.3.1 +identify==2.3.3 # via pre-commit idna==3.3 # via @@ -167,7 +167,7 @@ mypy-extensions==0.4.3 # -c requirements.txt # mypy # typing-inspect -natsort==7.1.1 +natsort==8.0.0 # via # -c requirements.txt # flytekit @@ -178,7 +178,7 @@ numpy==1.21.3 # -c requirements.txt # pandas # pyarrow -packaging==21.0 +packaging==21.2 # via # -c requirements.txt # pytest @@ -198,12 +198,12 @@ pluggy==1.0.0 # via pytest pre-commit==2.15.0 # via -r dev-requirements.in -protobuf==3.19.0 +protobuf==3.19.1 # via # -c requirements.txt # flyteidl # flytekit -py==1.10.0 +py==1.11.0 # via # -c requirements.txt # pytest @@ -220,7 +220,7 @@ pynacl==1.4.0 # via # -c requirements.txt # paramiko -pyparsing==3.0.3 +pyparsing==2.4.7 # via # -c requirements.txt # packaging @@ -263,7 +263,7 @@ pyyaml==5.4.1 # -c requirements.txt # docker-compose # pre-commit -regex==2021.10.23 +regex==2021.11.2 # via # -c requirements.txt # docker-image-py @@ -333,7 +333,7 @@ urllib3==1.26.7 # flytekit # requests # responses -virtualenv==20.9.0 +virtualenv==20.10.0 # via pre-commit websocket-client==0.59.0 # via @@ -343,7 +343,7 @@ wheel==0.37.0 # via # -c requirements.txt # flytekit -wrapt==1.13.2 +wrapt==1.13.3 # via # -c requirements.txt # deprecated diff --git a/doc-requirements.txt b/doc-requirements.txt index 02f4e05b0d..edef3bc605 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -10,7 +10,7 @@ alabaster==0.7.12 # via sphinx ansiwrap==0.8.4 # via papermill -astroid==2.8.3 +astroid==2.8.4 # via sphinx-autoapi attrs==21.2.0 # via jsonschema @@ -25,13 +25,13 @@ beautifulsoup4==4.10.0 # furo # sphinx-code-include # sphinx-material -black==21.9b0 +black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.0 +boto3==1.19.10 # via sagemaker-training -botocore==1.22.0 +botocore==1.22.11 # via # boto3 # s3transfer @@ -52,6 +52,8 @@ click==7.1.2 # flytekit # hmsclient # papermill +cloudpickle==2.0.0 + # via flytekit croniter==1.0.15 # via flytekit cryptography==35.0.0 @@ -84,7 +86,7 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.6 +flyteidl==0.21.8 # via flytekit furo @ git+git://github.com/flyteorg/furo@main # via -r doc-requirements.in @@ -92,7 +94,7 @@ gevent==21.8.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.41.0 +grpcio==1.41.1 # via # -r doc-requirements.in # flytekit @@ -104,11 +106,13 @@ imagesize==1.2.0 # via sphinx importlib-metadata==4.8.1 # via keyring +importlib-resources==5.4.0 + # via jsonschema inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.6 # via flytekit -ipython==7.28.0 +ipython==7.29.0 # via ipykernel ipython-genutils==0.2.0 # via @@ -129,13 +133,13 @@ jmespath==0.10.0 # via # boto3 # botocore -jsonschema==4.1.2 +jsonschema==4.2.0 # via nbformat jupyter-client==7.0.6 # via # ipykernel # nbclient -jupyter-core==4.8.1 +jupyter-core==4.9.1 # via # jupyter-client # nbconvert @@ -148,7 +152,7 @@ keyring==23.2.1 # via flytekit lazy-object-proxy==1.6.0 # via astroid -lxml==4.6.3 +lxml==4.6.4 # via sphinx-material markupsafe==2.0.1 # via jinja2 @@ -159,7 +163,7 @@ marshmallow==3.14.0 # marshmallow-jsonschema marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow-jsonschema==0.12.0 +marshmallow-jsonschema==0.13.0 # via flytekit matplotlib-inline==0.1.3 # via ipython @@ -169,7 +173,7 @@ mypy-extensions==0.4.3 # via # black # typing-inspect -natsort==7.1.1 +natsort==8.0.0 # via flytekit nbclient==0.5.4 # via @@ -193,7 +197,7 @@ numpy==1.21.3 # pyarrow # sagemaker-training # scipy -packaging==21.0 +packaging==21.2 # via # bleach # sphinx @@ -215,9 +219,9 @@ pickleshare==0.7.5 # via ipython platformdirs==2.4.0 # via black -prompt-toolkit==3.0.20 +prompt-toolkit==3.0.22 # via ipython -protobuf==3.19.0 +protobuf==3.19.1 # via # flyteidl # flytekit @@ -227,7 +231,7 @@ psutil==5.8.0 # via sagemaker-training ptyprocess==0.7.0 # via pexpect -py==1.10.0 +py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark @@ -274,7 +278,7 @@ pyyaml==6.0 # sphinx-autoapi pyzmq==22.3.0 # via jupyter-client -regex==2021.10.21 +regex==2021.11.2 # via # black # docker-image-py @@ -284,7 +288,7 @@ requests==2.26.0 # papermill # responses # sphinx -responses==0.14.0 +responses==0.15.0 # via flytekit retry==0.9.2 # via flytekit @@ -315,7 +319,7 @@ snowballstemmer==2.1.0 # via sphinx sortedcontainers==2.4.0 # via flytekit -soupsieve==2.2.1 +soupsieve==2.3 # via beautifulsoup4 sphinx==4.2.0 # via @@ -369,7 +373,7 @@ textwrap3==0.9.2 # via ansiwrap thrift==0.15.0 # via hmsclient -tomli==1.2.1 +tomli==1.2.2 # via black tornado==6.1 # via @@ -377,7 +381,7 @@ tornado==6.1 # jupyter-client tqdm==4.62.3 # via papermill -traitlets==5.1.0 +traitlets==5.1.1 # via # ipykernel # ipython @@ -412,13 +416,15 @@ werkzeug==2.0.2 # via sagemaker-training wheel==0.37.0 # via flytekit -wrapt==1.13.2 +wrapt==1.13.3 # via # astroid # deprecated # flytekit zipp==3.6.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources zope.event==4.5.0 # via gevent zope.interface==5.4.0 diff --git a/requirements-spark2.txt b/requirements-spark2.txt index bcbfd29934..03eee890f5 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -18,13 +18,13 @@ backcall==0.2.0 # via ipython bcrypt==3.2.0 # via paramiko -black==21.9b0 +black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.0 +boto3==1.19.10 # via sagemaker-training -botocore==1.22.0 +botocore==1.22.11 # via # boto3 # s3transfer @@ -45,6 +45,8 @@ click==7.1.2 # flytekit # hmsclient # papermill +cloudpickle==2.0.0 + # via flytekit croniter==1.0.15 # via flytekit cryptography==35.0.0 @@ -72,13 +74,13 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.6 +flyteidl==0.21.8 # via flytekit gevent==21.8.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.41.0 +grpcio==1.41.1 # via flytekit hmsclient==0.1.1 # via flytekit @@ -90,7 +92,7 @@ inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.6 # via flytekit -ipython==7.28.0 +ipython==7.29.0 # via ipykernel ipython-genutils==0.2.0 # via @@ -116,7 +118,7 @@ jupyter-client==7.0.6 # via # ipykernel # nbclient -jupyter-core==4.8.1 +jupyter-core==4.9.1 # via # jupyter-client # nbconvert @@ -136,7 +138,7 @@ marshmallow==3.14.0 # marshmallow-jsonschema marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow-jsonschema==0.12.0 +marshmallow-jsonschema==0.13.0 # via flytekit matplotlib-inline==0.1.3 # via ipython @@ -146,7 +148,7 @@ mypy-extensions==0.4.3 # via # black # typing-inspect -natsort==7.1.1 +natsort==8.0.0 # via flytekit nbclient==0.5.4 # via @@ -170,7 +172,7 @@ numpy==1.21.3 # pyarrow # sagemaker-training # scipy -packaging==21.0 +packaging==21.2 # via bleach pandas==1.3.4 # via flytekit @@ -190,9 +192,9 @@ pickleshare==0.7.5 # via ipython platformdirs==2.4.0 # via black -prompt-toolkit==3.0.20 +prompt-toolkit==3.0.22 # via ipython -protobuf==3.19.0 +protobuf==3.19.1 # via # flyteidl # flytekit @@ -202,7 +204,7 @@ psutil==5.8.0 # via sagemaker-training ptyprocess==0.7.0 # via pexpect -py==1.10.0 +py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark @@ -244,7 +246,7 @@ pyyaml==5.4.1 # papermill pyzmq==22.3.0 # via jupyter-client -regex==2021.10.21 +regex==2021.11.2 # via # black # docker-image-py @@ -253,7 +255,7 @@ requests==2.26.0 # flytekit # papermill # responses -responses==0.14.0 +responses==0.15.0 # via flytekit retry==0.9.2 # via flytekit @@ -292,7 +294,7 @@ textwrap3==0.9.2 # via ansiwrap thrift==0.15.0 # via hmsclient -tomli==1.2.1 +tomli==1.2.2 # via black tornado==6.1 # via @@ -300,7 +302,7 @@ tornado==6.1 # jupyter-client tqdm==4.62.3 # via papermill -traitlets==5.1.0 +traitlets==5.1.1 # via # ipykernel # ipython @@ -330,7 +332,7 @@ werkzeug==2.0.2 # via sagemaker-training wheel==0.37.0 # via flytekit -wrapt==1.13.2 +wrapt==1.13.3 # via # deprecated # flytekit diff --git a/requirements.txt b/requirements.txt index 22ef535bb3..22c4b7cf60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,13 +20,13 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -black==21.9b0 +black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.5 +boto3==1.19.10 # via sagemaker-training -botocore==1.22.5 +botocore==1.22.11 # via # boto3 # s3transfer @@ -81,7 +81,7 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.7 +flyteidl==0.21.8 # via flytekit gevent==21.8.0 # via sagemaker-training @@ -99,7 +99,7 @@ inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.6 # via flytekit -ipython==7.28.0 +ipython==7.29.0 # via ipykernel ipython-genutils==0.2.0 # via @@ -130,7 +130,7 @@ jupyter-client==7.0.6 # via # ipykernel # nbclient -jupyter-core==4.9.0 +jupyter-core==4.9.1 # via # jupyter-client # nbconvert @@ -160,7 +160,7 @@ mypy-extensions==0.4.3 # via # black # typing-inspect -natsort==7.1.1 +natsort==8.0.0 # via flytekit nbclient==0.5.4 # via @@ -184,7 +184,7 @@ numpy==1.21.3 # pyarrow # sagemaker-training # scipy -packaging==21.0 +packaging==21.2 # via bleach pandas==1.3.4 # via flytekit @@ -206,9 +206,9 @@ platformdirs==2.4.0 # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.21 +prompt-toolkit==3.0.22 # via ipython -protobuf==3.19.0 +protobuf==3.19.1 # via # flyteidl # flytekit @@ -218,7 +218,7 @@ psutil==5.8.0 # via sagemaker-training ptyprocess==0.7.0 # via pexpect -py==1.10.0 +py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark @@ -233,7 +233,7 @@ pygments==2.10.0 # nbconvert pynacl==1.4.0 # via paramiko -pyparsing==3.0.3 +pyparsing==2.4.7 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -263,7 +263,7 @@ pyyaml==5.4.1 # papermill pyzmq==22.3.0 # via jupyter-client -regex==2021.10.23 +regex==2021.11.2 # via # black # docker-image-py @@ -353,7 +353,7 @@ werkzeug==2.0.2 # via sagemaker-training wheel==0.37.0 # via flytekit -wrapt==1.13.2 +wrapt==1.13.3 # via # deprecated # flytekit diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index bb746a6e9b..0c164b7c63 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -18,7 +18,7 @@ croniter==1.0.15 # via flytekit cryptography==35.0.0 # via secretstorage -cycler==0.10.0 +cycler==0.11.0 # via matplotlib dataclasses-json==0.5.6 # via flytekit @@ -34,11 +34,11 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.12 # via flytekit -flyteidl==0.21.6 +flyteidl==0.21.8 # via flytekit -flytekit==0.23.0 +flytekit==0.23.1 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -grpcio==1.41.0 +grpcio==1.41.1 # via flytekit idna==3.3 # via requests @@ -61,13 +61,13 @@ marshmallow==3.14.0 # marshmallow-jsonschema marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow-jsonschema==0.12.0 +marshmallow-jsonschema==0.13.0 # via flytekit matplotlib==3.4.3 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in mypy-extensions==0.4.3 # via typing-inspect -natsort==7.1.1 +natsort==8.0.0 # via flytekit numpy==1.21.3 # via @@ -83,17 +83,17 @@ pathspec==0.9.0 # via scantree pillow==8.4.0 # via matplotlib -protobuf==3.19.0 +protobuf==3.19.1 # via # flyteidl # flytekit -py==1.10.0 +py==1.11.0 # via retry pyarrow==3.0.0 # via flytekit pycparser==2.20 # via cffi -pyparsing==2.4.7 +pyparsing==3.0.4 # via matplotlib python-dateutil==2.8.1 # via @@ -109,13 +109,13 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.10.21 +regex==2021.11.2 # via docker-image-py requests==2.26.0 # via # flytekit # responses -responses==0.14.0 +responses==0.15.0 # via flytekit retry==0.9.2 # via flytekit @@ -125,7 +125,6 @@ secretstorage==3.3.1 # via keyring six==1.16.0 # via - # cycler # flytekit # grpcio # python-dateutil @@ -148,7 +147,7 @@ wheel==0.37.0 # via # -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in # flytekit -wrapt==1.13.2 +wrapt==1.13.3 # via # deprecated # flytekit From 94f22cfcb448cd1f1a214f24858d2183535d9e24 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 4 Nov 2021 17:32:13 -0700 Subject: [PATCH 07/20] Fix test_dataclass_transformer test Signed-off-by: Eduardo Apolinario --- tests/flytekit/unit/core/test_type_engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index b91750b859..c1a4339afe 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -434,10 +434,10 @@ def test_dataclass_transformer(): "InnerstructSchema": { "additionalProperties": False, "properties": { - "a": {"format": "integer", "title": "a", "type": "number"}, + "a": {"title": "a", "type": "integer"}, "b": {"default": None, "title": "b", "type": ["string", "null"]}, "c": { - "items": {"format": "integer", "title": "c", "type": "number"}, + "items": {"title": "c", "type": "integer"}, "title": "c", "type": "array", }, From b441cf6447b9da44a8053fa6f5fe295b448078bc Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 4 Nov 2021 23:28:20 -0700 Subject: [PATCH 08/20] Constrain mashmallow-jsonschema temporarily Signed-off-by: Eduardo Apolinario --- dev-requirements.txt | 4 ++-- doc-requirements.txt | 4 ++-- requirements-spark2.txt | 10 ++++++---- requirements.in | 3 +++ requirements.txt | 10 ++++++---- .../remote/mock_flyte_repo/workflows/requirements.txt | 2 +- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index a9092b933d..36f215ce4c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -154,7 +154,7 @@ marshmallow-enum==1.5.1 # via # -c requirements.txt # dataclasses-json -marshmallow-jsonschema==0.13.0 +marshmallow-jsonschema==0.12.0 # via # -c requirements.txt # flytekit @@ -173,7 +173,7 @@ natsort==8.0.0 # flytekit nodeenv==1.6.0 # via pre-commit -numpy==1.21.3 +numpy==1.21.4 # via # -c requirements.txt # pandas diff --git a/doc-requirements.txt b/doc-requirements.txt index edef3bc605..9a03ec89b7 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -29,7 +29,7 @@ black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.10 +boto3==1.19.11 # via sagemaker-training botocore==1.22.11 # via @@ -190,7 +190,7 @@ nest-asyncio==1.5.1 # via # jupyter-client # nbclient -numpy==1.21.3 +numpy==1.21.4 # via # flytekit # pandas diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 03eee890f5..72d71fb106 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -22,7 +22,7 @@ black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.10 +boto3==1.19.11 # via sagemaker-training botocore==1.22.11 # via @@ -138,8 +138,10 @@ marshmallow==3.14.0 # marshmallow-jsonschema marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit +marshmallow-jsonschema==0.12.0 + # via + # -c requirements.in + # flytekit matplotlib-inline==0.1.3 # via ipython mistune==0.8.4 @@ -165,7 +167,7 @@ nest-asyncio==1.5.1 # via # jupyter-client # nbclient -numpy==1.21.3 +numpy==1.21.4 # via # flytekit # pandas diff --git a/requirements.in b/requirements.in index 3cba59729d..4be22be5d7 100644 --- a/requirements.in +++ b/requirements.in @@ -2,6 +2,9 @@ -e file:.#egg=flytekit cookiecutter attrs<21 +# Restrict mashmallow-jsonschema while we figure out if 0.13.0 is backwards +# compatible. +marshmallow-jsonschema<0.13.0 # We need to restrict constrain the versions of both jsonschema and pyyaml because of docker-compose (which is # used to run integration tests) pins those 2 libraries. We are in the process of removing docker-compose in # favor of a more generic solution that involves flytectl to stand-up the sandbox, described in diff --git a/requirements.txt b/requirements.txt index 22c4b7cf60..6f192f9650 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.10 +boto3==1.19.11 # via sagemaker-training botocore==1.22.11 # via @@ -150,8 +150,10 @@ marshmallow==3.14.0 # marshmallow-jsonschema marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit +marshmallow-jsonschema==0.12.0 + # via + # -r requirements.in + # flytekit matplotlib-inline==0.1.3 # via ipython mistune==0.8.4 @@ -177,7 +179,7 @@ nest-asyncio==1.5.1 # via # jupyter-client # nbclient -numpy==1.21.3 +numpy==1.21.4 # via # flytekit # pandas diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 0c164b7c63..e48a900344 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -69,7 +69,7 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.0.0 # via flytekit -numpy==1.21.3 +numpy==1.21.4 # via # matplotlib # opencv-python From e1a6a4d73031e42fb9900ab38e21b4b7691346bf Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 4 Nov 2021 23:50:11 -0700 Subject: [PATCH 09/20] Add --template flag and project-name argument to command Signed-off-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/init.py | 40 ++++++-------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index a17da0a547..7fcca1fd6c 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -3,37 +3,18 @@ from cookiecutter.prompt import read_user_variable -@click.group("init") -def init(): +@click.command("init") +@click.option("--template", default="simple-example", help="cookiecutter template to be used") +@click.argument("project-name") +def init(template, project_name): """ Create flyte-ready projects. """ - # TODO: Add `interactive` boolean flag to allow for programmatic access to this subcommand. - pass - - -@click.command("example") -def generate_simple_example(): - """ - This command creates a directory containing the minimal code necessary to start a Flyte-ready project. - The generated directory structure for the default values resemble: - flyte_example - ├── Dockerfile - ├── flyte.config - ├── LICENSE - ├── myapp - │   ├── __init__.py - │   └── workflows - │   ├── example.py - │   └── __init__.py - ├── README.md - └── requirements.txt - """ - print("What is the name of your project? This is the name of the directory that will be created.") - project_name = read_user_variable("project_name", "flyte_example") - print("What should we call your application? This serves as the top level package where your workflows will live.") + click.echo( + "What should we call your application? This serves as the top level package where your workflows will live." + ) app = read_user_variable("app", "myapp") - print("What should be the name of your example workflow?") + click.echo("What should be the name of your example workflow?") workflow_name = read_user_variable("workflow", "workflow_example") config = { @@ -53,8 +34,5 @@ def generate_simple_example(): # as described in https://cookiecutter.readthedocs.io/en/1.7.2/advanced/directories.html. # The idea is to extend the number of templates, each in their own subdirectory, for example # a tensorflow-based example. - directory="simple-example", + directory=template, ) - - -init.add_command(generate_simple_example) From e5ad645200e7059e9db4e53e6afb4f902f6d86ca Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 4 Nov 2021 23:59:10 -0700 Subject: [PATCH 10/20] Revert changes to dataclass transformer tests Signed-off-by: Eduardo Apolinario --- tests/flytekit/unit/core/test_type_engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index c1a4339afe..b91750b859 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -434,10 +434,10 @@ def test_dataclass_transformer(): "InnerstructSchema": { "additionalProperties": False, "properties": { - "a": {"title": "a", "type": "integer"}, + "a": {"format": "integer", "title": "a", "type": "number"}, "b": {"default": None, "title": "b", "type": ["string", "null"]}, "c": { - "items": {"title": "c", "type": "integer"}, + "items": {"format": "integer", "title": "c", "type": "number"}, "title": "c", "type": "array", }, From bf7f22d0f3c1fe7077420574690798e4f67a903f Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 5 Nov 2021 00:02:42 -0700 Subject: [PATCH 11/20] Constrain marshmallow-jsonschema in dev-requirements Signed-off-by: Eduardo Apolinario --- dev-requirements.in | 3 +++ dev-requirements.txt | 1 + 2 files changed, 4 insertions(+) diff --git a/dev-requirements.in b/dev-requirements.in index 9743d0fc20..ebf952d6d5 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -8,3 +8,6 @@ pytest mypy pre-commit codespell +# Restrict mashmallow-jsonschema while we figure out if 0.13.0 is backwards +# compatible. +marshmallow-jsonschema<0.13.0 diff --git a/dev-requirements.txt b/dev-requirements.txt index 36f215ce4c..9e4a177986 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -157,6 +157,7 @@ marshmallow-enum==1.5.1 marshmallow-jsonschema==0.12.0 # via # -c requirements.txt + # -r dev-requirements.in # flytekit mock==4.0.3 # via -r dev-requirements.in From 25832e2646e5a4f95b68e911e7adb10c8425c86f Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 5 Nov 2021 00:03:50 -0700 Subject: [PATCH 12/20] Revert "Constrain marshmallow-jsonschema in dev-requirements" This reverts commit bf7f22d0f3c1fe7077420574690798e4f67a903f. --- dev-requirements.in | 3 --- dev-requirements.txt | 1 - 2 files changed, 4 deletions(-) diff --git a/dev-requirements.in b/dev-requirements.in index ebf952d6d5..9743d0fc20 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -8,6 +8,3 @@ pytest mypy pre-commit codespell -# Restrict mashmallow-jsonschema while we figure out if 0.13.0 is backwards -# compatible. -marshmallow-jsonschema<0.13.0 diff --git a/dev-requirements.txt b/dev-requirements.txt index 9e4a177986..36f215ce4c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -157,7 +157,6 @@ marshmallow-enum==1.5.1 marshmallow-jsonschema==0.12.0 # via # -c requirements.txt - # -r dev-requirements.in # flytekit mock==4.0.3 # via -r dev-requirements.in From 25aa23cfd23c7539cdb48cd44d46710a2760be87 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 5 Nov 2021 00:16:29 -0700 Subject: [PATCH 13/20] Fix typo in Makefile Signed-off-by: Eduardo Apolinario --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9e10e0df45..b1088aece7 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ lint: ## Run linters mypy flytekit/core || true mypy flytekit/types || true mypy tests/flytekit/unit/core || true - # Exclude setup.py to fix erorr: Duplicate module named "setup" + # Exclude setup.py to fix error: Duplicate module named "setup" mypy plugins --exclude setup.py || true pre-commit run --all-files From f547ed9fadce488d7eca7bbb8fdd0b876109c419 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 5 Nov 2021 00:16:41 -0700 Subject: [PATCH 14/20] Fix regeneration of spark2 requirements file Signed-off-by: Eduardo Apolinario --- requirements-spark2.in | 2 +- requirements-spark2.txt | 35 +++++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/requirements-spark2.in b/requirements-spark2.in index 3f3d0f85c9..1e47380935 100644 --- a/requirements-spark2.in +++ b/requirements-spark2.in @@ -1,2 +1,2 @@ .[all-spark2.4] --c requirements.in +-r requirements.in diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 72d71fb106..09f90ba4a8 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -6,18 +6,22 @@ # -e file:.#egg=flytekit # via - # -c requirements.in # -r requirements-spark2.in + # -r requirements.in ansiwrap==0.8.4 # via papermill +arrow==1.2.1 + # via jinja2-time attrs==20.3.0 # via - # -c requirements.in + # -r requirements.in # jsonschema backcall==0.2.0 # via ipython bcrypt==3.2.0 # via paramiko +binaryornot==0.4.4 + # via cookiecutter black==21.10b0 # via papermill bleach==4.1.0 @@ -35,6 +39,8 @@ cffi==1.15.0 # bcrypt # cryptography # pynacl +chardet==4.0.0 + # via binaryornot charset-normalizer==2.0.7 # via requests checksumdir==1.2.0 @@ -42,11 +48,14 @@ checksumdir==1.2.0 click==7.1.2 # via # black + # cookiecutter # flytekit # hmsclient # papermill cloudpickle==2.0.0 # via flytekit +cookiecutter==1.7.3 + # via -r requirements.in croniter==1.0.15 # via flytekit cryptography==35.0.0 @@ -105,14 +114,19 @@ jeepney==0.7.1 # keyring # secretstorage jinja2==3.0.2 - # via nbconvert + # via + # cookiecutter + # jinja2-time + # nbconvert +jinja2-time==0.2.0 + # via cookiecutter jmespath==0.10.0 # via # boto3 # botocore jsonschema==3.2.0 # via - # -c requirements.in + # -r requirements.in # nbformat jupyter-client==7.0.6 # via @@ -140,7 +154,7 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.12.0 # via - # -c requirements.in + # -r requirements.in # flytekit matplotlib-inline==0.1.3 # via ipython @@ -194,6 +208,8 @@ pickleshare==0.7.5 # via ipython platformdirs==2.4.0 # via black +poyo==0.5.0 + # via cookiecutter prompt-toolkit==3.0.22 # via ipython protobuf==3.19.1 @@ -229,6 +245,7 @@ pyspark==3.2.0 # via flytekit python-dateutil==2.8.1 # via + # arrow # botocore # croniter # flytekit @@ -236,6 +253,8 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit +python-slugify==5.0.2 + # via cookiecutter pytimeparse==1.1.8 # via flytekit pytz==2018.4 @@ -244,7 +263,7 @@ pytz==2018.4 # pandas pyyaml==5.4.1 # via - # -c requirements.in + # -r requirements.in # papermill pyzmq==22.3.0 # via jupyter-client @@ -254,6 +273,7 @@ regex==2021.11.2 # docker-image-py requests==2.26.0 # via + # cookiecutter # flytekit # papermill # responses @@ -275,6 +295,7 @@ six==1.16.0 # via # bcrypt # bleach + # cookiecutter # flytekit # grpcio # jsonschema @@ -292,6 +313,8 @@ tenacity==8.0.1 # via papermill testpath==0.5.0 # via nbconvert +text-unidecode==1.3 + # via python-slugify textwrap3==0.9.2 # via ansiwrap thrift==0.15.0 From 3f91dca7b68fef70c7073653bd2ba8ae62c7832b Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 5 Nov 2021 08:49:45 -0700 Subject: [PATCH 15/20] Use click for prompts Signed-off-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/init.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index 7fcca1fd6c..efe4de93b5 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -1,6 +1,5 @@ import click from cookiecutter.main import cookiecutter -from cookiecutter.prompt import read_user_variable @click.command("init") @@ -10,12 +9,11 @@ def init(template, project_name): """ Create flyte-ready projects. """ - click.echo( - "What should we call your application? This serves as the top level package where your workflows will live." + app = click.prompt( + "What should we call your application? This serves as the top level package where your workflows will live.", + default="myapp", ) - app = read_user_variable("app", "myapp") - click.echo("What should be the name of your example workflow?") - workflow_name = read_user_variable("workflow", "workflow_example") + workflow_name = click.prompt("What should be the name of your example workflow?", default="workflow_example") config = { "project_name": project_name, From b7250956e9fdfcdc29b64fd8195ad996cad75bfa Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 5 Nov 2021 11:34:42 -0700 Subject: [PATCH 16/20] Changes default workflow name to `my_wf` Signed-off-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index efe4de93b5..1fbcacbe1d 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -13,7 +13,7 @@ def init(template, project_name): "What should we call your application? This serves as the top level package where your workflows will live.", default="myapp", ) - workflow_name = click.prompt("What should be the name of your example workflow?", default="workflow_example") + workflow_name = click.prompt("What should be the name of your example workflow?", default="my_wf") config = { "project_name": project_name, From f492bb522d40f5ec1a59fd4fd633ee63932a8bcd Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 5 Nov 2021 11:45:48 -0700 Subject: [PATCH 17/20] Remove the two inputs from simple-example Signed-off-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/init.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index 1fbcacbe1d..374a93f369 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -9,16 +9,10 @@ def init(template, project_name): """ Create flyte-ready projects. """ - app = click.prompt( - "What should we call your application? This serves as the top level package where your workflows will live.", - default="myapp", - ) - workflow_name = click.prompt("What should be the name of your example workflow?", default="my_wf") - config = { "project_name": project_name, - "app": app, - "workflow": workflow_name, + "app": "myapp", + "workflow": "my_wf", } cookiecutter( "https://github.com/flyteorg/flytekit-python-template.git", @@ -34,3 +28,5 @@ def init(template, project_name): # a tensorflow-based example. directory=template, ) + + click.echo(f"Visit the {project_name} directory and follow the next steps in the Getting started guide.") From b0fd8b4736f55e628aff1d945259f1ec63346118 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 5 Nov 2021 14:52:55 -0700 Subject: [PATCH 18/20] Add more descriptive help for the template flag Signed-off-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/init.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index 374a93f369..613aa12442 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -3,7 +3,11 @@ @click.command("init") -@click.option("--template", default="simple-example", help="cookiecutter template to be used") +@click.option( + "--template", + default="simple-example", + help="cookiecutter template folder name to be used in the repo - https://github.com/flyteorg/flytekit-python-template.git", +) @click.argument("project-name") def init(template, project_name): """ From 3efecec2be99ef2fa6ed09e8116f9c2b251117f7 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Mon, 8 Nov 2021 14:31:41 -0800 Subject: [PATCH 19/20] Set cookiecutter in setup.py Signed-off-by: Eduardo Apolinario --- dev-requirements.txt | 38 ++++++++++++++++++++++++++++++++++++++ doc-requirements.txt | 26 +++++++++++++++++++++++--- requirements-spark2.txt | 6 +++--- requirements.in | 1 - requirements.txt | 6 +++--- setup.py | 1 + 6 files changed, 68 insertions(+), 10 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index f7f685964e..2409f24d1a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -8,6 +8,10 @@ # via # -c requirements.txt # pytest-flyte +arrow==1.2.1 + # via + # -c requirements.txt + # jinja2-time attrs==20.3.0 # via # -c requirements.txt @@ -20,6 +24,10 @@ bcrypt==3.2.0 # via # -c requirements.txt # paramiko +binaryornot==0.4.4 + # via + # -c requirements.txt + # cookiecutter certifi==2021.10.8 # via # -c requirements.txt @@ -32,6 +40,10 @@ cffi==1.15.0 # pynacl cfgv==3.3.1 # via pre-commit +chardet==4.0.0 + # via + # -c requirements.txt + # binaryornot charset-normalizer==2.0.7 # via # -c requirements.txt @@ -43,6 +55,7 @@ checksumdir==1.2.0 click==7.1.2 # via # -c requirements.txt + # cookiecutter # flytekit cloudpickle==2.0.0 # via @@ -50,6 +63,10 @@ cloudpickle==2.0.0 # flytekit codespell==2.1.0 # via -r dev-requirements.in +cookiecutter==1.7.3 + # via + # -c requirements.txt + # flytekit coverage[toml]==6.1.1 # via -r dev-requirements.in croniter==1.0.15 @@ -129,7 +146,13 @@ jeepney==0.7.1 jinja2==3.0.2 # via # -c requirements.txt + # cookiecutter + # jinja2-time # pytest-flyte +jinja2-time==0.2.0 + # via + # -c requirements.txt + # cookiecutter joblib==1.1.0 # via -r dev-requirements.in jsonschema==3.2.0 @@ -196,6 +219,10 @@ platformdirs==2.4.0 # virtualenv pluggy==1.0.0 # via pytest +poyo==0.5.0 + # via + # -c requirements.txt + # cookiecutter pre-commit==2.15.0 # via -r dev-requirements.in protobuf==3.19.1 @@ -240,6 +267,7 @@ pytest-flyte @ git+git://github.com/flyteorg/pytest-flyte@main python-dateutil==2.8.1 # via # -c requirements.txt + # arrow # croniter # flytekit # pandas @@ -249,6 +277,10 @@ python-json-logger==2.0.2 # via # -c requirements.txt # flytekit +python-slugify==5.0.2 + # via + # -c requirements.txt + # cookiecutter pytimeparse==1.1.8 # via # -c requirements.txt @@ -270,6 +302,7 @@ regex==2021.11.2 requests==2.26.0 # via # -c requirements.txt + # cookiecutter # docker # docker-compose # flytekit @@ -290,6 +323,7 @@ six==1.16.0 # via # -c requirements.txt # bcrypt + # cookiecutter # dockerpty # flytekit # grpcio @@ -307,6 +341,10 @@ statsd==3.3.0 # via # -c requirements.txt # flytekit +text-unidecode==1.3 + # via + # -c requirements.txt + # python-slugify texttable==1.6.4 # via docker-compose toml==0.10.2 diff --git a/doc-requirements.txt b/doc-requirements.txt index bda65093f5..0e94b20be9 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -10,6 +10,8 @@ alabaster==0.7.12 # via sphinx ansiwrap==0.8.4 # via papermill +arrow==1.2.1 + # via jinja2-time astroid==2.8.4 # via sphinx-autoapi attrs==21.2.0 @@ -25,13 +27,15 @@ beautifulsoup4==4.10.0 # furo # sphinx-code-include # sphinx-material +binaryornot==0.4.4 + # via cookiecutter black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.12 +boto3==1.20.0 # via sagemaker-training -botocore==1.22.12 +botocore==1.23.0 # via # boto3 # s3transfer @@ -42,6 +46,8 @@ cffi==1.15.0 # bcrypt # cryptography # pynacl +chardet==4.0.0 + # via binaryornot charset-normalizer==2.0.7 # via requests checksumdir==1.2.0 @@ -49,11 +55,14 @@ checksumdir==1.2.0 click==7.1.2 # via # black + # cookiecutter # flytekit # hmsclient # papermill cloudpickle==2.0.0 # via flytekit +cookiecutter==1.7.3 + # via flytekit croniter==1.0.15 # via flytekit cryptography==35.0.0 @@ -126,9 +135,13 @@ jeepney==0.7.1 # secretstorage jinja2==3.0.2 # via + # cookiecutter + # jinja2-time # nbconvert # sphinx # sphinx-autoapi +jinja2-time==0.2.0 + # via cookiecutter jmespath==0.10.0 # via # boto3 @@ -219,6 +232,8 @@ pickleshare==0.7.5 # via ipython platformdirs==2.4.0 # via black +poyo==0.5.0 + # via cookiecutter prompt-toolkit==3.0.22 # via ipython protobuf==3.19.1 @@ -256,6 +271,7 @@ pyspark==3.2.0 # via flytekit python-dateutil==2.8.1 # via + # arrow # botocore # croniter # flytekit @@ -264,7 +280,9 @@ python-dateutil==2.8.1 python-json-logger==2.0.2 # via flytekit python-slugify[unidecode]==5.0.2 - # via sphinx-material + # via + # cookiecutter + # sphinx-material pytimeparse==1.1.8 # via flytekit pytz==2018.4 @@ -284,6 +302,7 @@ regex==2021.11.2 # docker-image-py requests==2.26.0 # via + # cookiecutter # flytekit # papermill # responses @@ -306,6 +325,7 @@ six==1.16.0 # via # bcrypt # bleach + # cookiecutter # flytekit # grpcio # pynacl diff --git a/requirements-spark2.txt b/requirements-spark2.txt index ac3a85ff50..da84f0b8c7 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -26,9 +26,9 @@ black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.12 +boto3==1.20.0 # via sagemaker-training -botocore==1.22.12 +botocore==1.23.0 # via # boto3 # s3transfer @@ -55,7 +55,7 @@ click==7.1.2 cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 - # via -r requirements.in + # via flytekit croniter==1.0.15 # via flytekit cryptography==35.0.0 diff --git a/requirements.in b/requirements.in index 3cba59729d..408185fca8 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1,5 @@ .[all] -e file:.#egg=flytekit -cookiecutter attrs<21 # We need to restrict constrain the versions of both jsonschema and pyyaml because of docker-compose (which is # used to run integration tests) pins those 2 libraries. We are in the process of removing docker-compose in diff --git a/requirements.txt b/requirements.txt index ee0f8a7c86..1f1f07d4f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,9 +24,9 @@ black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.12 +boto3==1.20.0 # via sagemaker-training -botocore==1.22.12 +botocore==1.23.0 # via # boto3 # s3transfer @@ -53,7 +53,7 @@ click==7.1.2 cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 - # via -r requirements.in + # via flytekit croniter==1.0.15 # via flytekit cryptography==35.0.0 diff --git a/setup.py b/setup.py index 015565fd7c..a31089c48d 100644 --- a/setup.py +++ b/setup.py @@ -95,6 +95,7 @@ "diskcache>=5.2.1", "checksumdir>=1.2.0", "cloudpickle>=2.0.0", + "cookiecutter>=1.7.3", ], extras_require=extras_require, scripts=[ From bf60412f49d36d510cdbefdb62f26bcf9a4c75ac Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Mon, 8 Nov 2021 16:39:13 -0800 Subject: [PATCH 20/20] Default top level directory to flyte (as opposed to myapp) Signed-off-by: Eduardo Apolinario --- flytekit/clis/sdk_in_container/init.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index 613aa12442..079aab54c3 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -15,7 +15,7 @@ def init(template, project_name): """ config = { "project_name": project_name, - "app": "myapp", + "app": "flyte", "workflow": "my_wf", } cookiecutter( @@ -33,4 +33,6 @@ def init(template, project_name): directory=template, ) - click.echo(f"Visit the {project_name} directory and follow the next steps in the Getting started guide.") + click.echo( + f"Visit the {project_name} directory and follow the next steps in the Getting started guide (https://docs.flyte.org/en/latest/getting_started.html) to proceed." + )