diff --git a/Makefile b/Makefile index 9e10e0df45..b1088aece7 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ lint: ## Run linters mypy flytekit/core || true mypy flytekit/types || true mypy tests/flytekit/unit/core || true - # Exclude setup.py to fix erorr: Duplicate module named "setup" + # Exclude setup.py to fix error: Duplicate module named "setup" mypy plugins --exclude setup.py || true pre-commit run --all-files diff --git a/dev-requirements.txt b/dev-requirements.txt index 4ced19ea15..2409f24d1a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -8,6 +8,10 @@ # via # -c requirements.txt # pytest-flyte +arrow==1.2.1 + # via + # -c requirements.txt + # jinja2-time attrs==20.3.0 # via # -c requirements.txt @@ -20,6 +24,10 @@ bcrypt==3.2.0 # via # -c requirements.txt # paramiko +binaryornot==0.4.4 + # via + # -c requirements.txt + # cookiecutter certifi==2021.10.8 # via # -c requirements.txt @@ -32,6 +40,10 @@ cffi==1.15.0 # pynacl cfgv==3.3.1 # via pre-commit +chardet==4.0.0 + # via + # -c requirements.txt + # binaryornot charset-normalizer==2.0.7 # via # -c requirements.txt @@ -43,6 +55,7 @@ checksumdir==1.2.0 click==7.1.2 # via # -c requirements.txt + # cookiecutter # flytekit cloudpickle==2.0.0 # via @@ -50,6 +63,10 @@ cloudpickle==2.0.0 # flytekit codespell==2.1.0 # via -r dev-requirements.in +cookiecutter==1.7.3 + # via + # -c requirements.txt + # flytekit coverage[toml]==6.1.1 # via -r dev-requirements.in croniter==1.0.15 @@ -60,6 +77,7 @@ cryptography==35.0.0 # via # -c requirements.txt # paramiko + # secretstorage dataclasses-json==0.5.6 # via # -c requirements.txt @@ -108,7 +126,7 @@ grpcio==1.41.1 # via # -c requirements.txt # flytekit -identify==2.3.3 +identify==2.3.4 # via pre-commit idna==3.3 # via @@ -120,10 +138,21 @@ importlib-metadata==4.8.1 # keyring iniconfig==1.1.1 # via pytest +jeepney==0.7.1 + # via + # -c requirements.txt + # keyring + # secretstorage jinja2==3.0.2 # via # -c requirements.txt + # cookiecutter + # jinja2-time # pytest-flyte +jinja2-time==0.2.0 + # via + # -c requirements.txt + # cookiecutter joblib==1.1.0 # via -r dev-requirements.in jsonschema==3.2.0 @@ -190,6 +219,10 @@ platformdirs==2.4.0 # virtualenv pluggy==1.0.0 # via pytest +poyo==0.5.0 + # via + # -c requirements.txt + # cookiecutter pre-commit==2.15.0 # via -r dev-requirements.in protobuf==3.19.1 @@ -206,7 +239,7 @@ pyarrow==3.0.0 # via # -c requirements.txt # flytekit -pycparser==2.20 +pycparser==2.21 # via # -c requirements.txt # cffi @@ -234,6 +267,7 @@ pytest-flyte @ git+git://github.com/flyteorg/pytest-flyte@main python-dateutil==2.8.1 # via # -c requirements.txt + # arrow # croniter # flytekit # pandas @@ -243,6 +277,10 @@ python-json-logger==2.0.2 # via # -c requirements.txt # flytekit +python-slugify==5.0.2 + # via + # -c requirements.txt + # cookiecutter pytimeparse==1.1.8 # via # -c requirements.txt @@ -264,6 +302,7 @@ regex==2021.11.2 requests==2.26.0 # via # -c requirements.txt + # cookiecutter # docker # docker-compose # flytekit @@ -276,10 +315,15 @@ retry==0.9.2 # via # -c requirements.txt # flytekit +secretstorage==3.3.1 + # via + # -c requirements.txt + # keyring six==1.16.0 # via # -c requirements.txt # bcrypt + # cookiecutter # dockerpty # flytekit # grpcio @@ -297,6 +341,10 @@ statsd==3.3.0 # via # -c requirements.txt # flytekit +text-unidecode==1.3 + # via + # -c requirements.txt + # python-slugify texttable==1.6.4 # via docker-compose toml==0.10.2 diff --git a/doc-requirements.txt b/doc-requirements.txt index 9dbf3687f6..0e94b20be9 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -10,10 +10,8 @@ alabaster==0.7.12 # via sphinx ansiwrap==0.8.4 # via papermill -appnope==0.1.2 - # via - # ipykernel - # ipython +arrow==1.2.1 + # via jinja2-time astroid==2.8.4 # via sphinx-autoapi attrs==21.2.0 @@ -29,13 +27,15 @@ beautifulsoup4==4.10.0 # furo # sphinx-code-include # sphinx-material +binaryornot==0.4.4 + # via cookiecutter black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.12 +boto3==1.20.0 # via sagemaker-training -botocore==1.22.12 +botocore==1.23.0 # via # boto3 # s3transfer @@ -46,6 +46,8 @@ cffi==1.15.0 # bcrypt # cryptography # pynacl +chardet==4.0.0 + # via binaryornot charset-normalizer==2.0.7 # via requests checksumdir==1.2.0 @@ -53,17 +55,21 @@ checksumdir==1.2.0 click==7.1.2 # via # black + # cookiecutter # flytekit # hmsclient # papermill cloudpickle==2.0.0 # via flytekit +cookiecutter==1.7.3 + # via flytekit croniter==1.0.15 # via flytekit cryptography==35.0.0 # via # -r doc-requirements.in # paramiko + # secretstorage css-html-js-minify==2.5.5 # via sphinx-material dataclasses-json==0.5.6 @@ -123,11 +129,19 @@ ipython-genutils==0.2.0 # nbformat jedi==0.18.0 # via ipython +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.2 # via + # cookiecutter + # jinja2-time # nbconvert # sphinx # sphinx-autoapi +jinja2-time==0.2.0 + # via cookiecutter jmespath==0.10.0 # via # boto3 @@ -174,7 +188,7 @@ mypy-extensions==0.4.3 # typing-inspect natsort==8.0.0 # via flytekit -nbclient==0.5.4 +nbclient==0.5.5 # via # nbconvert # papermill @@ -218,6 +232,8 @@ pickleshare==0.7.5 # via ipython platformdirs==2.4.0 # via black +poyo==0.5.0 + # via cookiecutter prompt-toolkit==3.0.22 # via ipython protobuf==3.19.1 @@ -236,7 +252,7 @@ py4j==0.10.9.2 # via pyspark pyarrow==3.0.0 # via flytekit -pycparser==2.20 +pycparser==2.21 # via cffi pygments==2.10.0 # via @@ -255,6 +271,7 @@ pyspark==3.2.0 # via flytekit python-dateutil==2.8.1 # via + # arrow # botocore # croniter # flytekit @@ -263,7 +280,9 @@ python-dateutil==2.8.1 python-json-logger==2.0.2 # via flytekit python-slugify[unidecode]==5.0.2 - # via sphinx-material + # via + # cookiecutter + # sphinx-material pytimeparse==1.1.8 # via flytekit pytz==2018.4 @@ -283,6 +302,7 @@ regex==2021.11.2 # docker-image-py requests==2.26.0 # via + # cookiecutter # flytekit # papermill # responses @@ -297,12 +317,15 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scipy==1.7.1 +scipy==1.7.2 # via sagemaker-training +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # bcrypt # bleach + # cookiecutter # flytekit # grpcio # pynacl diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py new file mode 100644 index 0000000000..079aab54c3 --- /dev/null +++ b/flytekit/clis/sdk_in_container/init.py @@ -0,0 +1,38 @@ +import click +from cookiecutter.main import cookiecutter + + +@click.command("init") +@click.option( + "--template", + default="simple-example", + help="cookiecutter template folder name to be used in the repo - https://github.com/flyteorg/flytekit-python-template.git", +) +@click.argument("project-name") +def init(template, project_name): + """ + Create flyte-ready projects. + """ + config = { + "project_name": project_name, + "app": "flyte", + "workflow": "my_wf", + } + cookiecutter( + "https://github.com/flyteorg/flytekit-python-template.git", + # TODO: remove this once we make the transition to cookie-cutter official. + checkout="cookie-cutter", + no_input=True, + # We do not want to clobber existing files/directories. + overwrite_if_exists=False, + extra_context=config, + # By specifying directory we can have multiple templates in the same repository, + # as described in https://cookiecutter.readthedocs.io/en/1.7.2/advanced/directories.html. + # The idea is to extend the number of templates, each in their own subdirectory, for example + # a tensorflow-based example. + directory=template, + ) + + click.echo( + f"Visit the {project_name} directory and follow the next steps in the Getting started guide (https://docs.flyte.org/en/latest/getting_started.html) to proceed." + ) diff --git a/flytekit/clis/sdk_in_container/pyflyte.py b/flytekit/clis/sdk_in_container/pyflyte.py index 676b1bc675..b23fa5d121 100644 --- a/flytekit/clis/sdk_in_container/pyflyte.py +++ b/flytekit/clis/sdk_in_container/pyflyte.py @@ -6,6 +6,7 @@ from flytekit.clis.sdk_in_container.constants import CTX_PACKAGES from flytekit.clis.sdk_in_container.fast_register import fast_register +from flytekit.clis.sdk_in_container.init import init from flytekit.clis.sdk_in_container.launch_plan import launch_plans from flytekit.clis.sdk_in_container.local_cache import local_cache from flytekit.clis.sdk_in_container.package import package @@ -112,6 +113,7 @@ def update_configuration_file(config_file_path): main.add_command(launch_plans) main.add_command(package) main.add_command(local_cache) +main.add_command(init) if __name__ == "__main__": main() diff --git a/requirements-spark2.in b/requirements-spark2.in index 3f3d0f85c9..1e47380935 100644 --- a/requirements-spark2.in +++ b/requirements-spark2.in @@ -1,2 +1,2 @@ .[all-spark2.4] --c requirements.in +-r requirements.in diff --git a/requirements-spark2.txt b/requirements-spark2.txt index db493d36ef..da84f0b8c7 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -6,29 +6,29 @@ # -e file:.#egg=flytekit # via - # -c requirements.in # -r requirements-spark2.in + # -r requirements.in ansiwrap==0.8.4 # via papermill -appnope==0.1.2 - # via - # ipykernel - # ipython +arrow==1.2.1 + # via jinja2-time attrs==20.3.0 # via - # -c requirements.in + # -r requirements.in # jsonschema backcall==0.2.0 # via ipython bcrypt==3.2.0 # via paramiko +binaryornot==0.4.4 + # via cookiecutter black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.12 +boto3==1.20.0 # via sagemaker-training -botocore==1.22.12 +botocore==1.23.0 # via # boto3 # s3transfer @@ -39,6 +39,8 @@ cffi==1.15.0 # bcrypt # cryptography # pynacl +chardet==4.0.0 + # via binaryornot charset-normalizer==2.0.7 # via requests checksumdir==1.2.0 @@ -46,15 +48,20 @@ checksumdir==1.2.0 click==7.1.2 # via # black + # cookiecutter # flytekit # hmsclient # papermill cloudpickle==2.0.0 # via flytekit +cookiecutter==1.7.3 + # via flytekit croniter==1.0.15 # via flytekit cryptography==35.0.0 - # via paramiko + # via + # paramiko + # secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 @@ -102,15 +109,24 @@ ipython-genutils==0.2.0 # nbformat jedi==0.18.0 # via ipython +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.2 - # via nbconvert + # via + # cookiecutter + # jinja2-time + # nbconvert +jinja2-time==0.2.0 + # via cookiecutter jmespath==0.10.0 # via # boto3 # botocore jsonschema==3.2.0 # via - # -c requirements.in + # -r requirements.in # nbformat jupyter-client==7.0.6 # via @@ -148,7 +164,7 @@ mypy-extensions==0.4.3 # typing-inspect natsort==8.0.0 # via flytekit -nbclient==0.5.4 +nbclient==0.5.5 # via # nbconvert # papermill @@ -190,6 +206,8 @@ pickleshare==0.7.5 # via ipython platformdirs==2.4.0 # via black +poyo==0.5.0 + # via cookiecutter prompt-toolkit==3.0.22 # via ipython protobuf==3.19.1 @@ -208,7 +226,7 @@ py4j==0.10.9.2 # via pyspark pyarrow==3.0.0 # via flytekit -pycparser==2.20 +pycparser==2.21 # via cffi pygments==2.10.0 # via @@ -225,6 +243,7 @@ pyspark==3.2.0 # via flytekit python-dateutil==2.8.1 # via + # arrow # botocore # croniter # flytekit @@ -232,6 +251,8 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit +python-slugify==5.0.2 + # via cookiecutter pytimeparse==1.1.8 # via flytekit pytz==2018.4 @@ -240,7 +261,7 @@ pytz==2018.4 # pandas pyyaml==5.4.1 # via - # -c requirements.in + # -r requirements.in # papermill pyzmq==22.3.0 # via jupyter-client @@ -250,6 +271,7 @@ regex==2021.11.2 # docker-image-py requests==2.26.0 # via + # cookiecutter # flytekit # papermill # responses @@ -263,12 +285,15 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scipy==1.7.1 +scipy==1.7.2 # via sagemaker-training +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # bcrypt # bleach + # cookiecutter # flytekit # grpcio # jsonschema @@ -286,6 +311,8 @@ tenacity==8.0.1 # via papermill testpath==0.5.0 # via nbconvert +text-unidecode==1.3 + # via python-slugify textwrap3==0.9.2 # via ansiwrap thrift==0.15.0 diff --git a/requirements.txt b/requirements.txt index 93e94bc0b4..1f1f07d4f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,10 +8,8 @@ # via -r requirements.in ansiwrap==0.8.4 # via papermill -appnope==0.1.2 - # via - # ipykernel - # ipython +arrow==1.2.1 + # via jinja2-time attrs==20.3.0 # via # -r requirements.in @@ -20,13 +18,15 @@ backcall==0.2.0 # via ipython bcrypt==3.2.0 # via paramiko +binaryornot==0.4.4 + # via cookiecutter black==21.10b0 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.19.12 +boto3==1.20.0 # via sagemaker-training -botocore==1.22.12 +botocore==1.23.0 # via # boto3 # s3transfer @@ -37,6 +37,8 @@ cffi==1.15.0 # bcrypt # cryptography # pynacl +chardet==4.0.0 + # via binaryornot charset-normalizer==2.0.7 # via requests checksumdir==1.2.0 @@ -44,15 +46,20 @@ checksumdir==1.2.0 click==7.1.2 # via # black + # cookiecutter # flytekit # hmsclient # papermill cloudpickle==2.0.0 # via flytekit +cookiecutter==1.7.3 + # via flytekit croniter==1.0.15 # via flytekit cryptography==35.0.0 - # via paramiko + # via + # paramiko + # secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 @@ -100,8 +107,17 @@ ipython-genutils==0.2.0 # nbformat jedi==0.18.0 # via ipython +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.2 - # via nbconvert + # via + # cookiecutter + # jinja2-time + # nbconvert +jinja2-time==0.2.0 + # via cookiecutter jmespath==0.10.0 # via # boto3 @@ -146,7 +162,7 @@ mypy-extensions==0.4.3 # typing-inspect natsort==8.0.0 # via flytekit -nbclient==0.5.4 +nbclient==0.5.5 # via # nbconvert # papermill @@ -188,6 +204,8 @@ pickleshare==0.7.5 # via ipython platformdirs==2.4.0 # via black +poyo==0.5.0 + # via cookiecutter prompt-toolkit==3.0.22 # via ipython protobuf==3.19.1 @@ -206,7 +224,7 @@ py4j==0.10.9.2 # via pyspark pyarrow==3.0.0 # via flytekit -pycparser==2.20 +pycparser==2.21 # via cffi pygments==2.10.0 # via @@ -223,6 +241,7 @@ pyspark==3.2.0 # via flytekit python-dateutil==2.8.1 # via + # arrow # botocore # croniter # flytekit @@ -230,6 +249,8 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit +python-slugify==5.0.2 + # via cookiecutter pytimeparse==1.1.8 # via flytekit pytz==2018.4 @@ -248,6 +269,7 @@ regex==2021.11.2 # docker-image-py requests==2.26.0 # via + # cookiecutter # flytekit # papermill # responses @@ -261,12 +283,15 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scipy==1.7.1 +scipy==1.7.2 # via sagemaker-training +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # bcrypt # bleach + # cookiecutter # flytekit # grpcio # jsonschema @@ -284,6 +309,8 @@ tenacity==8.0.1 # via papermill testpath==0.5.0 # via nbconvert +text-unidecode==1.3 + # via python-slugify textwrap3==0.9.2 # via ansiwrap thrift==0.15.0 diff --git a/setup.py b/setup.py index 015565fd7c..a31089c48d 100644 --- a/setup.py +++ b/setup.py @@ -95,6 +95,7 @@ "diskcache>=5.2.1", "checksumdir>=1.2.0", "cloudpickle>=2.0.0", + "cookiecutter>=1.7.3", ], extras_require=extras_require, scripts=[ diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 5f2124fbb1..e7ce990324 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -8,12 +8,16 @@ attrs==21.2.0 # via scantree certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography charset-normalizer==2.0.7 # via requests click==7.1.2 # via flytekit croniter==1.0.15 # via flytekit +cryptography==35.0.0 + # via secretstorage cycler==0.11.0 # via matplotlib dataclasses-json==0.5.6 @@ -40,6 +44,10 @@ idna==3.3 # via requests importlib-metadata==4.8.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage joblib==1.1.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in keyring==23.2.1 @@ -83,7 +91,9 @@ py==1.11.0 # via retry pyarrow==3.0.0 # via flytekit -pyparsing==3.0.4 +pycparser==2.21 + # via cffi +pyparsing==3.0.5 # via matplotlib python-dateutil==2.8.1 # via @@ -111,6 +121,8 @@ retry==0.9.2 # via flytekit scantree==0.0.1 # via dirhash +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # flytekit