Merge branch 'master' into feature/mpi-plugin

flyteorg · Oct 22, 2021 · 15ed8ae · 15ed8ae
2 parents 9a94e1d + f03ecb6
commit 15ed8ae
Show file tree

Hide file tree

Showing 43 changed files with 888 additions and 179 deletions.
diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
@@ -11,11 +11,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.7, 3.8, 3.9]
         spark-version-suffix: ["", "-spark2"]
         exclude:
         - python-version: 3.8
           spark-version-suffix: "-spark2"
+        - python-version: 3.9
+          spark-version-suffix: "-spark2"
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -31,7 +33,7 @@ jobs:
         key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install --upgrade pip==21.2.4 setuptools wheel
         make setup${{ matrix.spark-version-suffix }}
         pip freeze
     - name: Test with coverage
@@ -52,7 +54,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.8, 3.9]
         plugin-names:
           - flytekit-aws-athena
           - flytekit-aws-sagemaker
@@ -69,6 +71,7 @@ jobs:
           - flytekit-sqlalchemy
           - flytekit-pandera
           - flytekit-snowflake
+          - flytekit-modin
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -84,7 +87,7 @@ jobs:
         key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install --upgrade pip==21.2.4 setuptools wheel
         make setup
         cd plugins/${{ matrix.plugin-names }}
         # install in develop mode so setuptools doesn't override flytekit on current branch
@@ -112,7 +115,7 @@ jobs:
             ${{ runner.os }}-pip-
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade pip==21.2.4
           pip install -r dev-requirements.txt
       - name: Lint
         run: |
@@ -134,7 +137,7 @@ jobs:
           python-version: 3.8
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install --upgrade pip==21.2.4 setuptools wheel
           pip install -r doc-requirements.txt
       - name: Build the documentation
         run: make -C docs html
diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml
@@ -80,3 +80,17 @@ jobs:
           build_extra_args: "--compress=true --build-arg=VERSION=${{ steps.bump.outputs.version }} --build-arg=DOCKER_IMAGE=ghcr.io/flyteorg/flytekit:py38-${{ steps.bump.outputs.version }}"
           context: .
           dockerfile: Dockerfile.py38
+      - name: Build & Push Flytekit Python3.9 Docker Image to Github Registry
+        uses: whoan/docker-build-with-cache-action@v5
+        with:
+          # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package
+          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+          password: "${{ secrets.FLYTE_BOT_PAT }}"
+          image_name: ${{ github.repository_owner }}/flytekit
+          image_tag: py39-latest,py39-${{ github.sha }},py39-${{ steps.bump.outputs.version }}
+          push_git_tag: true
+          push_image_and_stages: true
+          registry: ghcr.io
+          build_extra_args: "--compress=true --build-arg=VERSION=${{ steps.bump.outputs.version }} --build-arg=DOCKER_IMAGE=ghcr.io/flyteorg/flytekit:py39-${{ steps.bump.outputs.version }}"
+          context: .
+          dockerfile: Dockerfile.py39
diff --git a/CODEOWNERS b/CODEOWNERS
@@ -1,3 +1,3 @@
 # These owners will be the default owners for everything in
 # the repo. Unless a later match takes precedence.
-*       @wild-endeavor @kumare3 @katrogan
+*       @wild-endeavor @kumare3 @eapolinario
diff --git a/Dockerfile.py39 b/Dockerfile.py39
@@ -0,0 +1,16 @@
+FROM python:3.9-slim-buster
+
+MAINTAINER Flyte Team <[email protected]>
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit
+
+RUN pip install awscli
+RUN pip install gsutil
+
+ARG VERSION
+ARG DOCKER_IMAGE
+
+RUN pip install -U flytekit==$VERSION
+
+WORKDIR /app
+
+ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE"
diff --git a/Makefile b/Makefile
@@ -12,7 +12,7 @@ help:
 
 .PHONY: install-piptools
 install-piptools:
-	pip install -U pip-tools
+	pip install -U pip-tools pip==21.2.4
 
 .PHONY: update_boilerplate
 update_boilerplate:

diff --git a/README.md b/README.md
@@ -24,6 +24,7 @@
 [![PyPI pyversions](https://img.shields.io/pypi/pyversions/flytekit.svg)](https://pypi.python.org/pypi/flytekit/)
 [![Docs](https://readthedocs.org/projects/flytekit/badge/?version=latest&style=plastic)](https://flytekit.rtfd.io)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
+[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://slack.flyte.org)
 
 Flytekit Python is the Python Library for easily authoring, testing, deploying, and interacting with Flyte tasks, workflows, and launch plans.
 
@@ -67,6 +68,53 @@ def my_workflow(x: int, y: int) -> int:
 ## 📖 How to Contribute to Flytekit
 You can find the detailed contribution guide [here](https://docs.flyte.org/projects/flytekit/en/latest/contributing.html). Plugins' contribution guide is included as well.
 
+## Code Guide
+The first version of the flytekit library was written circa 2017, before mypy typing was mainstream, and
+targeted Python 2. That legacy code will be fully deprecated and removed in 2022 but because there are still
+users of flytekit that rely on that legacy api, you'll see 2 separate and distinct code paths within this repo.
+Users and contributors should ignore the legacy sections. Below is a listing of the most important packages that
+comprise the new API:
+
+- `flytekit/core`
+  This holds all the core functionality of the new API.
+- `flytekit/types`
+  We bundle some special types like `FlyteFile, FlyteSchema etc` by default here.
+- `flytekit/extend`
+  This is the future home of extension points, and currently serves as the raw documentation for extensions.
+- `flytekit/extras`
+  This contains code that we want bundled with flytekit but not everyone may find useful (for example AWS and GCP
+  specific logic).
+- `flytekit/remote`
+  This implements the interface to interact with the Flyte service. Think of the code here as the Python-object version of Console.
+- `flytekit/testing`
+  is the future home for testing functionality like `mock` etc, and currently serves as documentation.
+  All test extensions should be imported from here.
+- `flytekit/models`
+  Protobuf generated Python code is not terribly user-friendly, so we improve upon those `flyteidl` classes here.
+- `plugins`
+  is the source of all plugins
+- `flytekit/bin/entrypoint.py`
+  The run time entrypoint for flytekit. When a task kicks off, this is where the click command goes.
+- `flytekit/clis`
+  This is the home for the clis.
+- `flytekit/configuration`
+  This holds all the configuration objects, but dependency on configuration should be carefully considered as it
+  makes compiled Flyte tasks and workflows less portable (i.e. if you run `pyflyte package` can someone else use
+  those serialized objects).
+
+Most of the other folders are for legacy Flytekit, support for which will be dropped in early 2022. For the most part,
+please ignore the following folders:
+
+- `flytekit/plugins`
+- `flytekit/common`
+  (the `translator.py` file is an exception)
+- `flytekit/engines`
+- `flytekit/interfaces`
+- `flytekit/sdk`
+- `flytekit/type_engines`
+
+Please also see the [overview section](https://docs.flyte.org/projects/flytekit/en/latest/design/index.html) of the formal flytekit documentation for more information.
+
 ## 🐞 File an Issue
 Refer to the [issues](https://docs.flyte.org/en/latest/community/contribute.html#issues) section in the contribution guide if you'd like to file an issue.
 

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -21,19 +21,19 @@ bcrypt==3.2.0
     # via
     #   -c requirements.txt
     #   paramiko
-certifi==2021.5.30
+certifi==2021.10.8
     # via
     #   -c requirements.txt
     #   requests
-cffi==1.14.6
+cffi==1.15.0
     # via
     #   -c requirements.txt
     #   bcrypt
     #   cryptography
     #   pynacl
 cfgv==3.3.1
     # via pre-commit
-charset-normalizer==2.0.6
+charset-normalizer==2.0.7
     # via
     #   -c requirements.txt
     #   requests
@@ -43,16 +43,17 @@ click==7.1.2
     #   flytekit
 codespell==2.1.0
     # via -r dev-requirements.in
-coverage[toml]==5.5
+coverage[toml]==6.0.2
     # via -r dev-requirements.in
 croniter==1.0.15
     # via
     #   -c requirements.txt
     #   flytekit
-cryptography==3.4.8
+cryptography==35.0.0
     # via
     #   -c requirements.txt
     #   paramiko
+    #   secretstorage
 dataclasses-json==0.5.6
     # via
     #   -c requirements.txt
@@ -77,7 +78,7 @@ distlib==0.3.3
     # via virtualenv
 distro==1.6.0
     # via docker-compose
-docker[ssh]==5.0.2
+docker[ssh]==5.0.3
     # via docker-compose
 docker-compose==1.29.2
     # via
@@ -91,23 +92,23 @@ dockerpty==0.4.1
     # via docker-compose
 docopt==0.6.2
     # via docker-compose
-docstring-parser==0.10
+docstring-parser==0.12
     # via
     #   -c requirements.txt
     #   flytekit
-filelock==3.1.0
+filelock==3.3.1
     # via virtualenv
-flyteidl==0.21.4
+flyteidl==0.21.6
     # via
     #   -c requirements.txt
     #   flytekit
 grpcio==1.41.0
     # via
     #   -c requirements.txt
     #   flytekit
-identify==2.2.15
+identify==2.3.0
     # via pre-commit
-idna==3.2
+idna==3.3
     # via
     #   -c requirements.txt
     #   requests
@@ -117,11 +118,16 @@ importlib-metadata==4.8.1
     #   keyring
 iniconfig==1.1.1
     # via pytest
-jinja2==3.0.1
+jeepney==0.7.1
+    # via
+    #   -c requirements.txt
+    #   keyring
+    #   secretstorage
+jinja2==3.0.2
     # via
     #   -c requirements.txt
     #   pytest-flyte
-joblib==1.0.1
+joblib==1.1.0
     # via -r dev-requirements.in
 jsonschema==3.2.0
     # via
@@ -135,7 +141,7 @@ markupsafe==2.0.1
     # via
     #   -c requirements.txt
     #   jinja2
-marshmallow==3.13.0
+marshmallow==3.14.0
     # via
     #   -c requirements.txt
     #   dataclasses-json
@@ -164,7 +170,7 @@ natsort==7.1.1
     #   flytekit
 nodeenv==1.6.0
     # via pre-commit
-numpy==1.21.2
+numpy==1.21.3
     # via
     #   -c requirements.txt
     #   pandas
@@ -173,11 +179,11 @@ packaging==21.0
     # via
     #   -c requirements.txt
     #   pytest
-pandas==1.3.3
+pandas==1.3.4
     # via
     #   -c requirements.txt
     #   flytekit
-paramiko==2.7.2
+paramiko==2.8.0
     # via
     #   -c requirements.txt
     #   docker
@@ -193,7 +199,7 @@ pluggy==1.0.0
     # via pytest
 pre-commit==2.15.0
     # via -r dev-requirements.in
-protobuf==3.18.0
+protobuf==3.19.0
     # via
     #   -c requirements.txt
     #   flyteidl
@@ -238,7 +244,7 @@ python-dateutil==2.8.1
     #   croniter
     #   flytekit
     #   pandas
-python-dotenv==0.19.0
+python-dotenv==0.19.1
     # via docker-compose
 python-json-logger==2.0.2
     # via
@@ -258,7 +264,7 @@ pyyaml==5.4.1
     #   -c requirements.txt
     #   docker-compose
     #   pre-commit
-regex==2021.9.24
+regex==2021.10.8
     # via
     #   -c requirements.txt
     #   docker-image-py
@@ -281,6 +287,10 @@ scantree==0.0.1
     # via
     #   -c requirements.txt
     #   dirhash
+secretstorage==3.3.1
+    # via
+    #   -c requirements.txt
+    #   keyring
 six==1.16.0
     # via
     #   -c requirements.txt
@@ -307,10 +317,13 @@ texttable==1.6.4
     # via docker-compose
 toml==0.10.2
     # via
-    #   coverage
     #   mypy
     #   pre-commit
     #   pytest
+tomli==1.2.1
+    # via
+    #   -c requirements.txt
+    #   coverage
 typing-extensions==3.10.0.2
     # via
     #   -c requirements.txt
@@ -336,7 +349,7 @@ wheel==0.37.0
     # via
     #   -c requirements.txt
     #   flytekit
-wrapt==1.12.1
+wrapt==1.13.2
     # via
     #   -c requirements.txt
     #   deprecated