[release v1.2] Backport v1.6

.github/workflows/pythonbuild.yml

@@ -65,6 +65,7 @@ jobs:
           - flytekit-deck-standard
           - flytekit-dolt
           - flytekit-duckdb
+          - flytekit-envd
           - flytekit-greatexpectations
           - flytekit-hive
           - flytekit-k8s-pod
@@ -156,19 +157,3 @@ jobs:
         uses: ludeeus/action-shellcheck@master
         with:
           ignore_paths: boilerplate
-
-  docs:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Fetch the code
-        uses: actions/checkout@v3
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip==21.2.4 setuptools wheel
-          pip install -r doc-requirements.txt
-      - name: Build the documentation
-        run: make -C docs html

.github/workflows/pythonpublish.yml

@@ -47,11 +47,28 @@ jobs:
         run: |
           make -C plugins build_all_plugins
           make -C plugins publish_all_plugins
-      # Added sleep because PYPI take some time in publish
-      - name: Sleep for 180 seconds
-        uses: jakejarvis/wait-action@master
-        with:
-          time: '180s'
+      - name: Sleep until pypi is available
+        id: pypiwait
+        run: |
+          # from refs/tags/v1.2.3 get 1.2.3 and make sure it's not an empty string
+          VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+          if [ -z "$VERSION" ]
+          then
+            echo "No tagged version found, exiting"
+            exit 1
+          fi
+          LINK="https://pypi.org/project/flytekit/${VERSION}"
+          for i in {1..60}; do
+            if curl -L -I -s -f ${LINK} >/dev/null; then
+              echo "Found pypi"
+              exit 0
+            else
+              echo "Did not find - Retrying in 10 seconds..."
+              sleep 10
+            fi
+          done
+          exit 1
+        shell: bash
     outputs:
       version: ${{ steps.bump.outputs.version }}
 
@@ -120,6 +137,91 @@ jobs:
           tags: ${{ steps.sqlalchemy-names.outputs.tags }}
           build-args: |
             VERSION=${{ needs.deploy.outputs.version }}
-          file: ./plugins/flytekit-sqlalchemy/Dockerfile.py${{ matrix.python-version }}
+            PYTHON_VERSION=${{ matrix.python-version }}
+          file: ./plugins/flytekit-sqlalchemy/Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  build-and-push-external-plugin-service-images:
+    runs-on: ubuntu-latest
+    needs: deploy
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to GitHub Container Registry
+        if: ${{ github.event_name == 'release' }}
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+          password: "${{ secrets.FLYTE_BOT_PAT }}"
+      - name: Prepare External Plugin Service Image Names
+        id: external-plugin-service-names
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/${{ github.repository_owner }}/external-plugin-service
+          tags: |
+            latest
+            ${{ github.sha }}
+            ${{ needs.deploy.outputs.version }}
+      - name: Push External Plugin Service Image to GitHub Registry
+        uses: docker/build-push-action@v2
+        with:
+          context: "."
+          platforms: linux/arm64, linux/amd64
+          push: ${{ github.event_name == 'release' }}
+          tags: ${{ steps.external-plugin-service-names.outputs.tags }}
+          build-args: |
+            VERSION=${{ needs.deploy.outputs.version }}
+          file: ./Dockerfile.external-plugin-service
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  build-and-push-spark-images:
+    runs-on: ubuntu-latest
+    needs: deploy
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to GitHub Container Registry
+        if: ${{ github.event_name == 'release' }}
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+          password: "${{ secrets.FLYTE_BOT_PAT }}"
+      - name: Prepare Spark Image Names
+        id: spark-names
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/${{ github.repository_owner }}/flytekit
+          tags: |
+            spark-latest
+            spark-${{ github.sha }}
+            spark-${{ needs.deploy.outputs.version }}
+      - name: Push Spark Image to GitHub Registry
+        uses: docker/build-push-action@v2
+        with:
+          context: "./plugins/flytekit-spark/"
+          platforms: linux/arm64, linux/amd64
+          push: ${{ github.event_name == 'release' }}
+          tags: ${{ steps.spark-names.outputs.tags }}
+          build-args: |
+            VERSION=${{ needs.deploy.outputs.version }}
+          file: ./plugins/flytekit-spark/Dockerfile
           cache-from: type=gha
           cache-to: type=gha,mode=max

.gitignore

@@ -32,3 +32,4 @@ htmlcov
 *.ipynb
 *dat
 docs/source/_tags/
+.hypothesis

.pre-commit-config.yaml

@@ -22,3 +22,7 @@ repos:
   rev: v0.8.0.4
   hooks:
   - id: shellcheck
+- repo: https://github.com/conorfalvey/check_pdb_hook
+  rev: 0.0.9
+  hooks:
+    - id: check_pdb_hook

.readthedocs.yml

@@ -9,6 +9,8 @@ build:
   os: ubuntu-20.04
   tools:
     python: "3.9"
+  apt_packages:
+    - graphviz
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

Dockerfile

@@ -16,8 +16,7 @@ RUN apt-get update && apt-get install build-essential -y
 RUN pip install -U flytekit==$VERSION \
 	flytekitplugins-pod==$VERSION \
 	flytekitplugins-deck-standard==$VERSION \
-	flytekitplugins-data-fsspec[aws]==$VERSION \
-	flytekitplugins-data-fsspec[gcp]==$VERSION \
+	flytekitplugins-envd==$VERSION \
 	scikit-learn
 
 RUN useradd -u 1000 flytekit

Dockerfile.dev

@@ -0,0 +1,32 @@
+# This Dockerfile is here to help with end-to-end testing
+# From flytekit
+# $ docker build -f Dockerfile.dev --build-arg PYTHON_VERSION=3.10 -t localhost:30000/flytekittest:someversion .
+# $ docker push localhost:30000/flytekittest:someversion
+# From your test user code
+# $ pyflyte run --image localhost:30000/flytekittest:someversion
+
+ARG PYTHON_VERSION
+FROM python:${PYTHON_VERSION}-slim-buster
+
+MAINTAINER Flyte Team <[email protected]>
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit
+
+WORKDIR /root
+
+ARG VERSION
+
+RUN apt-get update && apt-get install build-essential vim -y
+
+COPY . /flytekit
+
+# Pod tasks should be exposed in the default image
+RUN pip install -e /flytekit
+RUN pip install -e /flytekit/plugins/flytekit-k8s-pod
+RUN pip install -e /flytekit/plugins/flytekit-deck-standard
+RUN pip install scikit-learn
+
+ENV PYTHONPATH "/flytekit:/flytekit/plugins/flytekit-k8s-pod:/flytekit/plugins/flytekit-deck-standard:"
+
+RUN useradd -u 1000 flytekit
+RUN chown flytekit: /root
+USER flytekit

Dockerfile.external-plugin-service

@@ -0,0 +1,9 @@
+FROM python:3.9-slim-buster
+
+MAINTAINER Flyte Team <[email protected]>
+LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit
+
+ARG VERSION
+RUN pip install -U flytekit==$VERSION flytekitplugins-bigquery==$VERSION
+
+CMD pyflyte serve --port 8000

Makefile

@@ -26,6 +26,7 @@ setup: install-piptools ## Install requirements
 
 .PHONY: fmt
 fmt: ## Format code with black and isort
+	autoflake --remove-all-unused-imports --ignore-init-module-imports --ignore-pass-after-docstring --in-place -r flytekit plugins tests
 	pre-commit run black --all-files || true
 	pre-commit run isort --all-files || true
 

dev-requirements.in

@@ -2,6 +2,7 @@
 
 git+https://github.com/flyteorg/pytest-flyte@main#egg=pytest-flyte
 coverage[toml]
+hypothesis
 joblib
 mock
 pytest
@@ -21,3 +22,7 @@ tensorflow==2.8.1; platform_machine!='arm64' or platform_system!='Darwin'
 # we put this constraint while we do not have per-environment requirements files
 torch<=1.12.1
 scikit-learn
+types-protobuf<4
+types-croniter
+types-mock
+autoflake

doc-requirements.in

@@ -1,6 +1,7 @@
 .
 -e file:.#egg=flytekit
 
+grpcio<=1.49.1
 git+https://github.com/flyteorg/furo@main
 sphinx
 sphinx-gallery
@@ -11,7 +12,7 @@ sphinx-autoapi
 sphinx-copybutton
 sphinx_fontawesome
 sphinx-panels
-sphinxcontrib-yt
+sphinxcontrib-youtube
 cryptography
 google-api-core[grpc]
 scikit-learn