last error

.github/workflows/pythonbuild.yml

@@ -7,7 +7,7 @@ on:
   pull_request:
 
 env:
-  FLYTE_SDK_LOGGING_LEVEL: 10  # debug
+  FLYTE_SDK_LOGGING_LEVEL: 10 # debug
 
 jobs:
   build:
@@ -49,11 +49,11 @@ jobs:
           pip freeze
       - name: Test with coverage
         run: |
-          pytest tests/flytekit/unit -m "not sandbox_test" --cov=./ --cov-report=xml
+          make unit_test_codecov
       - name: Codecov
         uses: codecov/[email protected]
         with:
-          fail_ci_if_error: true # optional (default = false)
+          fail_ci_if_error: false
 
   build-plugins:
     runs-on: ubuntu-latest
@@ -68,6 +68,7 @@ jobs:
           - flytekit-aws-sagemaker
           - flytekit-bigquery
           - flytekit-data-fsspec
+          - flytekit-dbt
           - flytekit-deck-standard
           - flytekit-dolt
           - flytekit-greatexpectations
@@ -87,6 +88,7 @@ jobs:
           - flytekit-snowflake
           - flytekit-spark
           - flytekit-sqlalchemy
+          - flytekit-vaex
           - flytekit-whylogs
         exclude:
           # flytekit-modin depends on ray which does not have a 3.10 wheel yet.
@@ -109,8 +111,6 @@ jobs:
           # Issue tracked: https://github.com/whylabs/whylogs/issues/697
           - python-version: 3.10
             plugin-names: "flytekit-whylogs"
-
-
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -136,7 +136,6 @@ jobs:
         run: |
           cd plugins/${{ matrix.plugin-names }}
           coverage run -m pytest tests
-
   lint:
     runs-on: ubuntu-latest
     steps:
@@ -178,4 +177,6 @@ jobs:
           python -m pip install --upgrade pip==21.2.4 setuptools wheel
           pip install -r doc-requirements.txt
       - name: Build the documentation
-        run: make -C docs html
+        run: |
+          # TODO: Remove after buf migration is done and packages updated
+          PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python make -C docs html

.github/workflows/pythonpublish.yml

@@ -65,31 +65,61 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: "0"
-      - name: Build & Push Flytekit Python${{ matrix.python-version }} Docker Image to Github Registry
-        uses: whoan/docker-build-with-cache-action@v5
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to GitHub Container Registry
+        if: ${{ github.event_name == 'release' }}
+        uses: docker/login-action@v1
         with:
-          # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package
+          registry: ghcr.io
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
-          image_name: ${{ github.repository_owner }}/flytekit
-          image_tag: py${{ matrix.python-version }}-latest,py${{ matrix.python-version }}-${{ github.sha }},py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }}
-          push_git_tag: true
-          push_image_and_stages: true
-          registry: ghcr.io
-          build_extra_args: "--compress=true --build-arg=VERSION=${{ needs.deploy.outputs.version }} --build-arg=DOCKER_IMAGE=ghcr.io/flyteorg/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }}"
+      - name: Prepare Flytekit Image Names
+        id: flytekit-names
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/${{ github.repository_owner }}/flytekit
+          tags: |
+            py${{ matrix.python-version }}-latest
+            py${{ matrix.python-version }}-${{ github.sha }}
+            py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }}
+      - name: Build & Push Flytekit Python${{ matrix.python-version }} Docker Image to Github Registry
+        uses: docker/build-push-action@v2
+        with:
           context: .
-          dockerfile: Dockerfile.py${{ matrix.python-version }}
+          platforms: linux/arm64, linux/amd64
+          push: ${{ github.event_name == 'release' }}
+          tags: ${{ steps.flytekit-names.outputs.tags }}
+          build-args: |
+            VERSION=${{ needs.deploy.outputs.version }}
+            DOCKER_IMAGE=ghcr.io/${{ github.repository_owner }}/flytekit:py${{ matrix.python-version }}-${{ needs.deploy.outputs.version }}
+            PYTHON_VERSION=${{ matrix.python-version }}
+          file: Dockerfile
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Prepare SQLAlchemy Image Names
+        id: sqlalchemy-names
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            ghcr.io/${{ github.repository_owner }}/flytekit
+          tags: |
+            py${{ matrix.python-version }}-sqlalchemy-latest
+            py${{ matrix.python-version }}-sqlalchemy-${{ github.sha }}
+            py${{ matrix.python-version }}-sqlalchemy-${{ needs.deploy.outputs.version }}
       - name: Push SQLAlchemy Image to GitHub Registry
-        uses: whoan/docker-build-with-cache-action@v5
+        uses: docker/build-push-action@v2
         with:
-          # https://docs.github.com/en/packages/learn-github-packages/publishing-a-package
-          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
-          password: "${{ secrets.FLYTE_BOT_PAT }}"
-          image_name: ${{ github.repository_owner }}/flytekit
-          image_tag: py${{ matrix.python-version }}-sqlalchemy-latest,py${{ matrix.python-version }}-sqlalchemy-${{ github.sha }},py${{ matrix.python-version }}-sqlalchemy-${{ needs.deploy.outputs.version }}
-          push_git_tag: true
-          push_image_and_stages: true
-          registry: ghcr.io
-          build_extra_args: "--compress=true --build-arg=VERSION=${{ needs.deploy.outputs.version }}"
           context: "./plugins/flytekit-sqlalchemy/"
-          dockerfile: Dockerfile.py${{ matrix.python-version }}
+          platforms: linux/arm64, linux/amd64
+          push: ${{ github.event_name == 'release' }}
+          tags: ${{ steps.sqlalchemy-names.outputs.tags }}
+          build-args: |
+            VERSION=${{ needs.deploy.outputs.version }}
+          file: ./plugins/flytekit-sqlalchemy/Dockerfile.py${{ matrix.python-version }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.gitignore

@@ -31,3 +31,4 @@ docs/source/plugins/generated/
 htmlcov
 *.ipynb
 *dat
+source/_tags/

Dockerfile

@@ -0,0 +1,23 @@
+ARG PYTHON_VERSION
+FROM python:${PYTHON_VERSION}-slim-buster
+
+MAINTAINER Flyte Team <[email protected]>
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit
+
+WORKDIR /root
+ENV PYTHONPATH /root
+
+ARG VERSION
+ARG DOCKER_IMAGE
+
+RUN apt-get update && apt-get install build-essential -y
+
+# Pod tasks should be exposed in the default image
+RUN pip install -U flytekit==$VERSION \
+	flytekitplugins-pod==$VERSION \
+	flytekitplugins-deck-standard==$VERSION \
+	flytekitplugins-data-fsspec[aws]==$VERSION \
+	flytekitplugins-data-fsspec[gcp]==$VERSION \
+	scikit-learn
+
+ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE"

MANIFEST.in

@@ -3,7 +3,6 @@
 # include folders
 recursive-include flytekit *
 recursive-include flytekit_scripts *
-recursive-include plugins *
 
 # include specific files
 include README.md
@@ -25,6 +24,7 @@ recursive-exclude tests *
 recursive-exclude docs *
 recursive-exclude boilerplate *
 recursive-exclude .github *
+recursive-exclude plugins *
 
 # exclude dist folder:
 #  - contains the generated *.tar.gz and .whl files.

Makefile

@@ -49,9 +49,18 @@ spellcheck:  ## Runs a spellchecker over all code and documentation
 .PHONY: test
 test: lint unit_test
 
+.PHONY: unit_test_codecov
+unit_test_codecov:
+	# Ensure coverage file
+	rm coverage.xml || true
+	$(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" unit_test
+
 .PHONY: unit_test
 unit_test:
-	pytest -m "not sandbox_test" tests/flytekit/unit
+	# Skip tensorflow tests and run them with the necessary env var set so that a working (albeit slower)
+	# library is used to serialize/deserialize protobufs is used.
+	pytest -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} && \
+		PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python pytest tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS}
 
 requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt
 requirements-spark2.txt: requirements-spark2.in install-piptools

dev-requirements.in

@@ -12,4 +12,8 @@ codespell
 google-cloud-bigquery
 google-cloud-bigquery-storage
 IPython
-torch
+tensorflow==2.8.1
+# Newer versions of torch bring in nvidia dependencies that are not present in windows, so
+# we put this constraint while we do not have per-environment requirements files
+torch<=1.12.1
+scikit-learn