Merge branch 'master' into containertask-podtemplate

.github/workflows/docs_build.yml

@@ -0,0 +1,26 @@
+name: Docs Build
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+jobs:
+  docs_warnings:
+    name: Docs Warnings
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: "0"
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: Report Sphinx Warnings
+        id: sphinx-warnings
+        run: |
+          sudo apt-get install python3-sphinx
+          pip install -r doc-requirements.txt
+          SPHINXOPTS="-W" cd docs && make html

.github/workflows/pythonbuild.yml

@@ -16,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10"]
         exclude:
           # Ignore this test because we failed to install docker-py
           # docker-py will install pywin32==227, whereas pywin only added support for python 3.10 in version 301.
@@ -65,6 +65,7 @@ jobs:
           - flytekit-dbt
           - flytekit-deck-standard
           - flytekit-dolt
+          - flytekit-duckdb
           - flytekit-greatexpectations
           - flytekit-hive
           - flytekit-k8s-pod
@@ -169,11 +170,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Fetch the code
-        uses: actions/checkout@v2
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/checkout@v3
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip==21.2.4 setuptools wheel

.github/workflows/pythonpublish.yml

@@ -47,11 +47,28 @@ jobs:
         run: |
           make -C plugins build_all_plugins
           make -C plugins publish_all_plugins
-      # Added sleep because PYPI take some time in publish
-      - name: Sleep for 180 seconds
-        uses: jakejarvis/wait-action@master
-        with:
-          time: '180s'
+      - name: Sleep until pypi is available
+        id: pypiwait
+        run: |
+          # from refs/tags/v1.2.3 get 1.2.3 and make sure it's not an empty string
+          VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+          if [ -z "$VERSION" ]
+          then
+            echo "No tagged version found, exiting"
+            exit 1
+          fi
+          LINK="https://pypi.org/project/flytekit/${VERSION}"
+          for i in {1..60}; do
+            if curl -L -I -s -f ${LINK} >/dev/null; then
+              echo "Found pypi"
+              exit 0
+            else
+              echo "Did not find - Retrying in 10 seconds..."
+              sleep 10
+            fi
+          done
+          exit 1
+        shell: bash
     outputs:
       version: ${{ steps.bump.outputs.version }}
 
@@ -60,7 +77,7 @@ jobs:
     needs: deploy
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10"]
     steps:
       - uses: actions/checkout@v2
         with:

Dockerfile

@@ -4,10 +4,6 @@ FROM python:${PYTHON_VERSION}-slim-buster
 MAINTAINER Flyte Team <[email protected]>
 LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit
 
-RUN useradd -u 1000 flytekit
-RUN chown flytekit: /root
-USER flytekit
-
 WORKDIR /root
 ENV PYTHONPATH /root
 
@@ -24,4 +20,8 @@ RUN pip install -U flytekit==$VERSION \
 	flytekitplugins-data-fsspec[gcp]==$VERSION \
 	scikit-learn
 
+RUN useradd -u 1000 flytekit
+RUN chown flytekit: /root
+USER flytekit
+
 ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE"

Dockerfile.dev

@@ -0,0 +1,32 @@
+# This Dockerfile is here to help with end-to-end testing
+# From flytekit
+# $ docker build -f Dockerfile.dev --build-arg PYTHON_VERSION=3.10 -t localhost:30000/flytekittest:someversion .
+# $ docker push localhost:30000/flytekittest:someversion
+# From your test user code
+# $ pyflyte run --image localhost:30000/flytekittest:someversion
+
+ARG PYTHON_VERSION
+FROM python:${PYTHON_VERSION}-slim-buster
+
+MAINTAINER Flyte Team <[email protected]>
+LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit
+
+WORKDIR /root
+
+ARG VERSION
+
+RUN apt-get update && apt-get install build-essential vim -y
+
+COPY . /flytekit
+
+# Pod tasks should be exposed in the default image
+RUN pip install -e /flytekit
+RUN pip install -e /flytekit/plugins/flytekit-k8s-pod
+RUN pip install -e /flytekit/plugins/flytekit-deck-standard
+RUN pip install scikit-learn
+
+ENV PYTHONPATH "/flytekit:/flytekit/plugins/flytekit-k8s-pod:/flytekit/plugins/flytekit-deck-standard:"
+
+RUN useradd -u 1000 flytekit
+RUN chown flytekit: /root
+USER flytekit

dev-requirements.in

@@ -12,6 +12,7 @@ codespell
 google-cloud-bigquery
 google-cloud-bigquery-storage
 IPython
+keyrings.alt
 
 # Only install tensorflow if not running on an arm Mac.
 tensorflow==2.8.1; platform_machine!='arm64' or platform_system!='Darwin'

doc-requirements.in

@@ -47,4 +47,5 @@ ray                         # ray
 scikit-learn                # scikit-learn
 dask[distributed]           # dask
 vaex                        # vaex
-mlflow                       # mlflow
+mlflow                      # mlflow
+duckdb                      # duckdb