Eager workflows to support async workflows

.github/workflows/pythonbuild.yml

@@ -47,6 +47,50 @@ jobs:
         with:
           fail_ci_if_error: false
 
+  build-integration:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        # python 3.11 has intermittent issues with the docker build + push step
+        # https://github.com/flyteorg/flytekit/actions/runs/5800978835/job/15724237979?pr=1579
+        python-version: ["3.8", "3.9", "3.10"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements files
+          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
+      - name: Install dependencies
+        run: make setup && pip freeze
+      - name: Install FlyteCTL
+        uses: unionai-oss/flytectl-setup-action@master
+      - name: Setup Flyte Sandbox
+        run: |
+          flytectl demo start
+          flytectl config init
+      - name: Build and push to local registry
+        run: |
+          docker build . -f Dockerfile.dev -t localhost:30000/flytekit:dev --build-arg PYTHON_VERSION=${{ matrix.python-version }}
+          docker push localhost:30000/flytekit:dev
+      - name: Integration Test with coverage
+        env:
+          FLYTEKIT_IMAGE: localhost:30000/flytekit:dev
+          FLYTEKIT_CI: 1
+        run: make integration_test_codecov
+      - name: Codecov
+        uses: codecov/[email protected]
+        with:
+          fail_ci_if_error: false
+
   build-plugins:
     needs: build
     runs-on: ubuntu-latest

Makefile

@@ -1,6 +1,6 @@
 export REPOSITORY=flytekit
 
-PIP_COMPILE = pip-compile --upgrade --verbose
+PIP_COMPILE = pip-compile --upgrade --verbose --resolver=backtracking
 MOCK_FLYTE_REPO=tests/flytekit/integration/remote/mock_flyte_repo/workflows
 
 .SILENT: help
@@ -60,9 +60,19 @@ unit_test:
 	pytest -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} && \
 		PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python pytest tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS}
 
+.PHONY: integration_test_codecov
+integration_test_codecov:
+	# Ensure coverage file
+	rm coverage.xml || true
+	$(MAKE) CODECOV_OPTS="--cov=./ --cov-report=xml --cov-append" integration_test
+
+.PHONY: integration_test
+integration_test:
+	pytest tests/flytekit/integration/experimental ${CODECOV_OPTS}
+
 doc-requirements.txt: export CUSTOM_COMPILE_COMMAND := make doc-requirements.txt
 doc-requirements.txt: doc-requirements.in install-piptools
-	docker run --platform linux/amd64  --rm -it --volume .:/root python:3.9-slim-buster sh -c "cd /root && apt-get update && apt-get install git -y && pip install pip-tools && pip-compile --upgrade --verbose doc-requirements.in"
+	$(PIP_COMPILE) $<
 
 ${MOCK_FLYTE_REPO}/requirements.txt: export CUSTOM_COMPILE_COMMAND := make ${MOCK_FLYTE_REPO}/requirements.txt
 ${MOCK_FLYTE_REPO}/requirements.txt: ${MOCK_FLYTE_REPO}/requirements.in install-piptools

dev-requirements.in

@@ -17,7 +17,7 @@ keyrings.alt
 # Only install tensorflow if not running on an arm Mac.
 tensorflow==2.8.1; python_version<'3.11' and (platform_machine!='arm64' or platform_system!='Darwin')
 # Tensorflow release candidate supports python 3.11
-tensorflow==2.12.0rc1; python_version>='3.11' and (platform_machine!='arm64' or platform_system!='Darwin')
+tensorflow==2.13.0; python_version>='3.11' and (platform_machine!='arm64' or platform_system!='Darwin')
 
 # Newer versions of torch bring in nvidia dependencies that are not present in windows, so
 # we put this constraint while we do not have per-environment requirements files

doc-requirements.in

@@ -29,11 +29,13 @@ plotly                      # deck
 pandas_profiling            # deck
 dolt_integrations           # dolt
 great-expectations          # greatexpectations
+datasets                    # huggingface
 kubernetes                  # k8s-pod
 modin                       # modin
 pandera                     # pandera
 papermill                   # papermill
 jupyter                     # papermill
+polars                      # polars
 pyspark                     # spark
 sqlalchemy                  # sqlalchemy
 torch                       # pytorch