Remove requirements files (#1511)

* Remove mentions to spark Signed-off-by: Eduardo Apolinario <[email protected]> * Remove mentions to requirements.txt and dev-requirements.txt Signed-off-by: Eduardo Apolinario <[email protected]> * Remove plugins requirements. Signed-off-by: Eduardo Apolinario <[email protected]> * Remove all_requirements target from plugins makefile Signed-off-by: Eduardo Apolinario <[email protected]> * Use dev-requirements.in in lint gh action job Signed-off-by: Eduardo Apolinario <[email protected]> * Revert "Remove all_requirements target from plugins makefile" This reverts commit 50cbb4d. Signed-off-by: Eduardo Apolinario <[email protected]> * Revert "Remove plugins requirements." This reverts commit eae945c. Signed-off-by: Eduardo Apolinario <[email protected]> --------- Signed-off-by: Eduardo Apolinario <[email protected]> Co-authored-by: Eduardo Apolinario <[email protected]>
flyteorg · Feb 17, 2023 · 99d3d50 · 99d3d50
1 parent 74cbe58
commit 99d3d50
Show file tree

Hide file tree

Showing 8 changed files with 11 additions and 1,191 deletions.
diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
@@ -17,14 +17,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest]
         python-version: ["3.7", "3.8", "3.9", "3.10"]
-        spark-version-suffix: ["", "-spark2"]
         exclude:
-          - python-version: 3.8
-            spark-version-suffix: "-spark2"
-          - python-version: 3.9
-            spark-version-suffix: "-spark2"
-          - python-version: 3.10
-            spark-version-suffix: "-spark2"
           # Ignore this test because we failed to install docker-py
           # docker-py will install pywin32==227, whereas pywin only added support for python 3.10 in version 301.
           # For more detail, see https://github.com/flyteorg/flytekit/pull/856#issuecomment-1067152855
@@ -42,10 +35,10 @@ jobs:
           # This path is specific to Ubuntu
           path: ~/.cache/pip
           # Look to see if there is a cache hit for the corresponding requirements files
-          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
+          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
       - name: Install dependencies
         run: |
-          make setup${{ matrix.spark-version-suffix }}
+          make setup
           pip freeze
       - name: Test with coverage
         run: |
@@ -157,13 +150,13 @@ jobs:
       - uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.in') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip==21.2.4
-          pip install -r dev-requirements.txt
+          python -m pip install --upgrade pip
+          pip install -r dev-requirements.in
       - name: Lint
         run: |
           make lint

diff --git a/Makefile b/Makefile
@@ -22,11 +22,7 @@ update_boilerplate:
 
 .PHONY: setup
 setup: install-piptools ## Install requirements
-	pip-sync requirements.txt dev-requirements.txt
-
-.PHONY: setup-spark2
-setup-spark2: install-piptools ## Install requirements
-	pip-sync requirements-spark2.txt dev-requirements.txt
+	pip install -r dev-requirements.in
 
 .PHONY: fmt
 fmt: ## Format code with black and isort
@@ -63,18 +59,6 @@ unit_test:
 	pytest -m "not sandbox_test" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS} && \
 		PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python pytest tests/flytekit/unit/extras/tensorflow ${CODECOV_OPTS}
 
-requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt
-requirements-spark2.txt: requirements-spark2.in install-piptools
-	$(PIP_COMPILE) $<
-
-requirements.txt: export CUSTOM_COMPILE_COMMAND := make requirements.txt
-requirements.txt: requirements.in install-piptools
-	$(PIP_COMPILE) $<
-
-dev-requirements.txt: export CUSTOM_COMPILE_COMMAND := make dev-requirements.txt
-dev-requirements.txt: dev-requirements.in requirements.txt install-piptools
-	$(PIP_COMPILE) $<
-
 doc-requirements.txt: export CUSTOM_COMPILE_COMMAND := make doc-requirements.txt
 doc-requirements.txt: doc-requirements.in install-piptools
 	$(PIP_COMPILE) $<
@@ -84,7 +68,7 @@ ${MOCK_FLYTE_REPO}/requirements.txt: ${MOCK_FLYTE_REPO}/requirements.in install-
 	$(PIP_COMPILE) $<
 
 .PHONY: requirements
-requirements: requirements.txt dev-requirements.txt requirements-spark2.txt doc-requirements.txt ${MOCK_FLYTE_REPO}/requirements.txt ## Compile requirements
+requirements: doc-requirements.txt ${MOCK_FLYTE_REPO}/requirements.txt ## Compile requirements
 
 # TODO: Change this in the future to be all of flytekit
 .PHONY: coverage

diff --git a/dev-requirements.in b/dev-requirements.in
@@ -1,4 +1,4 @@
--c requirements.txt
+-r requirements.in
 
 git+https://github.com/flyteorg/pytest-flyte@main#egg=pytest-flyte
 coverage[toml]
@@ -12,7 +12,9 @@ codespell
 google-cloud-bigquery
 google-cloud-bigquery-storage
 IPython
-tensorflow==2.8.1
+
+# Only install tensorflow if not running on an arm Mac.
+tensorflow==2.8.1; platform_machine!='arm64' or platform_system!='Darwin'
 # Newer versions of torch bring in nvidia dependencies that are not present in windows, so
 # we put this constraint while we do not have per-environment requirements files
 torch<=1.12.1