Revert "IFU-main-2023-07-31 (#35)"

This reverts commit 4a72eb3.
ROCm · Aug 7, 2023 · c76bfce · c76bfce
1 parent 4a72eb3
commit c76bfce
Show file tree

Hide file tree

Showing 92 changed files with 2,017 additions and 718 deletions.
diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh
@@ -2,39 +2,49 @@ if [[ ${MATRIX_PACKAGE_TYPE} == "libtorch" ]]; then
     curl ${MATRIX_INSTALLATION} -o libtorch.zip
     unzip libtorch.zip
 else
-    # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159
-    conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
-    conda activate ${ENV_NAME}
-    INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"}
-
-    export OLD_PATH=${PATH}
-    # Workaround macos-arm64 runners. Issue: https://github.com/pytorch/test-infra/issues/4342
-    if [[ ${TARGET_OS} == 'macos-arm64' ]]; then
-        export PATH="${CONDA_PREFIX}/bin:${PATH}"
-    fi
-
-    # Make sure we remove previous installation if it exists, this issue seems to affect only
-    if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then
-        pip3 uninstall -y torch torchaudio torchvision
-    fi
-    eval $INSTALLATION
+    #special case for Python 3.11
+    if [[ ${MATRIX_PYTHON_VERSION} == '3.11' ]]; then
+        conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION}
+        conda activate ${ENV_NAME}
 
-    if [[ ${TARGET_OS} == 'linux' ]]; then
-        export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib"
-        export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH
-        ${PWD}/check_binary.sh
-    fi
+        INSTALLATION=${MATRIX_INSTALLATION/"-c pytorch"/"-c malfet -c pytorch"}
+        INSTALLATION=${INSTALLATION/"pytorch-cuda"/"pytorch-${MATRIX_CHANNEL}::pytorch-cuda"}
+        INSTALLATION=${INSTALLATION/"conda install"/"conda install -y"}
 
-    if [[ ${TARGET_OS} == 'windows' ]]; then
-        python  ./test/smoke_test/smoke_test.py
+        eval $INSTALLATION
+        python ./test/smoke_test/smoke_test.py
+        conda deactivate
+        conda env remove -n ${ENV_NAME}
     else
-        python3  ./test/smoke_test/smoke_test.py
-    fi
 
-    if [[ ${TARGET_OS} == 'macos-arm64' ]]; then
-        export PATH=${OLD_PATH}
-    fi
 
-    conda deactivate
-    conda env remove -n ${ENV_NAME}
+
+        # Special case Pypi installation package, only applicable to linux nightly CUDA 11.7 builds, wheel package
+        if [[ ${TARGET_OS} == 'linux' && ${MATRIX_GPU_ARCH_VERSION} == '11.7' && ${MATRIX_PACKAGE_TYPE} == 'manywheel' && ${MATRIX_CHANNEL} != 'nightly' ]]; then
+            conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
+            INSTALLATION_PYPI=${MATRIX_INSTALLATION/"cu117"/"cu117_pypi_cudnn"}
+            INSTALLATION_PYPI=${INSTALLATION_PYPI/"torchvision torchaudio"/""}
+            INSTALLATION_PYPI=${INSTALLATION_PYPI/"index-url"/"extra-index-url"}
+            conda run -p ${ENV_NAME}_pypi ${INSTALLATION_PYPI}
+            conda run -p ${ENV_NAME}_pypi python ./test/smoke_test/smoke_test.py --package torchonly
+            conda deactivate
+            conda env remove -p ${ENV_NAME}_pypi
+        fi
+
+        # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159
+        conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
+        conda activate ${ENV_NAME}
+        INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"}
+        eval $INSTALLATION
+
+        if [[ ${TARGET_OS} == 'linux' ]]; then
+            export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib"
+            export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH
+            ${PWD}/check_binary.sh
+        fi
+
+        python  ./test/smoke_test/smoke_test.py
+        conda deactivate
+        conda env remove -n ${ENV_NAME}
+    fi
 fi
diff --git a/.github/scripts/validate_pipy.sh b/.github/scripts/validate_pipy.sh
diff --git a/.github/scripts/validate_poetry.sh b/.github/scripts/validate_poetry.sh
diff --git a/.github/workflows/build-conda-images.yml b/.github/workflows/build-conda-images.yml
@@ -3,12 +3,7 @@ name: Build conda docker images
 on:
   push:
     branches:
-      - main
-      - release/*
-    tags:
-      # NOTE: Binary build pipelines should only get triggered on release candidate or nightly builds
-      # Release candidate tags look like: v1.11.0-rc1
-      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
+      main
     paths:
       - conda/Dockerfile
       - 'common/*'
@@ -24,14 +19,14 @@ env:
   DOCKER_BUILDKIT: 1
   DOCKER_ID: ${{ secrets.DOCKER_ID }}
   DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
-  WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }}
+  WITH_PUSH: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
 
 jobs:
   build-docker:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        cuda_version: ["11.8", "12.1", "cpu"]
+        cuda_version: ["11.6", "11.7", "11.8", "cpu"]
     env:
       CUDA_VERSION: ${{ matrix.cuda_version }}
     steps:

diff --git a/.github/workflows/build-libtorch-images.yml b/.github/workflows/build-libtorch-images.yml
@@ -3,12 +3,7 @@ name: Build libtorch docker images
 on:
   push:
     branches:
-      - main
-      - release/*
-    tags:
-      # NOTE: Binary build pipelines should only get triggered on release candidate or nightly builds
-      # Release candidate tags look like: v1.11.0-rc1
-      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
+      main
     paths:
       - .github/workflows/build-libtorch-images.yml
       - libtorch/Dockerfile
@@ -26,14 +21,14 @@ env:
   DOCKER_BUILDKIT: 1
   DOCKER_ID: ${{ secrets.DOCKER_ID }}
   DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
-  WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }}
+  WITH_PUSH: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
 
 jobs:
   build-docker-cuda:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        cuda_version: ["12.1", "11.8"]
+        cuda_version: ["11.8", "11.7", "11.6"]
     env:
       GPU_ARCH_TYPE: cuda
       GPU_ARCH_VERSION: ${{ matrix.cuda_version }}
@@ -49,10 +44,10 @@ jobs:
         run: |
           libtorch/build_docker.sh
   build-docker-rocm:
-    runs-on: linux.12xlarge
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        rocm_version: ["5.5", "5.6"]
+        rocm_version: ["5.3", "5.4.2"]
     env:
       GPU_ARCH_TYPE: rocm
       GPU_ARCH_VERSION: ${{ matrix.rocm_version }}

diff --git a/.github/workflows/build-magma-linux.yml b/.github/workflows/build-magma-linux.yml
@@ -30,7 +30,7 @@ jobs:
     runs-on: linux.2xlarge
     strategy:
       matrix:
-        cuda_version: ["121", "118"]
+        cuda_version: ["118", "117", "116"]
     steps:
       - name: Checkout PyTorch builder
         uses: actions/checkout@v3

diff --git a/.github/workflows/build-magma-windows.yml b/.github/workflows/build-magma-windows.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: windows-2019
     strategy:
       matrix:
-        cuda_version: ["121", "118"]
+        cuda_version: ["118", "117", "116"]
         config: ["Release", "Debug"]
     env:
       CUDA_VERSION: ${{ matrix.cuda_version }}

diff --git a/.github/workflows/build-manywheel-images.yml b/.github/workflows/build-manywheel-images.yml
@@ -3,12 +3,7 @@ name: Build manywheel docker images
 on:
   push:
     branches:
-      - main
-      - release/*
-    tags:
-      # NOTE: Binary build pipelines should only get triggered on release candidate or nightly builds
-      # Release candidate tags look like: v1.11.0-rc1
-      - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
+      main
     paths:
       - .github/workflows/build-manywheel-images.yml
       - manywheel/Dockerfile
@@ -28,14 +23,14 @@ env:
   DOCKER_BUILDKIT: 1
   DOCKER_ID: ${{ secrets.DOCKER_ID }}
   DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
-  WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }}
+  WITH_PUSH: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
 
 jobs:
   build-docker-cuda:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        cuda_version: ["12.1", "11.8"]
+        cuda_version: ["11.8", "11.7", "11.6"]
     env:
       GPU_ARCH_TYPE: cuda
       GPU_ARCH_VERSION: ${{ matrix.cuda_version }}
@@ -51,10 +46,10 @@ jobs:
         run: |
           manywheel/build_docker.sh
   build-docker-rocm:
-    runs-on: linux.12xlarge
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        rocm_version: ["5.5", "5.6"]
+        rocm_version: ["5.3", "5.4.2"]
     env:
       GPU_ARCH_TYPE: rocm
       GPU_ARCH_VERSION: ${{ matrix.rocm_version }}

diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml
@@ -22,6 +22,10 @@ on:
         default: ""
         required: false
         type: string
+      limit-win-builds:
+        description: "Limit windows builds to single python/cuda config"
+        default: "disable"
+        type: string
   workflow_dispatch:
     inputs:
       os:
@@ -49,6 +53,11 @@ on:
         default: ""
         required: false
         type: string
+      limit-win-builds:
+        description: "Limit windows builds to single python/cuda config"
+        default: "disable"
+        required: false
+        type: string
 
 jobs:
   win:
@@ -57,6 +66,7 @@ jobs:
     with:
       channel: ${{ inputs.channel }}
       ref: ${{ inputs.ref || github.ref }}
+      limit-win-builds: ${{ inputs.limit-win-builds }}
 
   linux:
     if:  inputs.os == 'linux' || inputs.os == 'all'

diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml
@@ -55,14 +55,4 @@ jobs:
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TARGET_OS="linux"
         eval "$(conda shell.bash hook)"
-
-        # Special case PyPi installation package. And Install of PyPi package via poetry
-        if [[ ${MATRIX_PACKAGE_TYPE} == "manywheel" ]] && \
-           ([[ ${MATRIX_GPU_ARCH_VERSION} == "12.1" && ${MATRIX_CHANNEL} != "release" ]] || \
-            [[ ${MATRIX_GPU_ARCH_VERSION} == "11.7" && ${MATRIX_CHANNEL} == "release" ]]); then
-          source ./.github/scripts/validate_pipy.sh --runtime-error-check disabled
-          source ./.github/scripts/validate_poetry.sh --runtime-error-check disabled
-        fi
-
-        # Standart case: Validate binaries
         source ./.github/scripts/validate_binaries.sh
diff --git a/.github/workflows/validate-nightly-binaries.yml b/.github/workflows/validate-nightly-binaries.yml
@@ -3,8 +3,8 @@ name: cron
 
 on:
   schedule:
-    # At 3:30 pm UTC (8:30 am PDT)
-    - cron: "30 15 * * *"
+    # At 2:30 pm UTC (7:30 am PDT)
+    - cron: "30 14 * * *"
   # Have the ability to trigger this job manually through the API
   workflow_dispatch:
   push:
@@ -17,10 +17,19 @@ on:
       - .github/workflows/validate-macos-binaries.yml
       - .github/workflows/validate-macos-arm64-binaries.yml
       - test/smoke_test/*
-
+  pull_request:
+    paths:
+      - .github/workflows/validate-nightly-binaries.yml
+      - .github/workflows/validate-linux-binaries.yml
+      - .github/workflows/validate-windows-binaries.yml
+      - .github/workflows/validate-macos-binaries.yml
+      - .github/workflows/validate-macos-arm64-binaries.yml
+      - .github/scripts/validate_binaries.sh
+      - test/smoke_test/*
 jobs:
   nightly:
     uses: ./.github/workflows/validate-binaries.yml
     with:
       channel: nightly
       os: all
+      limit-win-builds: enable
diff --git a/.github/workflows/validate-release-binaries.yml b/.github/workflows/validate-release-binaries.yml
@@ -3,8 +3,8 @@ name: cron
 
 on:
   schedule:
-    # At 3 am UTC (7 am PDT)
-    - cron: "0 3 * * *"
+    # At 3 am and 2 pm UTC (7 am and 8 pm PDT)
+    - cron: "0 3,14 * * *"
   # Have the ability to trigger this job manually through the API
   workflow_dispatch:
   push:
@@ -17,19 +17,11 @@ on:
       - .github/workflows/validate-macos-binaries.yml
       - .github/workflows/validate-macos-arm64-binaries.yml
       - test/smoke_test/*
-  pull_request:
-    paths:
-      - .github/workflows/validate-nightly-binaries.yml
-      - .github/workflows/validate-linux-binaries.yml
-      - .github/workflows/validate-windows-binaries.yml
-      - .github/workflows/validate-macos-binaries.yml
-      - .github/workflows/validate-macos-arm64-binaries.yml
-      - .github/scripts/validate_binaries.sh
-      - test/smoke_test/*
 
 jobs:
   release:
     uses: ./.github/workflows/validate-binaries.yml
     with:
       channel: release
       os: all
+      limit-win-builds: enable