Build Linux AARCH wheels for Python 3.7 and 3.9

coqui-ai · Aug 4, 2022 · b617281 · b617281
1 parent d0706c4
commit b617281
Show file tree

Hide file tree

Showing 7 changed files with 95 additions and 66 deletions.
diff --git a/.github/actions/multistrap/action.yml b/.github/actions/multistrap/action.yml
@@ -1,9 +1,13 @@
 name: "multistrap install"
 description: "Install a system root using multistrap"
 inputs:
-  multistrap-config:
-    description: "Multistrap config file to use"
+  arch:
+    description: "Target arch"
     required: true
+  python-version:
+    description: "Python version"
+    required: false
+    default: ""
   packages:
     description: "Extra packages to install"
     required: false
@@ -20,12 +24,22 @@ runs:
       run: |
         set -xe
 
-        multistrap_conf="${{ inputs.multistrap-config }}"
-        if [ "$multistrap_conf" = "multistrap_raspbian_buster.conf" ]; then
-          wget http://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring_20120528.2_all.deb && sudo dpkg -i raspbian-archive-keyring_20120528.2_all.deb
-        elif [ "$multistrap_conf" = "multistrap_raspbian_bullseye.conf" ]; then
+        multistrap_conf=""
+        if [ "${{ inputs.arch }}" = "armv7" ]; then
           wget http://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring_20120528.2_all.deb && sudo dpkg -i raspbian-archive-keyring_20120528.2_all.deb
         fi
+        if [ "${{ inputs.arch }}" = "armv7" ] && [ "${{ inputs.python-version }}" = "3.7" ]; then
+          multistrap_conf=multistrap_raspbian_buster.conf
+        fi
+        if [ "${{ inputs.arch }}" = "armv7" ] && [ "${{ inputs.python-version }}" = "3.9" ]; then
+          multistrap_conf=multistrap_raspbian_bullseye.conf
+        fi
+        if [ "${{ inputs.arch }}" = "aarch64" ] && [ "${{ inputs.python-version }}" = "3.7" ]; then
+          multistrap_conf=multistrap_armbian64_buster.conf
+        fi
+        if [ "${{ inputs.arch }}" = "aarch64" ] && [ "${{ inputs.python-version }}" = "3.9" ]; then
+          multistrap_conf=multistrap_armbian64_bullseye.conf
+        fi
 
         # Retry 2 times because Rasbian mirrors are often failing
         if ! multistrap -d ${{ env.SYSTEM_RASPBIAN }} -f ${{ github.workspace }}/native_client/${multistrap_conf}; then

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -637,12 +637,12 @@ jobs:
     needs: [ build-nodejs-Linux, train-test-model-Linux ]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         # https://nodejs.org/en/about/releases/
         nodejs-version: [12, 14, 16, 17]
         models: ["test"]
         samplerate: ["16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite
@@ -689,11 +689,11 @@ jobs:
     needs: [ build-nodejs-Linux, train-test-model-Linux ]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         electronjs-version: [12.0.0, 13.0.0, 14.0.0, 15.0.0, 16.0.0]
         models: ["test"]
         samplerate: ["16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite
@@ -1169,12 +1169,6 @@ jobs:
       - uses: actions/download-artifact@v2
         with:
           name: stt-tflite-3.7-aarch64.whl
-      - uses: actions/download-artifact@v2
-        with:
-          name: stt-tflite-3.8-armv7.whl
-      - uses: actions/download-artifact@v2
-        with:
-          name: stt-tflite-3.8-aarch64.whl
       - uses: actions/download-artifact@v2
         with:
           name: stt-tflite-3.9-armv7.whl
@@ -1926,12 +1920,12 @@ jobs:
     needs: [build-nodejs-macOS, train-test-model-Linux]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         # https://nodejs.org/en/about/releases/
         nodejs-version: [12, 14, 16, 17]
         models: ["test"]
         samplerate: ["16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite
@@ -2342,11 +2336,11 @@ jobs:
     needs: [ build-nodejs-Windows, train-test-model-Linux ]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         nodejs-version: [12, 14, 16]
         models: ["test"]
         samplerate: ["16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: tmp/
       STT_TEST_MODEL: tmp/output_graph.tflite
@@ -2510,12 +2504,12 @@ jobs:
     needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         # https://nodejs.org/en/about/releases/
         nodejs-version: [12, 17]
         models: ["test", "prod"]
         samplerate: ["8000", "16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite
@@ -2562,11 +2556,11 @@ jobs:
     needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         electronjs-version: [12.0.0, 16.0.0]
         models: ["test", "prod"]
         samplerate: ["8000", "16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite
@@ -2614,12 +2608,12 @@ jobs:
     needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         # https://nodejs.org/en/about/releases/
         nodejs-version: [12, 17]
         models: ["test", "prod"]
         samplerate: ["8000", "16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite
@@ -2664,11 +2658,11 @@ jobs:
     needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         electronjs-version: [12.0.0, 16.0.0]
         models: ["test", "prod"]
         samplerate: ["8000", "16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite
@@ -2717,12 +2711,12 @@ jobs:
     needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         # https://nodejs.org/en/about/releases/
         nodejs-version: [12, 16]
         models: ["test", "prod"]
         samplerate: ["8000", "16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: tmp/
       STT_TEST_MODEL: tmp/output_graph.tflite
@@ -2780,11 +2774,11 @@ jobs:
     needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         electronjs-version: [12.0.0, 16.0.0]
         models: ["test", "prod"]
         samplerate: ["8000", "16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: tmp/
       STT_TEST_MODEL: tmp/output_graph.tflite
@@ -2854,7 +2848,7 @@ jobs:
       - name: Install chroot
         uses: ./.github/actions/multistrap
         with:
-          multistrap-config: multistrap_raspbian_buster.conf
+          arch: "armv7"
       - run: ./ci_scripts/tf-setup.sh
       - run: ./ci_scripts/armv7-build.sh
       - run: ./ci_scripts/package.sh
@@ -2883,7 +2877,7 @@ jobs:
       - name: Install chroot
         uses: ./.github/actions/multistrap
         with:
-          multistrap-config: multistrap_armbian64_buster.conf
+          arch: "aarch64"
       - run: ./ci_scripts/tf-setup.sh
       - run: ./ci_scripts/aarch64-build.sh
       - run: ./ci_scripts/package.sh
@@ -2900,15 +2894,15 @@ jobs:
     runs-on: ubuntu-20.04
     needs: [build-lib-LinuxArmv7, swig_Linux]
     strategy:
-      fail-fast: false
       matrix:
         include:
           - python-version: "3.7"
             system-raspbian: multistrap-raspbian-buster
-            multistrap-config: multistrap_raspbian_buster.conf
+            arch: "armv7"
           - python-version: "3.9"
             system-raspbian: multistrap-raspbian-bullseye
-            multistrap-config: multistrap_raspbian_bullseye.conf
+            arch: "armv7"
+      fail-fast: false
     env:
       DEBIAN_FRONTEND: "noninteractive"
       SYSTEM_TARGET: rpi3
@@ -2940,7 +2934,8 @@ jobs:
       - name: "Install chroot"
         uses: ./.github/actions/multistrap
         with:
-          multistrap-config: ${{ matrix.multistrap-config }}
+          arch: "${{ matrix.arch }}"
+          python-version: "${{ matrix.python-version }}"
       - id: get_numpy
         uses: ./.github/actions/numpy_vers
         with:
@@ -2996,7 +2991,7 @@ jobs:
       - name: "Install chroot"
         uses: ./.github/actions/multistrap
         with:
-          multistrap-config: multistrap_raspbian_buster.conf
+          arch: "armv7"
       - uses: actions/setup-node@v2
         with:
           node-version: 12
@@ -3037,11 +3032,18 @@ jobs:
     needs: [build-lib-LinuxAarch64, swig_Linux]
     strategy:
       matrix:
-        python-version: ["3.7"]
+        include:
+          - python-version: "3.7"
+            system-raspbian: multistrap-armbian64-buster
+            arch: "aarch64"
+          - python-version: "3.9"
+            system-raspbian: multistrap-armbian64-bullseye
+            arch: "aarch64"
+      fail-fast: false
     env:
       DEBIAN_FRONTEND: "noninteractive"
       SYSTEM_TARGET: rpi3-armv8
-      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-armbian64-buster
+      SYSTEM_RASPBIAN: "${{ github.workspace }}/${{ matrix.system-raspbian }}"
     steps:
       - run: |
           sudo apt-get install -y --no-install-recommends
@@ -3071,7 +3073,8 @@ jobs:
       - name: "Install chroot"
         uses: ./.github/actions/multistrap
         with:
-          multistrap-config: multistrap_armbian64_buster.conf
+          arch: "${{ matrix.arch }}"
+          python-version: "${{ matrix.python-version }}"
       - id: get_numpy
         uses: ./.github/actions/numpy_vers
         with:
@@ -3127,7 +3130,7 @@ jobs:
       - name: "Install chroot"
         uses: ./.github/actions/multistrap
         with:
-          multistrap-config: multistrap_armbian64_buster.conf
+          arch: "aarch64"
       - uses: actions/setup-node@v2
         with:
           node-version: 12
@@ -3167,18 +3170,15 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         include:
           - arch: "armv7"
             system: "buster"
-            multistrap-config: "multistrap_raspbian_buster.conf"
           - arch: "armv7"
             system: "bullseye"
-            multistrap-config: "multistrap_raspbian_bullseye.conf"
           - arch: "aarch64"
             system: "buster"
-            multistrap-config: "multistrap_armbian64_buster.conf"
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp
       DEBIAN_FRONTEND: "noninteractive"
@@ -3190,7 +3190,7 @@ jobs:
       - name: "Install and setup chroot"
         uses: ./.github/actions/multistrap
         with:
-          multistrap-config: ${{ matrix.multistrap-config }}
+          arch: "${{ matrix.arch }}"
           packages: "bash wget curl sox xxd libatlas3-base libopenblas-base ca-certificates python3 python3-pip gnupg libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libdbus-1-3 libgdk-pixbuf2.0-0 libgtk-3-0 libgbm1 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libx11-xcb1 libxcb-dri3-0 libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 xauth"
       - name: "Create a chroot tarball"
         run: |
@@ -3205,7 +3205,6 @@ jobs:
     needs: [ build-lib-LinuxArmv7, build-lib-LinuxAarch64, train-test-model-Linux, build-test-chroot ]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         include:
           - arch: "armv7"
@@ -3256,6 +3255,7 @@ jobs:
             system: "buster"
             models: "prod"
             samplerate: "16000"
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp
       DEBIAN_FRONTEND: "noninteractive"
@@ -3310,7 +3310,6 @@ jobs:
     needs: [ build-python-LinuxArmv7, build-python-LinuxAarch64, train-test-model-Linux, build-test-chroot ]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         include:
           - arch: "armv7"
@@ -3373,6 +3372,7 @@ jobs:
             python-version: "3.7"
             models: "prod"
             samplerate: "16000"
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp
       DEBIAN_FRONTEND: "noninteractive"
@@ -3427,13 +3427,13 @@ jobs:
     needs: [ build-nodejs-LinuxArmv7, build-nodejs-LinuxAarch64, train-test-model-Linux, build-test-chroot ]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
-      fail-fast: false
       matrix:
         arch: ["armv7", "aarch64"]
         # https://nodejs.org/en/about/releases/
         nodejs-version: [12, 14, 16, 17]
         models: ["test"]
         samplerate: ["16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp
       DEBIAN_FRONTEND: "noninteractive"
@@ -3490,12 +3490,12 @@ jobs:
     # Disable this task because it seems qemu does not work super-well with ElectronJS
     if: ${{ github.event_name == 'disabled' }}
     strategy:
-      fail-fast: false
       matrix:
         arch: ["armv7", "aarch64"]
         electronjs-version: [12.0.0, 13.0.0, 14.0.0, 15.0.0, 16.0.0]
         models: ["test"]
         samplerate: ["16000"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp
       DEBIAN_FRONTEND: "noninteractive"

diff --git a/Dockerfile.build b/Dockerfile.build
@@ -6,7 +6,7 @@ FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04
 # >> START Install base software
 
 # Get basic packages
-RUN apt-get update && apt-get install -y --no-install-recommends --fix-broken \
+RUN apt-get update && apt-get install -y --no-install-recommends \
         apt-utils \
         bash-completion \
         build-essential \