Refactor CI Builds and tests to make our lives easier

.github/actions/multistrap/action.yml

@@ -22,11 +22,11 @@ runs:
 
         multistrap_conf=""
         if [ "${{ inputs.arch }}" = "armv7" ]; then
-          multistrap_conf=multistrap_raspbian_buster.conf
+          multistrap_conf=multistrap_raspbian_bullseye.conf
           wget http://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring_20120528.2_all.deb && sudo dpkg -i raspbian-archive-keyring_20120528.2_all.deb
         fi
         if [ "${{ inputs.arch }}" = "aarch64" ]; then
-          multistrap_conf=multistrap_armbian64_buster.conf
+          multistrap_conf=multistrap_armbian64_bullseye.conf
         fi
 
         # Retry 2 times because Rasbian mirrors are often failing

.github/workflows/build-and-test.yml

@@ -850,6 +850,9 @@ jobs:
           # Test FLAC input
           time ./bin/run-ci-ldc93s1-flac.sh --epochs 1
 
+          # Test LM gen
+          time ./bin/run-ci-lm-gen-batch.sh
+
           # Test LM opt
           time ./bin/run-ci-lm-opt.sh
   training-sdb-tests:
@@ -2808,7 +2811,7 @@ jobs:
     runs-on: ubuntu-20.04
     env:
       SYSTEM_TARGET: rpi3
-      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-raspbian-buster
+      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-raspbian-bullseye
     steps:
       - uses: actions/checkout@v2
         with:
@@ -2837,15 +2840,15 @@ jobs:
     runs-on: ubuntu-20.04
     env:
       SYSTEM_TARGET: rpi3-armv8
-      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-armbian64-buster
+      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-armbian64-bullseye
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
           submodules: 'recursive'
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.7"
+          python-version: "3.9"
       - name: Install chroot
         uses: ./.github/actions/multistrap
         with:
@@ -2871,7 +2874,7 @@ jobs:
     env:
       DEBIAN_FRONTEND: "noninteractive"
       SYSTEM_TARGET: rpi3
-      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-raspbian-buster
+      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-raspbian-bullseye
     steps:
       - uses: actions/checkout@v2
         with:
@@ -2930,7 +2933,7 @@ jobs:
     needs: [build-lib-LinuxArmv7, swig_Linux]
     env:
       SYSTEM_TARGET: rpi3
-      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-raspbian-buster
+      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-raspbian-bullseye
     steps:
       - uses: actions/checkout@v2
         with:
@@ -2996,11 +2999,11 @@ jobs:
     needs: [build-lib-LinuxAarch64, swig_Linux]
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.9]
     env:
       DEBIAN_FRONTEND: "noninteractive"
       SYSTEM_TARGET: rpi3-armv8
-      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-armbian64-buster
+      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-armbian64-bullseye
     steps:
       - run: |
           sudo apt-get install -y --no-install-recommends
@@ -3061,7 +3064,7 @@ jobs:
     needs: [build-lib-LinuxAarch64, swig_Linux]
     env:
       SYSTEM_TARGET: rpi3-armv8
-      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-armbian64-buster
+      SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-armbian64-bullseye
     steps:
       - uses: actions/checkout@v2
         with:
@@ -3214,7 +3217,7 @@ jobs:
     strategy:
       matrix:
         arch: [ "armv7", "aarch64" ]
-        python-version: [3.7]
+        python-version: [3.9]
         models: ["test", "prod"]
         samplerate: ["8000", "16000"]
     env:
@@ -3440,7 +3443,7 @@ jobs:
           submodules: 'recursive'
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.7"
+          python-version: "3.9"
       - uses: actions/setup-java@v2
         with:
           distribution: "temurin"

Dockerfile.train

@@ -39,6 +39,8 @@ RUN apt-get update && \
         libvorbisfile3 \
         libopusfile0 \
         libsndfile1 \
+        libboost-program-options-dev \
+        libboost-thread-dev \
         sox \
         libsox-fmt-mp3 \
         python3-venv \

bin/run-ci-lm-gen-batch.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# This test optimizes the scorer for testing purposes
+
+set -xe
+
+lm_path="./data/lm"
+sources_lm_filepath="./data/smoke_test/vocab.txt"
+
+# Force only one visible device because we have a single-sample dataset
+# and when trying to run on multiple devices (like GPUs), this will break
+
+python data/lm/generate_lm_batch.py \
+    --input_txt "${sources_lm_filepath}" \
+    --output_dir "${lm_path}" \
+    --top_k_list 30000 \
+    --arpa_order_list "4" \
+    --max_arpa_memory "85%" \
+    --arpa_prune_list "0|0|2" \
+    --binary_a_bits 255 \
+    --binary_q_bits 8 \
+    --binary_type trie \
+    --kenlm_bins /code/kenlm/build/bin/ \
+    -j 1