diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
index ec1a7aa17..b51f0113b 100644
--- a/.github/workflows/build-and-test.yml
+++ b/.github/workflows/build-and-test.yml
@@ -1539,7 +1539,7 @@ jobs:
         uses: ./.github/actions/get_cache_key
         with:
           arch: "macOS-${{ matrix.arch }}"
-          extras: "0"
+          extras: "1"
       - id: check_artifact_exists
         uses: ./.github/actions/check_artifact_exists
         with:
@@ -1569,6 +1569,41 @@ jobs:
           else
             sudo xcode-select --switch /Applications/Xcode_12.1.1.app/Contents/Developer
           fi
+      - uses: actions/cache@v2
+        id: cache
+        if: matrix.arch == 'arm64'
+        with:
+          path: ~/arm-target
+          key: ${{ runner.os }}-arm-brew-cache
+      - name: Install arm64 deps
+        if: matrix.arch == 'arm64' && steps.cache.outputs.cache-hit != 'true'
+        run: |
+          set -xe
+          mkdir -p ~/arm-target/bin
+          mkdir -p ~/arm-target/brew-cache
+          export PATH="$HOME/arm-target/bin:$PATH"
+
+          cd ~/arm-target
+          mkdir arm-brew && curl -L https://github.com/Homebrew/brew/tarball/77359472938971510506cb2337a665db1d46534d | tar xz --strip 1 -C arm-brew
+
+          export HOMEBREW_NO_AUTO_UPDATE=1
+          export HOMEBREW_CACHE=~/arm-target/brew-cache
+          export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
+          ~/arm-target/arm-brew/bin/brew fetch --deps --bottle-tag=arm64_big_sur sox opusfile |\
+             grep -E "(Downloaded to:|Already downloaded:)" |\
+             grep -v pkg-config |\
+             awk '{ print $3 }' |\
+             xargs -n 1 ~/arm-target/arm-brew/bin/brew install --ignore-dependencies --force-bottle
+
+          # Install host version of pkg-config so we can call it in the build system
+          ~/arm-target/arm-brew/bin/brew install pkg-config
+          ln -s ~/arm-target/arm-homebrew/bin/pkg-config ~/arm-target/bin/arm-pkg-config
+      - run: |
+          echo "$HOME/arm-target/bin" >> $GITHUB_PATH
+        if: matrix.arch == 'arm64'
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: matrix.arch == 'arm64'
       - run: ./ci_scripts/tf-setup.sh
       - run: |
           # Taken from https://www.tensorflow.org/install/source
diff --git a/ci_scripts/all-vars.sh b/ci_scripts/all-vars.sh
index cea546126..710886ade 100755
--- a/ci_scripts/all-vars.sh
+++ b/ci_scripts/all-vars.sh
@@ -27,24 +27,7 @@ if [ "${OS}" = "Darwin" ]; then
     export DS_ROOT_TASK=${CI_TASK_DIR}
     export DS_CPU_COUNT=$(sysctl hw.ncpu |cut -d' ' -f2)
     export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root"
-
-    export HOMEBREW_NO_AUTO_UPDATE=1
-    export BREW_URL=https://github.com/Homebrew/brew/tarball/2.2.17
-
-    export BUILDS_BREW="${CI_TASK_DIR}/homebrew-builds"
-    export TESTS_BREW="${CI_TASK_DIR}/homebrew-tests"
-
-    export NVM_DIR=$TESTS_BREW/.nvm/ && mkdir -p $NVM_DIR
-    export PKG_CONFIG_PATH="${BUILDS_BREW}/lib/pkgconfig"
-
-    if [ -f "${BUILDS_BREW}/bin/brew" ]; then
-        export PATH=${BUILDS_BREW}/bin/:${BUILDS_BREW}/opt/node@12/bin:$PATH
-    fi;
-
-    if [ -f "${TESTS_BREW}/bin/brew" ]; then
-        export PATH=${TESTS_BREW}/bin/:$PATH
-    fi;
-fi;
+fi
 
 export CI_ARTIFACTS_DIR=${CI_ARTIFACTS_DIR:-${CI_TASK_DIR}/artifacts}
 export CI_TMP_DIR=${CI_TMP_DIR:-/tmp}
diff --git a/ci_scripts/host-build.sh b/ci_scripts/host-build.sh
index 6fdea44e3..fb4606604 100755
--- a/ci_scripts/host-build.sh
+++ b/ci_scripts/host-build.sh
@@ -2,7 +2,11 @@
 
 set -xe
 
-macos_arch=$1
+macos_target_arch=$1
+SYSTEM_TARGET=host
+if [ "$(uname)-$(uname -m)" = "Darwin-x86_64" -a "${macos_target_arch}" = "arm64" ]; then
+  SYSTEM_TARGET="darwin-arm64"
+fi
 
 source $(dirname "$0")/all-vars.sh
 source $(dirname "$0")/all-utils.sh
@@ -16,20 +20,7 @@ BAZEL_TARGETS="
 "
 
 BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}"
-if [ "${OS}" = "Darwin" ]; then
-  if [ "${macos_arch}" = "arm64" ]; then
-    BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS_MACOS_ARM64} ${BAZEL_EXTRA_FLAGS_MACOS_ARM64}"
-    EXTRA_LOCAL_CFLAGS="-mmacosx-version-min=11.0 -target arm64-apple-macos11 -DNO_SOX"
-    EXTRA_LOCAL_LDFLAGS="-mmacosx-version-min=11.0 -target arm64-apple-macos11"
-  else
-    BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS_MACOS_X86_64} ${BAZEL_EXTRA_FLAGS_MACOS_X86_64}"
-    EXTRA_LOCAL_CFLAGS="-mmacosx-version-min=10.10 -target x86_64-apple-macos10.10"
-    EXTRA_LOCAL_LDFLAGS="-mmacosx-version-min=10.10 -target x86_64-apple-macos10.10"
-  fi
-fi
-
 BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
-SYSTEM_TARGET=host
 
 do_bazel_build
 
diff --git a/ci_scripts/ios-build.sh b/ci_scripts/ios-build.sh
index 23259e490..7bd15695b 100755
--- a/ci_scripts/ios-build.sh
+++ b/ci_scripts/ios-build.sh
@@ -17,6 +17,6 @@ BAZEL_TARGETS="
 BAZEL_BUILD_FLAGS="${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}"
 
 BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
-SYSTEM_TARGET=host
+SYSTEM_TARGET=
 
 do_bazel_build
diff --git a/ci_scripts/tf-vars.sh b/ci_scripts/tf-vars.sh
index 628cb35f1..ec465d0f5 100755
--- a/ci_scripts/tf-vars.sh
+++ b/ci_scripts/tf-vars.sh
@@ -135,19 +135,36 @@ fi
 # Build for generic amd64 platforms, no device-specific optimization
 # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for targetting specific CPUs
 
+if [ "${OS}" != "${CI_MSYS_VERSION}" ]; then
+    BAZEL_EXTRA_FLAGS="--config=noaws --config=nogcp --config=nohdfs --config=nonccl"
+fi
+
 if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
     BAZEL_OPT_FLAGS="--copt=/arch:AVX"
 elif [ "${OS}" = "Darwin" ]; then
-    if [ "$(uname -m)" = "arm64" ]; then
-        # clang on M1 Macs doesn't support -march=x86-64
-        BAZEL_OPT_FLAGS_MACOS_X86_64=""
+    FROM="$(uname | tr '[:upper:]' '[:lower:]')-$(uname -m)"
+    if [ "$SYSTEM_TARGET" = "host" ]; then
+        TO="$FROM"
     else
-        BAZEL_OPT_FLAGS_MACOS_X86_64="--copt=-mtune=generic --copt=-march=x86-64 --copt=-msse --copt=-msse2 --copt=-msse3 --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx"
+        TO="$SYSTEM_TARGET"
+    fi
+
+    if [ "$FROM" = "darwin-x86_64" -a "$TO" = "darwin-x86_64" ]; then
+        BAZEL_OPT_FLAGS="--copt=-mtune=generic --copt=-march=x86-64 --copt=-msse --copt=-msse2 --copt=-msse3 --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx"
+        if [ "${CI}" = true ]; then
+            BAZEL_EXTRA_FLAGS="${BAZEL_EXTRA_FLAGS} --macos_minimum_os 10.10 --macos_sdk_version 10.15"
+        fi
+    elif [ "$FROM" = "darwin-x86_64" -a "$TO" = "darwin-arm64" ]; then
+        BAZEL_OPT_FLAGS=""
+        if [ "${CI}" = true ]; then
+            BAZEL_EXTRA_FLAGS="--config=macos_arm64 --xcode_version 12.2 --macos_minimum_os 11.0 --macos_sdk_version 11.0"
+        fi
+    elif [ "$FROM" = "darwin-arm64" -a "$TO" = "darwin-arm64" ]; then
+        BAZEL_OPT_FLAGS=""
+    elif [ "$FROM" = "darwin-arm64" -a "$TO" = "darwin-x86_64" ]; then
+        echo "TensorFlow does not support building for x86_64 on arm64" 1>&2
+        exit 1
     fi
-    BAZEL_OPT_FLAGS_MACOS_ARM64="--xcode_version 12.2"
-    BAZEL_OPT_FLAGS="--xcode_version 12.2"
-else
-    BAZEL_OPT_FLAGS="--copt=-mtune=generic --copt=-march=x86-64 --copt=-msse --copt=-msse2 --copt=-msse3 --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx"
 fi
 
 BAZEL_OUTPUT_CACHE_DIR="${DS_ROOT_TASK}/.bazel_cache/"
@@ -175,19 +192,6 @@ BAZEL_ANDROID_X86_64_FLAGS="--config=android_x86_64"
 BAZEL_IOS_ARM64_FLAGS="--config=ios_arm64"
 BAZEL_IOS_X86_64_FLAGS="--config=ios_x86_64"
 
-if [ "${OS}" != "${CI_MSYS_VERSION}" ]; then
-    BAZEL_EXTRA_FLAGS="--config=noaws --config=nogcp --config=nohdfs --config=nonccl"
-fi
-
-if [ "${OS}" = "Darwin" ]; then
-    BAZEL_EXTRA_FLAGS_MACOS_X86_64="${BAZEL_EXTRA_FLAGS}"
-    BAZEL_EXTRA_FLAGS_MACOS_ARM64="${BAZEL_EXTRA_FLAGS} --config=macos_arm64"
-    if [ "${CI}" = true ]; then
-        BAZEL_EXTRA_FLAGS_MACOS_X86_64="${BAZEL_EXTRA_FLAGS_MACOS_X86_64} --macos_minimum_os 10.10 --macos_sdk_version 10.15"
-        BAZEL_EXTRA_FLAGS_MACOS_ARM64="${BAZEL_EXTRA_FLAGS_MACOS_ARM64} --macos_minimum_os 11.0 --macos_sdk_version 11.0"
-    fi
-fi
-
 ### Define build targets that we will re-ues in sourcing scripts.
 BUILD_TARGET_LIB_CPP_API="//tensorflow:tensorflow_cc"
 BUILD_TARGET_LITE_LIB="//tensorflow/lite:libtensorflowlite.so"
diff --git a/native_client/definitions.mk b/native_client/definitions.mk
index 167b69698..aa1d19b09 100644
--- a/native_client/definitions.mk
+++ b/native_client/definitions.mk
@@ -34,6 +34,9 @@ ifeq ($(OS),Linux)
 LINK_STT := $(LINK_STT) -llzma -lbz2
 SOX_LDFLAGS     := -L$(ROOT_DIR)/sox-build/lib -lsox
 else ifeq ($(OS),Darwin)
+CFLAGS                  := -mmacosx-version-min=10.10 -target x86_64-apple-macos10.10
+LDFLAGS                 := -mmacosx-version-min=10.10 -target x86_64-apple-macos10.10
+
 SOX_CFLAGS              := $(shell pkg-config --cflags sox)
 LIBSOX_PATH             := $(shell echo `pkg-config --libs-only-L sox | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l sox | sed -e 's/^-l//'`.dylib)
 LIBOPUSFILE_PATH        := $(shell echo `pkg-config --libs-only-L opusfile | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l opusfile | sed -e 's/^-l//'`.dylib)
@@ -120,6 +123,18 @@ SOX_LDFLAGS     :=
 LDFLAGS         :=
 endif
 
+ifeq ($(TARGET),darwin-arm64)
+CFLAGS                  := -mmacosx-version-min=11.0 -target arm64-apple-macos11
+LDFLAGS                 := -mmacosx-version-min=11.0 -target arm64-apple-macos11
+
+SOX_CFLAGS              := $(shell arm-pkg-config --cflags sox)
+LIBSOX_PATH             := $(shell echo `arm-pkg-config --libs-only-L sox | sed -e 's/^-L//'`/lib`arm-pkg-config --libs-only-l sox | sed -e 's/^-l//'`.dylib)
+LIBOPUSFILE_PATH        := $(shell echo `arm-pkg-config --libs-only-L opusfile | sed -e 's/^-L//'`/lib`arm-pkg-config --libs-only-l opusfile | sed -e 's/^-l//'`.dylib)
+LIBSOX_STATIC_DEPS      := $(shell echo `otool -L $(LIBSOX_PATH) | tail -n +2 | cut -d' ' -f1 | grep /opt/ | sed -E "s/\.[[:digit:]]+\.dylib/\.a/" | tr '\n' ' '`)
+LIBOPUSFILE_STATIC_DEPS := $(shell echo `otool -L $(LIBOPUSFILE_PATH) | tail -n +2 | cut -d' ' -f1 | grep /opt/ | sed -E "s/\.[[:digit:]]+\.dylib/\.a/" | tr '\n' ' '`)
+SOX_LDFLAGS             := $(LIBSOX_STATIC_DEPS) $(LIBOPUSFILE_STATIC_DEPS) -framework CoreAudio -lz
+endif
+
 # -Wl,--no-as-needed is required to force linker not to evict libs it thinks we
 # dont need ; will fail the build on OSX because that option does not exists
 ifeq ($(OS),Linux)