ci: Enable opt-dist for dist-aarch64-linux builds

Move the CI dist-aarch64-linux job to an aarch64 runner and enable
optimised dist builds with the opt-dist pipeline.

For the time being, disable bolt on aarch64 due to upstream bolt bugs.

src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile

@@ -0,0 +1,54 @@
+FROM ubuntu:22.04
+
+WORKDIR /build
+
+RUN apt-get update && \
+	apt-get install -y pkg-config curl xz-utils wget unzip git make patch perl lbzip2 gcc g++ libc6-dev cmake python3 doxygen graphviz zstd libzstd-dev openssl libssl-dev
+
+ENV PATH=/rustroot/bin:$PATH
+
+WORKDIR /tmp
+COPY host-aarch64/dist-aarch64-linux/shared.sh /tmp/
+
+ENV CC=gcc CXX=g++
+
+# Build LLVM+Clang
+COPY host-aarch64/dist-aarch64-linux/build-clang.sh /tmp/
+RUN ./build-clang.sh
+ENV CC=clang CXX=clang++
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV PGO_HOST=aarch64-unknown-linux-gnu
+ENV HOSTS=aarch64-unknown-linux-gnu
+
+ENV CPATH=/usr/include/aarch64-linux-gnu/:$CPATH
+
+ENV RUST_CONFIGURE_ARGS \
+      --build=aarch64-unknown-linux-gnu \
+      --enable-full-tools \
+      --enable-profiler \
+      --enable-sanitizers \
+      --enable-compiler-docs \
+      --set target.aarch64-unknown-linux-gnu.linker=clang \
+      --set target.aarch64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \
+      --set target.aarch64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
+      --set llvm.link-shared=true \
+      --set llvm.thin-lto=true \
+      --set llvm.libzstd=true \
+      --set llvm.ninja=false \
+      --set rust.debug-assertions=false \
+      --set rust.jemalloc \
+      --set rust.use-lld=true \
+      --set rust.lto=thin \
+      --set rust.codegen-units=1
+
+
+ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
+      ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci --  python3 ../x.py dist \
+      --host $HOSTS --target $HOSTS --include-default-paths build-manifest bootstrap
+
+ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=clang
+ENV DIST_SRC 1
+ENV DIST_REQUIRE_ALL_TOOLS 1

src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -ex
+
+source shared.sh
+
+# Try to keep the LLVM version here in sync with src/ci/scripts/install-clang.sh
+LLVM=llvmorg-19.1.5
+
+mkdir llvm-project
+cd llvm-project
+
+curl -L https://github.com/llvm/llvm-project/archive/$LLVM.tar.gz | \
+  tar xzf - --strip-components=1
+
+mkdir clang-build
+cd clang-build
+
+# For whatever reason the default set of include paths for clang is different
+# than that of gcc. As a result we need to manually include our sysroot's
+# include path, /rustroot/include, to clang's default include path.
+INC="/rustroot/include:/usr/include"
+
+# We need compiler-rt for the profile runtime (used later to PGO the LLVM build)
+# but sanitizers aren't currently building. Since we don't need those, just
+# disable them. BOLT is used for optimizing LLVM.
+hide_output \
+    cmake ../llvm \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX=/rustroot \
+      -DCOMPILER_RT_BUILD_SANITIZERS=OFF \
+      -DCOMPILER_RT_BUILD_XRAY=OFF \
+      -DCOMPILER_RT_BUILD_MEMPROF=OFF \
+      -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \
+      -DLLVM_TARGETS_TO_BUILD=AArch64 \
+      -DLLVM_INCLUDE_BENCHMARKS=OFF \
+      -DLLVM_INCLUDE_TESTS=OFF \
+      -DLLVM_INCLUDE_EXAMPLES=OFF \
+      -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \
+      -DC_INCLUDE_DIRS="$INC"
+
+hide_output make -j$(nproc)
+hide_output make install
+
+cd ../..
+rm -rf llvm-project

src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+hide_output() {
+  set +x
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  "$@" &> /tmp/build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  set -x
+}

src/ci/github-actions/jobs.yml

@@ -128,17 +128,17 @@ auto:
   - image: aarch64-gnu-debug
     <<: *job-aarch64-linux
 
+  - image: dist-aarch64-linux
+    env:
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-aarch64-linux
+
   - image: arm-android
     <<: *job-linux-4c
 
   - image: armhf-gnu
     <<: *job-linux-4c
 
-  - image: dist-aarch64-linux
-    env:
-      CODEGEN_BACKENDS: llvm,cranelift
-    <<: *job-linux-4c
-
   - image: dist-android
     <<: *job-linux-4c
 

src/tools/opt-dist/src/bolt.rs

@@ -1,6 +1,7 @@
 use anyhow::Context;
 use camino::{Utf8Path, Utf8PathBuf};
 
+use crate::environment::Environment;
 use crate::exec::cmd;
 use crate::training::BoltProfile;
 use crate::utils::io::copy_file;
@@ -45,13 +46,21 @@ pub fn with_bolt_instrumented<F: FnOnce(&Utf8Path) -> anyhow::Result<R>, R>(
 }
 
 /// Optimizes the file at `path` with BOLT in-place using the given `profile`.
-pub fn bolt_optimize(path: &Utf8Path, profile: &BoltProfile) -> anyhow::Result<()> {
+pub fn bolt_optimize(
+    path: &Utf8Path,
+    profile: &BoltProfile,
+    env: &Environment,
+) -> anyhow::Result<()> {
     // Copy the artifact to a new location, so that we do not use the same input and output file.
     // BOLT cannot handle optimizing when the input and output is the same file, because it performs
     // in-place patching.
     let temp_path = tempfile::NamedTempFile::new()?.into_temp_path();
     copy_file(path, &temp_path)?;
 
+    // FIXME: cdsplit in llvm-bolt is currently broken on AArch64, drop this once it's fixed upstream
+    let split_strategy =
+        if env.host_tuple().starts_with("aarch64") { "profile2" } else { "cdsplit" };
+
     cmd(&["llvm-bolt"])
         .arg(temp_path.display())
         .arg("-data")
@@ -65,7 +74,7 @@ pub fn bolt_optimize(path: &Utf8Path, profile: &BoltProfile) -> anyhow::Result<(
         // Split function code into hot and code regions
         .arg("-split-functions")
         // Split using best available strategy (three-way splitting, Cache-Directed Sort)
-        .arg("-split-strategy=cdsplit")
+        .arg(format!("-split-strategy={split_strategy}"))
         // Split as many basic blocks as possible
         .arg("-split-all-cold")
         // Move jump tables to a separate section

src/tools/opt-dist/src/main.rs

@@ -146,6 +146,21 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)>
             let target_triple =
                 std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
 
+            let is_aarch64 = target_triple.starts_with("aarch64");
+
+            let mut skip_tests = vec![
+                // Fails because of linker errors, as of June 2023.
+                "tests/ui/process/nofile-limit.rs".to_string(),
+            ];
+
+            if is_aarch64 {
+                skip_tests.extend([
+                    // Those tests fail only inside of Docker on aarch64, as of December 2024
+                    "tests/ui/consts/promoted_running_out_of_memory_issue-130687.rs".to_string(),
+                    "tests/ui/consts/large_const_alloc.rs".to_string(),
+                ]);
+            }
+
             let checkout_dir = Utf8PathBuf::from("/checkout");
             let env = EnvironmentBuilder::default()
                 .host_tuple(target_triple)
@@ -155,11 +170,9 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)>
                 .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts"))
                 .build_dir(checkout_dir.join("obj"))
                 .shared_llvm(true)
-                .use_bolt(true)
-                .skipped_tests(vec![
-                    // Fails because of linker errors, as of June 2023.
-                    "tests/ui/process/nofile-limit.rs".to_string(),
-                ])
+                // FIXME: Enable bolt for aarch64 once it's fixed upstream. Broken as of December 2024.
+                .use_bolt(!is_aarch64)
+                .skipped_tests(skip_tests)
                 .build()?;
 
             (env, shared.build_args)
@@ -304,7 +317,8 @@ fn execute_pipeline(
             // the final dist build. However, when BOLT optimizes an artifact, it does so *in-place*,
             // therefore it will actually optimize all the hard links, which means that the final
             // packaged `libLLVM.so` file *will* be BOLT optimized.
-            bolt_optimize(&llvm_lib, &llvm_profile).context("Could not optimize LLVM with BOLT")?;
+            bolt_optimize(&llvm_lib, &llvm_profile, env)
+                .context("Could not optimize LLVM with BOLT")?;
 
             let rustc_lib = io::find_file_in_dir(&libdir, "librustc_driver", ".so")?;
 
@@ -319,7 +333,7 @@ fn execute_pipeline(
             print_free_disk_space()?;
 
             // Now optimize the library with BOLT.
-            bolt_optimize(&rustc_lib, &rustc_profile)
+            bolt_optimize(&rustc_lib, &rustc_profile, env)
                 .context("Could not optimize rustc with BOLT")?;
 
             // LLVM is not being cleared here, we want to use the BOLT-optimized LLVM