From 02b958dd439209f4cbbd57fac59d3af8860d21f0 Mon Sep 17 00:00:00 2001 From: Kajetan Puchalski Date: Tue, 3 Dec 2024 13:46:26 +0000 Subject: [PATCH] ci: Enable opt-dist for dist-aarch64-linux builds Move the CI dist-aarch64-linux job to an aarch64 runner and enable optimised dist builds with the opt-dist pipeline. For the time being, disable bolt on aarch64 due to upstream bolt bugs. --- .../dist-aarch64-linux/Dockerfile | 54 +++++++++++++++++++ .../dist-aarch64-linux/build-clang.sh | 46 ++++++++++++++++ .../host-aarch64/dist-aarch64-linux/shared.sh | 16 ++++++ .../host-x86_64/dist-aarch64-linux/Dockerfile | 32 ----------- .../aarch64-linux-gnu.defconfig | 10 ---- src/ci/github-actions/jobs.yml | 10 ++-- src/tools/opt-dist/src/bolt.rs | 13 ++++- src/tools/opt-dist/src/main.rs | 28 +++++++--- 8 files changed, 153 insertions(+), 56 deletions(-) create mode 100644 src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile create mode 100755 src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh create mode 100644 src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh delete mode 100644 src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile delete mode 100644 src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile new file mode 100644 index 0000000000000..451cebd556704 --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile @@ -0,0 +1,54 @@ +FROM ubuntu:22.04 + +WORKDIR /build + +RUN apt-get update && \ + apt-get install -y pkg-config curl xz-utils wget unzip git make patch perl lbzip2 gcc g++ libc6-dev cmake python3 doxygen graphviz zstd libzstd-dev openssl libssl-dev + +ENV PATH=/rustroot/bin:$PATH + +WORKDIR /tmp +COPY host-aarch64/dist-aarch64-linux/shared.sh /tmp/ + +ENV CC=gcc CXX=g++ + +# Build LLVM+Clang +COPY host-aarch64/dist-aarch64-linux/build-clang.sh /tmp/ +RUN ./build-clang.sh +ENV CC=clang CXX=clang++ + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV PGO_HOST=aarch64-unknown-linux-gnu +ENV HOSTS=aarch64-unknown-linux-gnu + +ENV CPATH=/usr/include/aarch64-linux-gnu/:$CPATH + +ENV RUST_CONFIGURE_ARGS \ + --build=aarch64-unknown-linux-gnu \ + --enable-full-tools \ + --enable-profiler \ + --enable-sanitizers \ + --enable-compiler-docs \ + --set target.aarch64-unknown-linux-gnu.linker=clang \ + --set target.aarch64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \ + --set target.aarch64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \ + --set llvm.link-shared=true \ + --set llvm.thin-lto=true \ + --set llvm.libzstd=true \ + --set llvm.ninja=false \ + --set rust.debug-assertions=false \ + --set rust.jemalloc \ + --set rust.use-lld=true \ + --set rust.lto=thin \ + --set rust.codegen-units=1 + + +ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \ + ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \ + --host $HOSTS --target $HOSTS --include-default-paths build-manifest bootstrap + +ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=clang +ENV DIST_SRC 1 +ENV DIST_REQUIRE_ALL_TOOLS 1 diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh new file mode 100755 index 0000000000000..fb5f4d8c4f795 --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/build-clang.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -ex + +source shared.sh + +# Try to keep the LLVM version here in sync with src/ci/scripts/install-clang.sh +LLVM=llvmorg-19.1.5 + +mkdir llvm-project +cd llvm-project + +curl -L https://github.com/llvm/llvm-project/archive/$LLVM.tar.gz | \ + tar xzf - --strip-components=1 + +mkdir clang-build +cd clang-build + +# For whatever reason the default set of include paths for clang is different +# than that of gcc. As a result we need to manually include our sysroot's +# include path, /rustroot/include, to clang's default include path. +INC="/rustroot/include:/usr/include" + +# We need compiler-rt for the profile runtime (used later to PGO the LLVM build) +# but sanitizers aren't currently building. Since we don't need those, just +# disable them. BOLT is used for optimizing LLVM. +hide_output \ + cmake ../llvm \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/rustroot \ + -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ + -DCOMPILER_RT_BUILD_XRAY=OFF \ + -DCOMPILER_RT_BUILD_MEMPROF=OFF \ + -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \ + -DLLVM_TARGETS_TO_BUILD=AArch64 \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;bolt" \ + -DC_INCLUDE_DIRS="$INC" + +hide_output make -j$(nproc) +hide_output make install + +cd ../.. +rm -rf llvm-project diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh b/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh new file mode 100644 index 0000000000000..dc86dddd464f2 --- /dev/null +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/shared.sh @@ -0,0 +1,16 @@ +#!/bin/sh +hide_output() { + set +x + on_err=" +echo ERROR: An error was encountered with the build. +cat /tmp/build.log +exit 1 +" + trap "$on_err" ERR + bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & + PING_LOOP_PID=$! + "$@" &> /tmp/build.log + trap - ERR + kill $PING_LOOP_PID + set -x +} diff --git a/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile deleted file mode 100644 index 18972387e34d7..0000000000000 --- a/src/ci/docker/host-x86_64/dist-aarch64-linux/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM ubuntu:22.04 - -COPY scripts/cross-apt-packages.sh /scripts/ -RUN sh /scripts/cross-apt-packages.sh - -COPY scripts/crosstool-ng.sh /scripts/ -RUN sh /scripts/crosstool-ng.sh - -COPY scripts/rustbuild-setup.sh /scripts/ -RUN sh /scripts/rustbuild-setup.sh -WORKDIR /tmp - -COPY scripts/crosstool-ng-build.sh /scripts/ -COPY host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig /tmp/crosstool.defconfig -RUN /scripts/crosstool-ng-build.sh - -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - -ENV PATH=$PATH:/x-tools/aarch64-unknown-linux-gnu/bin - -ENV CC_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-gcc \ - AR_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-ar \ - CXX_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnu-g++ - -ENV HOSTS=aarch64-unknown-linux-gnu - -ENV RUST_CONFIGURE_ARGS \ - --enable-full-tools \ - --enable-profiler \ - --enable-sanitizers -ENV SCRIPT python3 ../x.py dist --host $HOSTS --target $HOSTS diff --git a/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig b/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig deleted file mode 100644 index 520b1667c8be1..0000000000000 --- a/src/ci/docker/host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.defconfig +++ /dev/null @@ -1,10 +0,0 @@ -CT_CONFIG_VERSION="4" -CT_PREFIX_DIR="/x-tools/${CT_TARGET}" -CT_USE_MIRROR=y -CT_MIRROR_BASE_URL="https://ci-mirrors.rust-lang.org/rustc" -CT_ARCH_ARM=y -CT_ARCH_64=y -CT_KERNEL_LINUX=y -CT_LINUX_V_4_1=y -CT_GLIBC_V_2_17=y -CT_CC_LANG_CXX=y diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 454c855c75d25..02c85f3c0cf08 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -128,17 +128,17 @@ auto: - image: aarch64-gnu-debug <<: *job-aarch64-linux + - image: dist-aarch64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift + <<: *job-aarch64-linux + - image: arm-android <<: *job-linux-4c - image: armhf-gnu <<: *job-linux-4c - - image: dist-aarch64-linux - env: - CODEGEN_BACKENDS: llvm,cranelift - <<: *job-linux-4c - - image: dist-android <<: *job-linux-4c diff --git a/src/tools/opt-dist/src/bolt.rs b/src/tools/opt-dist/src/bolt.rs index d9efbb8d880e7..0f1fda38115c7 100644 --- a/src/tools/opt-dist/src/bolt.rs +++ b/src/tools/opt-dist/src/bolt.rs @@ -1,6 +1,7 @@ use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; +use crate::environment::Environment; use crate::exec::cmd; use crate::training::BoltProfile; use crate::utils::io::copy_file; @@ -45,13 +46,21 @@ pub fn with_bolt_instrumented anyhow::Result, R>( } /// Optimizes the file at `path` with BOLT in-place using the given `profile`. -pub fn bolt_optimize(path: &Utf8Path, profile: &BoltProfile) -> anyhow::Result<()> { +pub fn bolt_optimize( + path: &Utf8Path, + profile: &BoltProfile, + env: &Environment, +) -> anyhow::Result<()> { // Copy the artifact to a new location, so that we do not use the same input and output file. // BOLT cannot handle optimizing when the input and output is the same file, because it performs // in-place patching. let temp_path = tempfile::NamedTempFile::new()?.into_temp_path(); copy_file(path, &temp_path)?; + // FIXME: cdsplit in llvm-bolt is currently broken on AArch64, drop this once it's fixed upstream + let split_strategy = + if env.host_tuple().starts_with("aarch64") { "profile2" } else { "cdsplit" }; + cmd(&["llvm-bolt"]) .arg(temp_path.display()) .arg("-data") @@ -65,7 +74,7 @@ pub fn bolt_optimize(path: &Utf8Path, profile: &BoltProfile) -> anyhow::Result<( // Split function code into hot and code regions .arg("-split-functions") // Split using best available strategy (three-way splitting, Cache-Directed Sort) - .arg("-split-strategy=cdsplit") + .arg(format!("-split-strategy={split_strategy}")) // Split as many basic blocks as possible .arg("-split-all-cold") // Move jump tables to a separate section diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index c871200f3cfcb..aa05b5f0e76ca 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -146,6 +146,21 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing"); + let is_aarch64 = target_triple.starts_with("aarch64"); + + let mut skip_tests = vec![ + // Fails because of linker errors, as of June 2023. + "tests/ui/process/nofile-limit.rs".to_string(), + ]; + + if is_aarch64 { + skip_tests.extend([ + // Those tests fail only inside of Docker on aarch64, as of December 2024 + "tests/ui/consts/promoted_running_out_of_memory_issue-130687.rs".to_string(), + "tests/ui/consts/large_const_alloc.rs".to_string(), + ]); + } + let checkout_dir = Utf8PathBuf::from("/checkout"); let env = EnvironmentBuilder::default() .host_tuple(target_triple) @@ -155,11 +170,9 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec)> .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")) .build_dir(checkout_dir.join("obj")) .shared_llvm(true) - .use_bolt(true) - .skipped_tests(vec![ - // Fails because of linker errors, as of June 2023. - "tests/ui/process/nofile-limit.rs".to_string(), - ]) + // FIXME: Enable bolt for aarch64 once it's fixed upstream. Broken as of December 2024. + .use_bolt(!is_aarch64) + .skipped_tests(skip_tests) .build()?; (env, shared.build_args) @@ -304,7 +317,8 @@ fn execute_pipeline( // the final dist build. However, when BOLT optimizes an artifact, it does so *in-place*, // therefore it will actually optimize all the hard links, which means that the final // packaged `libLLVM.so` file *will* be BOLT optimized. - bolt_optimize(&llvm_lib, &llvm_profile).context("Could not optimize LLVM with BOLT")?; + bolt_optimize(&llvm_lib, &llvm_profile, env) + .context("Could not optimize LLVM with BOLT")?; let rustc_lib = io::find_file_in_dir(&libdir, "librustc_driver", ".so")?; @@ -319,7 +333,7 @@ fn execute_pipeline( print_free_disk_space()?; // Now optimize the library with BOLT. - bolt_optimize(&rustc_lib, &rustc_profile) + bolt_optimize(&rustc_lib, &rustc_profile, env) .context("Could not optimize rustc with BOLT")?; // LLVM is not being cleared here, we want to use the BOLT-optimized LLVM