From 0fd1926203d1551a1708ee913b3a7593cd1915f7 Mon Sep 17 00:00:00 2001 From: Tyler Mandry Date: Tue, 7 Nov 2023 18:11:07 -0800 Subject: [PATCH 1/2] Build Fuchsia in CI --- .github/workflows/ci.yml | 3 + src/bootstrap/src/core/build_steps/compile.rs | 2 +- .../host-x86_64/dist-various-2/Dockerfile | 4 +- .../x86_64-gnu-integration/Dockerfile | 68 +++++++++++++++++++ .../x86_64-gnu-integration/build-fuchsia.sh | 54 +++++++++++++++ .../build-fuchsia-toolchain.sh | 8 +-- src/ci/github-actions/ci.yml | 4 ++ src/ci/shared.sh | 2 +- 8 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile create mode 100755 src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh rename src/ci/docker/{host-x86_64/dist-various-2 => scripts}/build-fuchsia-toolchain.sh (85%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f659a1c3060c..b59a46d9f071d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -288,6 +288,9 @@ jobs: - name: x86_64-gnu-aux os: ubuntu-20.04-4core-16gb env: {} + - name: x86_64-gnu-integration + os: ubuntu-20.04-16core-64gb + env: {} - name: x86_64-gnu-debug os: ubuntu-20.04-8core-32gb env: {} diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 51e4195827fc6..d961791c295bb 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1715,7 +1715,7 @@ impl Step for Assemble { let dst_exe = exe("rust-lld", target_compiler.host); builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); let self_contained_lld_dir = libdir_bin.join("gcc-ld"); - t!(fs::create_dir(&self_contained_lld_dir)); + t!(fs::create_dir_all(&self_contained_lld_dir)); let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { compiler: build_compiler, target: target_compiler.host, diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile index 01b46118b9c1e..ae71885320005 100644 --- a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile @@ -79,8 +79,8 @@ RUN env \ rm -rf /build/* WORKDIR /tmp -COPY host-x86_64/dist-various-2/shared.sh /tmp/ -COPY host-x86_64/dist-various-2/build-fuchsia-toolchain.sh /tmp/ +COPY scripts/shared.sh /tmp/ +COPY scripts/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/ RUN /tmp/build-solaris-toolchain.sh x86_64 amd64 solaris-i386 pc diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile new file mode 100644 index 0000000000000..ba65ba9bed460 --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-integration/Dockerfile @@ -0,0 +1,68 @@ +FROM ubuntu:22.04 + +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + g++ \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python3 \ + git \ + cmake \ + libssl-dev \ + sudo \ + xz-utils \ + pkg-config \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# Duplicated in dist-various-2 Dockerfile. +# FIXME: Move to canonical triple +ENV \ + AR_x86_64_fuchsia=x86_64-unknown-fuchsia-ar \ + CC_x86_64_fuchsia=x86_64-unknown-fuchsia-clang \ + CFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ + CXX_x86_64_fuchsia=x86_64-unknown-fuchsia-clang++ \ + CXXFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \ + LDFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib" + +WORKDIR /tmp +COPY scripts/shared.sh /tmp/ +COPY scripts/build-fuchsia-toolchain.sh /tmp/ +RUN /tmp/build-fuchsia-toolchain.sh + +ENV CARGO_TARGET_X86_64_FUCHSIA_AR /usr/local/bin/llvm-ar +ENV CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS \ + -C panic=abort \ + -C force-unwind-tables=yes \ + -C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \ + -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \ + -Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib + +ENV TARGETS=x86_64-fuchsia +ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnu + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV RUST_INSTALL_DIR /checkout/obj/install +RUN mkdir -p $RUST_INSTALL_DIR/etc + +ENV RUST_CONFIGURE_ARGS \ + --prefix=$RUST_INSTALL_DIR \ + --sysconfdir=etc \ + --enable-lld \ + --llvm-libunwind=in-tree \ + --enable-extended \ + --disable-docs \ + --set target.x86_64-fuchsia.cc=/usr/local/bin/clang \ + --set target.x86_64-fuchsia.cxx=/usr/local/bin/clang++ \ + --set target.x86_64-fuchsia.ar=/usr/local/bin/llvm-ar \ + --set target.x86_64-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \ + --set target.x86_64-fuchsia.linker=/usr/local/bin/ld.lld +ENV SCRIPT \ + python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \ + bash ../src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh b/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh new file mode 100755 index 0000000000000..efccadd88f239 --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-integration/build-fuchsia.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Downloads and builds the Fuchsia operating system using a toolchain installed +# in $RUST_INSTALL_DIR. + +set -euf -o pipefail + +INTEGRATION_SHA=06ae16d18bd8e4db9a3fc062f678a170025d9f1a +PICK_REFS=( + refs/changes/14/948614/1 + refs/changes/33/943833/20 +) + +checkout=fuchsia +jiri=.jiri_root/bin/jiri + +set -x + +# This script will: +# - create a directory named "fuchsia" if it does not exist +# - download "jiri" to "fuchsia/.jiri_root/bin" +curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" \ + | base64 --decode \ + | bash -s $checkout + +cd $checkout + +$jiri init \ + -partial=true \ + -analytics-opt=false \ + . + +$jiri import \ + -name=integration \ + -revision=$INTEGRATION_SHA \ + -overwrite=true \ + flower \ + "https://fuchsia.googlesource.com/integration" + +if [ -d ".git" ]; then + # Wipe out any local changes if we're reusing a checkout. + git checkout --force JIRI_HEAD +fi + +$jiri update -autoupdate=false + +echo integration commit = $(git -C integration rev-parse HEAD) + +for git_ref in "${PICK_REFS[@]}"; do + git fetch https://fuchsia.googlesource.com/fuchsia $git_ref + git cherry-pick --no-commit FETCH_HEAD +done + +bash scripts/rust/build_fuchsia_from_rust_ci.sh diff --git a/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh b/src/ci/docker/scripts/build-fuchsia-toolchain.sh similarity index 85% rename from src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh rename to src/ci/docker/scripts/build-fuchsia-toolchain.sh index d762b4672c62e..beea2f522fde9 100755 --- a/src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh +++ b/src/ci/docker/scripts/build-fuchsia-toolchain.sh @@ -4,13 +4,13 @@ set -ex source shared.sh FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64 -FUCHSIA_SDK_ID=4xjxrGUrDbQ6_zJwj6cDN1IbWsWV5aCQXC_zO_Hu0XkC -FUCHSIA_SDK_SHA256=e318f1ac652b0db43aff32708fa70337521b5ac595e5a0905c2ff33bf1eed179 +FUCHSIA_SDK_ID=MrhQwtmP8CpZre-i_PNOREcThbUcrX3bA-45d6WQr-cC +FUCHSIA_SDK_SHA256=32b850c2d98ff02a59adefa2fcf34e44471385b51cad7ddb03ee3977a590afe7 FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk CLANG_DOWNLOAD_URL=\ https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64 -CLANG_DOWNLOAD_ID=vU0vNjSihOV4Q6taQYCpy03JXGiCyVwxen3rFMNMIgsC -CLANG_DOWNLOAD_SHA256=bd4d2f3634a284e57843ab5a4180a9cb4dc95c6882c95c317a7deb14c34c220b +CLANG_DOWNLOAD_ID=Tpc85d1ZwSlZ6UKl2d96GRUBGNA5JKholOKe24sRDr0C +CLANG_DOWNLOAD_SHA256=4e973ce5dd59c12959e942a5d9df7a19150118d03924a86894e29edb8b110ebd install_clang() { mkdir -p clang_download diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index da29ffb8e5f9b..f8412e7f32636 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -470,6 +470,9 @@ jobs: - name: x86_64-gnu-aux <<: *job-linux-4c + - name: x86_64-gnu-integration + <<: *job-linux-16c + - name: x86_64-gnu-debug <<: *job-linux-8c @@ -735,6 +738,7 @@ jobs: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c + master: name: master runs-on: ubuntu-latest diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 8a88c56194c60..720394af249b2 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -8,7 +8,7 @@ export MIRRORS_BASE="https://ci-mirrors.rust-lang.org/rustc" # See https://unix.stackexchange.com/questions/82598 -# Duplicated in docker/dist-various-2/shared.sh +# Duplicated in docker/scripts/shared.sh function retry { echo "Attempting with retry:" "$@" local n=1 From 44b2ead5fad1ec4bb8f49962299545fa0c6e84c7 Mon Sep 17 00:00:00 2001 From: Tyler Mandry Date: Fri, 17 Nov 2023 12:13:41 -0800 Subject: [PATCH 2/2] Add try builders --- .github/workflows/ci.yml | 6 ++++++ src/ci/github-actions/ci.yml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b59a46d9f071d..5cc658ae04c46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -586,6 +586,12 @@ jobs: env: CODEGEN_BACKENDS: "llvm,cranelift" os: ubuntu-20.04-16core-64gb + - name: x86_64-gnu-integration + os: ubuntu-20.04-16core-64gb + env: {} + - name: dist-various-2 + os: ubuntu-20.04-8core-32gb + env: {} timeout-minutes: 600 runs-on: "${{ matrix.os }}" steps: diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index f8412e7f32636..073f51330fdb8 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -737,6 +737,10 @@ jobs: env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c + - name: x86_64-gnu-integration + <<: *job-linux-16c + - name: dist-various-2 + <<: *job-linux-8c master: