From a68d5314fbd5d6dece2c09d71cc8fea179884c7c Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sat, 23 Nov 2019 12:43:21 -0500 Subject: [PATCH 1/2] Add toolstate checking into bootstrap This is not yet actually used by CI, but implements the logic for checking that tools are properly building on beta/stable and during beta cutoff week. This attempts to mirror the checking functionality in src/ci/docker/x86_64-gnu-tools/checktools.sh, and called scripts. It does not attempt to run the relevant steps (that functionality was originally desired to be moved into bootstrap as well, but doing so proved more difficult than expected). This is intended as a way to centralize and make clearer the logic involved in toolstate checking. In particular, the previous logic was spread across numerous python and shell scripts in such a way that made interpretation quite difficult. --- src/bootstrap/builder.rs | 1 + src/bootstrap/lib.rs | 27 --- src/bootstrap/toolstate.rs | 405 +++++++++++++++++++++++++++++++++++++ 3 files changed, 406 insertions(+), 27 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 99b8ddf7db1f0..5c0b43c1d24e1 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -368,6 +368,7 @@ impl<'a> Builder<'a> { check::Rustdoc ), Kind::Test => describe!( + crate::toolstate::ToolStateCheck, test::Tidy, test::Ui, test::CompileFail, diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 7ea2bb126a641..1f4a4f923e048 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -169,7 +169,6 @@ mod job { pub use crate::config::Config; use crate::flags::Subcommand; use crate::cache::{Interned, INTERNER}; -use crate::toolstate::ToolState; const LLVM_TOOLS: &[&str] = &[ "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility @@ -1074,32 +1073,6 @@ impl Build { } } - /// Updates the actual toolstate of a tool. - /// - /// The toolstates are saved to the file specified by the key - /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be - /// done. The file is updated immediately after this function completes. - pub fn save_toolstate(&self, tool: &str, state: ToolState) { - if let Some(ref path) = self.config.save_toolstates { - if let Some(parent) = path.parent() { - // Ensure the parent directory always exists - t!(std::fs::create_dir_all(parent)); - } - let mut file = t!(fs::OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(path)); - - let mut current_toolstates: HashMap, ToolState> = - serde_json::from_reader(&mut file).unwrap_or_default(); - current_toolstates.insert(tool.into(), state); - t!(file.seek(SeekFrom::Start(0))); - t!(file.set_len(0)); - t!(serde_json::to_writer(file, ¤t_toolstates)); - } - } - fn in_tree_crates(&self, root: &str) -> Vec<&Crate> { let mut ret = Vec::new(); let mut list = vec![INTERNER.intern_str(root)]; diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs index e86209be91fe1..bec28534cc29e 100644 --- a/src/bootstrap/toolstate.rs +++ b/src/bootstrap/toolstate.rs @@ -1,4 +1,28 @@ use serde::{Deserialize, Serialize}; +use build_helper::t; +use std::time; +use std::fs; +use std::io::{Seek, SeekFrom}; +use std::collections::HashMap; +use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use std::fmt; +use std::process::Command; +use std::path::PathBuf; +use std::env; + +// Each cycle is 42 days long (6 weeks); the last week is 35..=42 then. +const BETA_WEEK_START: u64 = 35; + +#[cfg(linux)] +const OS: Option<&str> = Some("linux"); + +#[cfg(windows)] +const OS: Option<&str> = Some("windows"); + +#[cfg(all(not(linux), not(windows)))] +const OS: Option<&str> = None; + +type ToolstateData = HashMap, ToolState>; #[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] @@ -12,9 +36,390 @@ pub enum ToolState { BuildFail = 0, } +impl fmt::Display for ToolState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", match self { + ToolState::TestFail => "test-fail", + ToolState::TestPass => "test-pass", + ToolState::BuildFail => "build-fail", + }) + } +} + impl Default for ToolState { fn default() -> Self { // err on the safe side ToolState::BuildFail } } + +/// Number of days after the last promotion of beta. +/// Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. +/// The Wednesday after this has value 0. +/// We track this value to prevent regressing tools in the last week of the 6-week cycle. +fn days_since_beta_promotion() -> u64 { + let since_epoch = t!(time::SystemTime::UNIX_EPOCH.elapsed()); + (since_epoch.as_secs() / 86400 - 20) % 42 +} + +// These tools must test-pass on the beta/stable channels. +// +// On the nightly channel, their build step must be attempted, but they may not +// be able to build successfully. +static STABLE_TOOLS: &[(&str, &str)] = &[ + ("book", "src/doc/book"), + ("nomicon", "src/doc/nomicon"), + ("reference", "src/doc/reference"), + ("rust-by-example", "src/doc/rust-by-example"), + ("edition-guide", "src/doc/edition-guide"), + ("rls", "src/tools/rls"), + ("rustfmt", "src/tools/rustfmt"), + ("clippy-driver", "src/tools/clippy"), +]; + +// These tools are permitted to not build on the beta/stable channels. +// +// We do require that we checked whether they build or not on the tools builder, +// though, as otherwise we will be unable to file an issue if they start +// failing. +static NIGHTLY_TOOLS: &[(&str, &str)] = &[ + ("miri", "src/tools/miri"), + ("embedded-book", "src/doc/embedded-book"), + ("rustc-guide", "src/doc/rustc-guide"), +]; + +fn print_error(tool: &str, submodule: &str) { + eprintln!(""); + eprintln!("We detected that this PR updated '{}', but its tests failed.", tool); + eprintln!(""); + eprintln!("If you do intend to update '{}', please check the error messages above and", tool); + eprintln!("commit another update."); + eprintln!(""); + eprintln!("If you do NOT intend to update '{}', please ensure you did not accidentally", tool); + eprintln!("change the submodule at '{}'. You may ask your reviewer for the", submodule); + eprintln!("proper steps."); + std::process::exit(3); +} + +fn check_changed_files(toolstates: &HashMap, ToolState>) { + // Changed files + let output = std::process::Command::new("git") + .arg("diff") + .arg("--name-status") + .arg("HEAD") + .arg("HEAD^") + .output(); + let output = match output { + Ok(o) => o, + Err(e) => { + eprintln!("Failed to get changed files: {:?}", e); + std::process::exit(1); + } + }; + + let output = t!(String::from_utf8(output.stdout)); + + for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { + let changed = output.lines().any(|l| { + l.starts_with("M") && l.ends_with(submodule) + }); + eprintln!("Verifying status of {}...", tool); + if !changed { + continue; + } + + eprintln!("This PR updated '{}', verifying if status is 'test-pass'...", submodule); + if toolstates[*tool] != ToolState::TestPass { + print_error(tool, submodule); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ToolStateCheck; + +impl Step for ToolStateCheck { + type Output = (); + + /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. + /// + /// This tool in `src/tools` will verify the validity of all our links in the + /// documentation to ensure we don't have a bunch of dead ones. + fn run(self, builder: &Builder<'_>) { + if builder.config.dry_run { + return; + } + + let days_since_beta_promotion = days_since_beta_promotion(); + let in_beta_week = days_since_beta_promotion >= BETA_WEEK_START; + let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable"); + let toolstates = builder.toolstates(); + + let mut did_error = false; + + for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { + if !toolstates.contains_key(*tool) { + did_error = true; + eprintln!("error: Tool `{}` was not recorded in tool state.", tool); + } + } + + if did_error { + std::process::exit(1); + } + + check_changed_files(&toolstates); + + for (tool, _) in STABLE_TOOLS.iter() { + let state = toolstates[*tool]; + + if state != ToolState::TestPass { + if !is_nightly { + did_error = true; + eprintln!("error: Tool `{}` should be test-pass but is {}", tool, state); + } else if in_beta_week { + did_error = true; + eprintln!("error: Tool `{}` should be test-pass but is {} during beta week.", + tool, state); + } + } + } + + if did_error { + std::process::exit(1); + } + + if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() { + commit_toolstate_change(&toolstates, in_beta_week); + } + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("check-tools") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ToolStateCheck); + } +} + +impl Builder<'_> { + fn toolstates(&self) -> HashMap, ToolState> { + if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } + let mut file = t!(fs::OpenOptions::new() + .create(true) + .write(true) + .read(true) + .open(path)); + + serde_json::from_reader(&mut file).unwrap_or_default() + } else { + Default::default() + } + } + + /// Updates the actual toolstate of a tool. + /// + /// The toolstates are saved to the file specified by the key + /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be + /// done. The file is updated immediately after this function completes. + pub fn save_toolstate(&self, tool: &str, state: ToolState) { + if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } + let mut file = t!(fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(path)); + + let mut current_toolstates: HashMap, ToolState> = + serde_json::from_reader(&mut file).unwrap_or_default(); + current_toolstates.insert(tool.into(), state); + t!(file.seek(SeekFrom::Start(0))); + t!(file.set_len(0)); + t!(serde_json::to_writer(file, ¤t_toolstates)); + } + } +} + +/// This function `commit_toolstate_change` provides functionality for pushing a change +/// to the `rust-toolstate` repository. +/// +/// The function relies on a GitHub bot user, which should have a Personal access +/// token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for +/// some reason you need to change the token, please update the Azure Pipelines +/// variable group. +/// +/// 1. Generate a new Personal access token: +/// +/// * Login to the bot account, and go to Settings -> Developer settings -> +/// Personal access tokens +/// * Click "Generate new token" +/// * Enable the "public_repo" permission, then click "Generate token" +/// * Copy the generated token (should be a 40-digit hexadecimal number). +/// Save it somewhere secure, as the token would be gone once you leave +/// the page. +/// +/// 2. Update the variable group in Azure Pipelines +/// +/// * Ping a member of the infrastructure team to do this. +/// +/// 4. Replace the email address below if the bot account identity is changed +/// +/// * See +/// if a private email by GitHub is wanted. +fn commit_toolstate_change( + current_toolstate: &ToolstateData, + in_beta_week: bool, +) { + fn git_config(key: &str, value: &str) { + let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status(); + let success = match status { + Ok(s) => s.success(), + Err(_) => false, + }; + if !success { + panic!("git config key={} value={} successful (status: {:?})", key, value, status); + } + } + + git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com"); + git_config("user.name", "Rust Toolstate Update"); + git_config("credential.helper", "store"); + + let credential = format!( + "https://{}:x-oauth-basic@github.com\n", + t!(env::var("TOOLSTATE_REPO_ACCESS_TOKEN")), + ); + let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials"); + t!(fs::write(&git_credential_path, credential)); + + let status = Command::new("git").arg("clone") + .arg("--depth=1") + .arg(t!(env::var("TOOLSTATE_REPO"))) + .status(); + let success = match status { + Ok(s) => s.success(), + Err(_) => false, + }; + if !success { + panic!("git clone successful (status: {:?})", status); + } + + let old_toolstate = t!(fs::read("rust-toolstate/_data/latest.json")); + let old_toolstate: Vec = t!(serde_json::from_slice(&old_toolstate)); + + let message = format!("({} CI update)", OS.expect("linux/windows only")); + let mut success = false; + for _ in 1..=5 { + // Update the toolstate results (the new commit-to-toolstate mapping) in the toolstate repo. + change_toolstate(¤t_toolstate, &old_toolstate, in_beta_week); + + // `git commit` failing means nothing to commit. + let status = t!(Command::new("git") + .current_dir("rust-toolstate") + .arg("commit") + .arg("-a") + .arg("-m") + .arg(&message) + .status()); + if !status.success() { + success = true; + break; + } + + let status = t!(Command::new("git") + .current_dir("rust-toolstate") + .arg("push") + .arg("origin") + .arg("master") + .status()); + // If we successfully push, exit. + if status.success() { + success = true; + break; + } + eprintln!("Sleeping for 3 seconds before retrying push"); + std::thread::sleep(std::time::Duration::from_secs(3)); + let status = t!(Command::new("git") + .current_dir("rust-toolstate") + .arg("fetch") + .arg("origin") + .arg("master") + .status()); + assert!(status.success()); + let status = t!(Command::new("git") + .current_dir("rust-toolstate") + .arg("reset") + .arg("--hard") + .arg("origin/master") + .status()); + assert!(status.success()); + } + + if !success { + panic!("Failed to update toolstate repository with new data"); + } +} + +fn change_toolstate( + current_toolstate: &ToolstateData, + old_toolstate: &[RepoState], + in_beta_week: bool, +) { + let mut regressed = false; + for repo_state in old_toolstate { + let tool = &repo_state.tool; + let state = if cfg!(linux) { + &repo_state.linux + } else if cfg!(windows) { + &repo_state.windows + } else { + unimplemented!() + }; + let new_state = current_toolstate[tool.as_str()]; + + if new_state != *state { + eprintln!("The state of `{}` has changed from `{}` to `{}`", tool, state, new_state); + if (new_state as u8) < (*state as u8) { + if !["rustc-guide", "miri", "embedded-book"].contains(&tool.as_str()) { + regressed = true; + } + } + } + } + + if regressed && in_beta_week { + std::process::exit(1); + } + + let commit = t!(std::process::Command::new("git") + .arg("rev-parse") + .arg("HEAD") + .output()); + let commit = t!(String::from_utf8(commit.stdout)); + + let toolstate_serialized = t!(serde_json::to_string(¤t_toolstate)); + + let history_path = format!("rust-toolstate/history/{}.tsv", OS.expect("linux/windows only")); + let mut file = t!(fs::read_to_string(&history_path)); + let end_of_first_line = file.find('\n').unwrap(); + file.insert_str(end_of_first_line, &format!("{}\t{}\n", commit, toolstate_serialized)); + t!(fs::write(&history_path, file)); +} + +#[derive(Debug, Serialize, Deserialize)] +struct RepoState { + tool: String, + windows: ToolState, + linux: ToolState, + commit: String, + datetime: String, +} From 97d936423c914c4e3402bfecfd6943e1edf23815 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sat, 23 Nov 2019 18:08:49 -0500 Subject: [PATCH 2/2] Move CI to new builtin check-toolstate --- src/bootstrap/toolstate.rs | 2 + src/ci/azure-pipelines/auto.yml | 2 +- src/ci/azure-pipelines/master.yml | 5 +- src/ci/docker/x86_64-gnu-tools/Dockerfile | 4 +- .../x86_64-gnu-tools/checkregression.py | 48 -------- src/ci/docker/x86_64-gnu-tools/checktools.sh | 116 +----------------- src/ci/docker/x86_64-gnu-tools/repo.sh | 68 ---------- src/ci/publish_toolstate.sh | 33 +++++ 8 files changed, 39 insertions(+), 239 deletions(-) delete mode 100755 src/ci/docker/x86_64-gnu-tools/checkregression.py delete mode 100644 src/ci/docker/x86_64-gnu-tools/repo.sh create mode 100644 src/ci/publish_toolstate.sh diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs index bec28534cc29e..a6d9ef38e57b1 100644 --- a/src/bootstrap/toolstate.rs +++ b/src/bootstrap/toolstate.rs @@ -290,6 +290,8 @@ fn commit_toolstate_change( } } + // If changing anything here, then please check that src/ci/publish_toolstate.sh is up to date + // as well. git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com"); git_config("user.name", "Rust Toolstate Update"); git_config("credential.helper", "store"); diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 70d6bad297dce..5248e41725836 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -163,7 +163,7 @@ jobs: NO_LLVM_ASSERTIONS: 1 # MSVC tools tests x86_64-msvc-tools: - SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json # 32/64-bit MinGW builds. diff --git a/src/ci/azure-pipelines/master.yml b/src/ci/azure-pipelines/master.yml index e2baa923d99f7..9c5a15a3cf4ed 100644 --- a/src/ci/azure-pipelines/master.yml +++ b/src/ci/azure-pipelines/master.yml @@ -16,10 +16,7 @@ steps: - checkout: self fetchDepth: 2 -- script: | - export MESSAGE_FILE=$(mktemp -t msg.XXXXXX) - . src/ci/docker/x86_64-gnu-tools/repo.sh - commit_toolstate_change "$MESSAGE_FILE" "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN" +- script: src/ci/publish_toolstate.sh displayName: Publish toolstate env: TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN) diff --git a/src/ci/docker/x86_64-gnu-tools/Dockerfile b/src/ci/docker/x86_64-gnu-tools/Dockerfile index 7687a6ca23e18..51193402ee842 100644 --- a/src/ci/docker/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/x86_64-gnu-tools/Dockerfile @@ -17,9 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -COPY x86_64-gnu-tools/checkregression.py /tmp/ COPY x86_64-gnu-tools/checktools.sh /tmp/ -COPY x86_64-gnu-tools/repo.sh /tmp/ # Run rustbook with `linkcheck` feature enabled ENV CHECK_LINKS 1 @@ -27,4 +25,4 @@ ENV CHECK_LINKS 1 ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --save-toolstates=/tmp/toolstate/toolstates.json -ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstate/toolstates.json linux +ENV SCRIPT /tmp/checktools.sh ../x.py diff --git a/src/ci/docker/x86_64-gnu-tools/checkregression.py b/src/ci/docker/x86_64-gnu-tools/checkregression.py deleted file mode 100755 index 4fbb8c4d20349..0000000000000 --- a/src/ci/docker/x86_64-gnu-tools/checkregression.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -## This script has two purposes: detect any tool that *regressed*, which is used -## during the week before the beta branches to reject PRs; and detect any tool -## that *changed* to see if we need to update the toolstate repo. - -import sys -import json - -# Regressions for these tools during the beta cutoff week do not cause failure. -# See `status_check` in `checktools.sh` for tools that have to pass on the -# beta/stable branches. -REGRESSION_OK = ["rustc-guide", "miri", "embedded-book"] - -if __name__ == '__main__': - os_name = sys.argv[1] - toolstate_file = sys.argv[2] - current_state = sys.argv[3] - verb = sys.argv[4] # 'regressed' or 'changed' - - with open(toolstate_file, 'r') as f: - toolstate = json.load(f) - with open(current_state, 'r') as f: - current = json.load(f) - - regressed = False - for cur in current: - tool = cur['tool'] - state = cur[os_name] - new_state = toolstate.get(tool, '') - if verb == 'regressed': - updated = new_state < state - elif verb == 'changed': - updated = new_state != state - else: - print('Unknown verb {}'.format(updated)) - sys.exit(2) - if updated: - print( - 'The state of "{}" has {} from "{}" to "{}"' - .format(tool, verb, state, new_state) - ) - if not (verb == 'regressed' and tool in REGRESSION_OK): - regressed = True - - if regressed: - sys.exit(1) diff --git a/src/ci/docker/x86_64-gnu-tools/checktools.sh b/src/ci/docker/x86_64-gnu-tools/checktools.sh index ebb8c0bda53ee..e57fe2271398f 100755 --- a/src/ci/docker/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/x86_64-gnu-tools/checktools.sh @@ -3,18 +3,6 @@ set -eu X_PY="$1" -TOOLSTATE_FILE="$(realpath -m $2)" -OS="$3" -COMMIT="$(git rev-parse HEAD)" -CHANGED_FILES="$(git diff --name-status HEAD HEAD^)" -SIX_WEEK_CYCLE="$(( ($(date +%s) / 86400 - 20) % 42 ))" -# ^ Number of days after the last promotion of beta. -# Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. -# The Wednesday after this has value 0. -# We track this value to prevent regressing tools in the last week of the 6-week cycle. - -mkdir -p "$(dirname $TOOLSTATE_FILE)" -touch "$TOOLSTATE_FILE" # Try to test all the tools and store the build/test success in the TOOLSTATE_FILE @@ -34,106 +22,4 @@ python2.7 "$X_PY" test --no-fail-fast \ set -e -cat "$TOOLSTATE_FILE" -echo - -# This function checks if a particular tool is *not* in status "test-pass". -check_tool_failed() { - grep -vq '"'"$1"'":"test-pass"' "$TOOLSTATE_FILE" -} - -# This function checks that if a tool's submodule changed, the tool's state must improve -verify_submodule_changed() { - echo "Verifying status of $1..." - if echo "$CHANGED_FILES" | grep -q "^M[[:blank:]]$2$"; then - echo "This PR updated '$2', verifying if status is 'test-pass'..." - if check_tool_failed "$1"; then - echo - echo "⚠️ We detected that this PR updated '$1', but its tests failed." - echo - echo "If you do intend to update '$1', please check the error messages above and" - echo "commit another update." - echo - echo "If you do NOT intend to update '$1', please ensure you did not accidentally" - echo "change the submodule at '$2'. You may ask your reviewer for the" - echo "proper steps." - exit 3 - fi - fi -} - -# deduplicates the submodule check and the assertion that on beta some tools MUST be passing. -# $1 should be "submodule_changed" to only check tools that got changed by this PR, -# or "beta_required" to check all tools that have $2 set to "beta". -check_dispatch() { - if [ "$1" = submodule_changed ]; then - # ignore $2 (branch id) - verify_submodule_changed $3 $4 - elif [ "$2" = beta ]; then - echo "Requiring test passing for $3..." - if check_tool_failed "$3"; then - exit 4 - fi - fi -} - -# List all tools here. -# This function gets called with "submodule_changed" for each PR that changed a submodule, -# and with "beta_required" for each PR that lands on beta/stable. -# The purpose of this function is to *reject* PRs if a tool is not "test-pass" and -# (a) the tool's submodule has been updated, or (b) we landed on beta/stable and the -# tool has to "test-pass" on that branch. -status_check() { - check_dispatch $1 beta book src/doc/book - check_dispatch $1 beta nomicon src/doc/nomicon - check_dispatch $1 beta reference src/doc/reference - check_dispatch $1 beta rust-by-example src/doc/rust-by-example - check_dispatch $1 beta edition-guide src/doc/edition-guide - check_dispatch $1 beta rls src/tools/rls - check_dispatch $1 beta rustfmt src/tools/rustfmt - check_dispatch $1 beta clippy-driver src/tools/clippy - # These tools are not required on the beta/stable branches, but they *do* cause - # PRs to fail if a submodule update does not fix them. - # They will still cause failure during the beta cutoff week, unless `checkregression.py` - # exempts them from that. - check_dispatch $1 nightly miri src/tools/miri - check_dispatch $1 nightly embedded-book src/doc/embedded-book - check_dispatch $1 nightly rustc-guide src/doc/rustc-guide -} - -# If this PR is intended to update one of these tools, do not let the build pass -# when they do not test-pass. - -status_check "submodule_changed" - -CHECK_NOT="$(readlink -f "$(dirname $0)/checkregression.py")" -# This callback is called by `commit_toolstate_change`, see `repo.sh`. -change_toolstate() { - # only update the history - if python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" changed; then - echo 'Toolstate is not changed. Not updating.' - else - if [ $SIX_WEEK_CYCLE -ge 35 ]; then - # Reject any regressions during the week before beta cutoff. - python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" regressed - fi - sed -i "1 a\\ -$COMMIT\t$(cat "$TOOLSTATE_FILE") -" "history/$OS.tsv" - fi -} - -if [ "$RUST_RELEASE_CHANNEL" = nightly ]; then - if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then - . "$(dirname $0)/repo.sh" - MESSAGE_FILE=$(mktemp -t msg.XXXXXX) - echo "($OS CI update)" > "$MESSAGE_FILE" - commit_toolstate_change "$MESSAGE_FILE" change_toolstate - rm -f "$MESSAGE_FILE" - fi - exit 0 -fi - -# abort compilation if an important tool doesn't build -# (this code is reachable if not on the nightly channel) -status_check "beta_required" +python2.7 "$X_PY" test check-tools diff --git a/src/ci/docker/x86_64-gnu-tools/repo.sh b/src/ci/docker/x86_64-gnu-tools/repo.sh deleted file mode 100644 index 82700a00fb6af..0000000000000 --- a/src/ci/docker/x86_64-gnu-tools/repo.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh - -# This file provides the function `commit_toolstate_change` for pushing a change -# to the `rust-toolstate` repository. -# -# The function relies on a GitHub bot user, which should have a Personal access -# token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for -# some reason you need to change the token, please update the Azure Pipelines -# variable group. -# -# 1. Generate a new Personal access token: -# -# * Login to the bot account, and go to Settings -> Developer settings -> -# Personal access tokens -# * Click "Generate new token" -# * Enable the "public_repo" permission, then click "Generate token" -# * Copy the generated token (should be a 40-digit hexadecimal number). -# Save it somewhere secure, as the token would be gone once you leave -# the page. -# -# 2. Update the variable group in Azure Pipelines -# -# * Ping a member of the infrastructure team to do this. -# -# 4. Replace the email address below if the bot account identity is changed -# -# * See -# if a private email by GitHub is wanted. - -commit_toolstate_change() { - OLDFLAGS="$-" - set -eu - - git config --global user.email '7378925+rust-toolstate-update@users.noreply.github.com' - git config --global user.name 'Rust Toolstate Update' - git config --global credential.helper store - printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \ - > "$HOME/.git-credentials" - git clone --depth=1 $TOOLSTATE_REPO - - cd rust-toolstate - FAILURE=1 - MESSAGE_FILE="$1" - shift - for RETRY_COUNT in 1 2 3 4 5; do - # Call the callback. - # - If we are in the `auto` branch (pre-landing), this is called from `checktools.sh` and - # the callback is `change_toolstate` in that file. The purpose of this is to publish the - # test results (the new commit-to-toolstate mapping) in the toolstate repo. - # - If we are in the `master` branch (post-landing), this is called by the CI pipeline - # and the callback is `src/tools/publish_toolstate.py`. The purpose is to publish - # the new "current" toolstate in the toolstate repo. - "$@" - # `git commit` failing means nothing to commit. - FAILURE=0 - git commit -a -F "$MESSAGE_FILE" || break - # On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering. - git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1) - FAILURE=1 - git fetch origin master - git reset --hard origin/master - done - cd .. - - set +eu - set "-$OLDFLAGS" - return $FAILURE -} diff --git a/src/ci/publish_toolstate.sh b/src/ci/publish_toolstate.sh new file mode 100644 index 0000000000000..d8ff74078220e --- /dev/null +++ b/src/ci/publish_toolstate.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +set -eu + +# The following lines are also found in src/bootstrap/toolstate.rs, +# so if updating here, please also update that file. + +export MESSAGE_FILE=$(mktemp -t msg.XXXXXX) + +git config --global user.email '7378925+rust-toolstate-update@users.noreply.github.com' +git config --global user.name 'Rust Toolstate Update' +git config --global credential.helper store +printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \ + > "$HOME/.git-credentials" +git clone --depth=1 $TOOLSTATE_REPO + +cd rust-toolstate +FAILURE=1 +for RETRY_COUNT in 1 2 3 4 5; do + # The purpose is to publish the new "current" toolstate in the toolstate repo. + "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" \ + "$(git log --format=%s -n1 HEAD)" \ + "$MESSAGE_FILE" \ + "$TOOLSTATE_REPO_ACCESS_TOKEN" + # `git commit` failing means nothing to commit. + FAILURE=0 + git commit -a -F "$MESSAGE_FILE" || break + # On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering. + git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1) + FAILURE=1 + git fetch origin master + git reset --hard origin/master +done