From 8f3844ff07449ebd045ba8fd9adcac62e957cfe8 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 19 Jul 2021 12:25:29 +0200 Subject: [PATCH 1/9] refactor adding rustc and std into extended builds --- src/bootstrap/dist.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 92853378e5881..1d7dbeddee981 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1345,7 +1345,15 @@ impl Step for Extended { builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target)); - let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) }); + let mut tarballs = Vec::new(); + + // When rust-std package split from rustc, we needed to ensure that during + // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering + // the std files during uninstall. To do this ensure that rustc comes + // before rust-std in the list below. + tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) })); + tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std")); + let cargo_installer = builder.ensure(Cargo { compiler, target }); let rustfmt_installer = builder.ensure(Rustfmt { compiler, target }); let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target }); @@ -1358,7 +1366,6 @@ impl Step for Extended { let analysis_installer = builder.ensure(Analysis { compiler, target }); let docs_installer = builder.ensure(Docs { host: target }); - let std_installer = builder.ensure(Std { compiler, target }); let etc = builder.src.join("src/etc/installer"); @@ -1367,12 +1374,6 @@ impl Step for Extended { return; } - // When rust-std package split from rustc, we needed to ensure that during - // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering - // the std files during uninstall. To do this ensure that rustc comes - // before rust-std in the list below. - let mut tarballs = Vec::new(); - tarballs.push(rustc_installer); tarballs.push(cargo_installer); tarballs.push(clippy_installer); tarballs.extend(rust_demangler_installer.clone()); @@ -1384,7 +1385,6 @@ impl Step for Extended { if let Some(analysis_installer) = analysis_installer { tarballs.push(analysis_installer); } - tarballs.push(std_installer.expect("missing std")); if let Some(docs_installer) = docs_installer { tarballs.push(docs_installer); } From 0c8baa729dc3bb61713fdc198062e721c1ffa7c1 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 19 Jul 2021 13:06:24 +0200 Subject: [PATCH 2/9] refactor gating of dist docs --- src/bootstrap/dist.rs | 21 +++++++++------------ src/bootstrap/install.rs | 8 ++------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 1d7dbeddee981..b0764791907b3 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -51,11 +51,12 @@ pub struct Docs { } impl Step for Docs { - type Output = Option; + type Output = GeneratedTarball; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/doc") + let default = run.builder.config.docs; + run.path("src/doc").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -63,11 +64,8 @@ impl Step for Docs { } /// Builds the `rust-docs` installer component. - fn run(self, builder: &Builder<'_>) -> Option { + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { let host = self.host; - if !builder.config.docs { - return None; - } builder.default_doc(&[]); let dest = "share/doc/rust/html"; @@ -76,7 +74,7 @@ impl Step for Docs { tarball.set_product_name("Rust Documentation"); tarball.add_bulk_dir(&builder.doc_out(host), dest); tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); - Some(tarball.generate()) + tarball.generate() } } @@ -1354,6 +1352,10 @@ impl Step for Extended { tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) })); tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std")); + if builder.config.docs { + tarballs.push(builder.ensure(Docs { host: target })); + } + let cargo_installer = builder.ensure(Cargo { compiler, target }); let rustfmt_installer = builder.ensure(Rustfmt { compiler, target }); let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target }); @@ -1365,8 +1367,6 @@ impl Step for Extended { let mingw_installer = builder.ensure(Mingw { host: target }); let analysis_installer = builder.ensure(Analysis { compiler, target }); - let docs_installer = builder.ensure(Docs { host: target }); - let etc = builder.src.join("src/etc/installer"); // Avoid producing tarballs during a dry run. @@ -1385,9 +1385,6 @@ impl Step for Extended { if let Some(analysis_installer) = analysis_installer { tarballs.push(analysis_installer); } - if let Some(docs_installer) = docs_installer { - tarballs.push(docs_installer); - } if target.contains("pc-windows-gnu") { tarballs.push(mingw_installer.unwrap()); } diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index 8a1b6df0dafe3..1eb516fb77125 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -139,12 +139,8 @@ macro_rules! install { install!((self, builder, _config), Docs, "src/doc", _config.docs, only_hosts: false, { - if let Some(tarball) = builder.ensure(dist::Docs { host: self.target }) { - install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); - } else { - panic!("docs are not available to install, \ - check that `build.docs` is true in `config.toml`"); - } + let tarball = builder.ensure(dist::Docs { host: self.target }); + install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); }; Std, "library/std", true, only_hosts: false, { for target in &builder.targets { From 7167f8b83f8dc441541a5bfeb087c5b25f787a84 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 19 Jul 2021 13:42:54 +0200 Subject: [PATCH 3/9] change output of dist cargo and clippy to be consistent with other tools --- src/bootstrap/dist.rs | 16 ++++++++-------- src/bootstrap/install.rs | 8 ++++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index b0764791907b3..62ca8bff405af 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -951,7 +951,7 @@ pub struct Cargo { } impl Step for Cargo { - type Output = GeneratedTarball; + type Output = Option; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -969,7 +969,7 @@ impl Step for Cargo { }); } - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; @@ -994,7 +994,7 @@ impl Step for Cargo { } } - tarball.generate() + Some(tarball.generate()) } } @@ -1106,7 +1106,7 @@ pub struct Clippy { } impl Step for Clippy { - type Output = GeneratedTarball; + type Output = Option; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1124,7 +1124,7 @@ impl Step for Clippy { }); } - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); @@ -1145,7 +1145,7 @@ impl Step for Clippy { tarball.add_file(clippy, "bin", 0o755); tarball.add_file(cargoclippy, "bin", 0o755); tarball.add_legal_and_readme_to("share/doc/clippy"); - tarball.generate() + Some(tarball.generate()) } } @@ -1374,8 +1374,8 @@ impl Step for Extended { return; } - tarballs.push(cargo_installer); - tarballs.push(clippy_installer); + tarballs.extend(cargo_installer); + tarballs.extend(clippy_installer); tarballs.extend(rust_demangler_installer.clone()); tarballs.extend(rls_installer.clone()); tarballs.extend(rust_analyzer_installer.clone()); diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index 1eb516fb77125..72a9ff2d5a794 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -154,7 +154,9 @@ install!((self, builder, _config), } }; Cargo, "cargo", Self::should_build(_config), only_hosts: true, { - let tarball = builder.ensure(dist::Cargo { compiler: self.compiler, target: self.target }); + let tarball = builder + .ensure(dist::Cargo { compiler: self.compiler, target: self.target }) + .expect("missing cargo"); install_sh(builder, "cargo", self.compiler.stage, Some(self.target), &tarball); }; Rls, "rls", Self::should_build(_config), only_hosts: true, { @@ -178,7 +180,9 @@ install!((self, builder, _config), } }; Clippy, "clippy", Self::should_build(_config), only_hosts: true, { - let tarball = builder.ensure(dist::Clippy { compiler: self.compiler, target: self.target }); + let tarball = builder + .ensure(dist::Clippy { compiler: self.compiler, target: self.target }) + .expect("missing clippy"); install_sh(builder, "clippy", self.compiler.stage, Some(self.target), &tarball); }; Miri, "miri", Self::should_build(_config), only_hosts: true, { From e60b348b0d0122d9522df0d84f47c64ebea2e4c1 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 19 Jul 2021 15:09:30 +0200 Subject: [PATCH 4/9] refactor gating of tools --- src/bootstrap/dist.rs | 149 +++++++++++++++++++++--------------------- 1 file changed, 75 insertions(+), 74 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 62ca8bff405af..2237f5e3e0cb5 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -8,6 +8,7 @@ //! out to `rust-installer` still. This may one day be replaced with bits and //! pieces of `rustup.rs`! +use std::collections::HashSet; use std::env; use std::fs; use std::path::{Path, PathBuf}; @@ -45,6 +46,18 @@ fn missing_tool(tool_name: &str, skip: bool) { } } +fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool { + if !builder.config.extended { + return false; + } + + if let Some(tools) = &builder.config.tools { + tools.is_empty() || tools.contains(tool) + } else { + true + } +} + #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] pub struct Docs { pub host: TargetSelection, @@ -671,11 +684,10 @@ pub struct Analysis { impl Step for Analysis { type Output = Option; - const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("analysis").default_condition(builder.config.extended) + let default = should_build_extended_tool(&run.builder, "analysis"); + run.path("analysis").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -696,7 +708,6 @@ impl Step for Analysis { fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; - assert!(builder.config.extended); if compiler.host != builder.config.build { return None; } @@ -955,7 +966,8 @@ impl Step for Cargo { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("cargo") + let default = should_build_extended_tool(&run.builder, "cargo"); + run.path("cargo").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -1009,7 +1021,8 @@ impl Step for Rls { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("rls") + let default = should_build_extended_tool(&run.builder, "rls"); + run.path("rls").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -1026,7 +1039,6 @@ impl Step for Rls { fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; - assert!(builder.config.extended); let rls = builder .ensure(tool::Rls { compiler, target, extra_features: Vec::new() }) @@ -1055,7 +1067,8 @@ impl Step for RustAnalyzer { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("rust-analyzer") + let default = should_build_extended_tool(&run.builder, "rust-analyzer"); + run.path("rust-analyzer").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -1078,7 +1091,6 @@ impl Step for RustAnalyzer { } let compiler = self.compiler; let target = self.target; - assert!(builder.config.extended); if target.contains("riscv64") { // riscv64 currently has an LLVM bug that makes rust-analyzer unable @@ -1110,7 +1122,8 @@ impl Step for Clippy { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("clippy") + let default = should_build_extended_tool(&run.builder, "clippy"); + run.path("clippy").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -1127,7 +1140,6 @@ impl Step for Clippy { fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; - assert!(builder.config.extended); // Prepare the image directory // We expect clippy to build, because we've exited this step above if tool @@ -1160,7 +1172,8 @@ impl Step for Miri { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("miri") + let default = should_build_extended_tool(&run.builder, "miri"); + run.path("miri").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -1183,7 +1196,6 @@ impl Step for Miri { } let compiler = self.compiler; let target = self.target; - assert!(builder.config.extended); let miri = builder .ensure(tool::Miri { compiler, target, extra_features: Vec::new() }) @@ -1219,7 +1231,8 @@ impl Step for Rustfmt { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("rustfmt") + let default = should_build_extended_tool(&run.builder, "rustfmt"); + run.path("rustfmt").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -1344,6 +1357,17 @@ impl Step for Extended { builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target)); let mut tarballs = Vec::new(); + let mut built_tools = HashSet::new(); + macro_rules! add_tool { + ($name:expr => $step:expr) => { + if should_build_extended_tool(builder, $name) { + if let Some(tarball) = builder.ensure($step) { + tarballs.push(tarball); + built_tools.insert($name); + } + } + }; + } // When rust-std package split from rustc, we needed to ensure that during // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering @@ -1356,16 +1380,17 @@ impl Step for Extended { tarballs.push(builder.ensure(Docs { host: target })); } - let cargo_installer = builder.ensure(Cargo { compiler, target }); - let rustfmt_installer = builder.ensure(Rustfmt { compiler, target }); + add_tool!("cargo" => Cargo { compiler, target }); + add_tool!("rustfmt" => Rustfmt { compiler, target }); + add_tool!("rls" => Rls { compiler, target }); + add_tool!("rust-analyzer" => RustAnalyzer { compiler, target }); + add_tool!("llvm-tools" => LlvmTools { target }); + add_tool!("clippy" => Clippy { compiler, target }); + add_tool!("miri" => Miri { compiler, target }); + add_tool!("analysis" => Analysis { compiler, target }); + let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target }); - let rls_installer = builder.ensure(Rls { compiler, target }); - let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target }); - let llvm_tools_installer = builder.ensure(LlvmTools { target }); - let clippy_installer = builder.ensure(Clippy { compiler, target }); - let miri_installer = builder.ensure(Miri { compiler, target }); let mingw_installer = builder.ensure(Mingw { host: target }); - let analysis_installer = builder.ensure(Analysis { compiler, target }); let etc = builder.src.join("src/etc/installer"); @@ -1374,17 +1399,8 @@ impl Step for Extended { return; } - tarballs.extend(cargo_installer); - tarballs.extend(clippy_installer); tarballs.extend(rust_demangler_installer.clone()); - tarballs.extend(rls_installer.clone()); - tarballs.extend(rust_analyzer_installer.clone()); - tarballs.extend(miri_installer.clone()); - tarballs.extend(rustfmt_installer.clone()); - tarballs.extend(llvm_tools_installer); - if let Some(analysis_installer) = analysis_installer { - tarballs.push(analysis_installer); - } + if target.contains("pc-windows-gnu") { tarballs.push(mingw_installer.unwrap()); } @@ -1434,17 +1450,11 @@ impl Step for Extended { if rust_demangler_installer.is_none() { contents = filter(&contents, "rust-demangler"); } - if rls_installer.is_none() { - contents = filter(&contents, "rls"); - } - if rust_analyzer_installer.is_none() { - contents = filter(&contents, "rust-analyzer"); - } - if miri_installer.is_none() { - contents = filter(&contents, "miri"); - } - if rustfmt_installer.is_none() { - contents = filter(&contents, "rustfmt"); + + for tool in &["rls", "rust-analyzer", "miri", "rustfmt"] { + if !built_tools.contains(tool) { + contents = filter(&contents, tool); + } } let ret = tmp.join(p.file_name().unwrap()); t!(fs::write(&ret, &contents)); @@ -1485,16 +1495,11 @@ impl Step for Extended { if rust_demangler_installer.is_some() { prepare("rust-demangler"); } - if rls_installer.is_some() { - prepare("rls"); - } - if rust_analyzer_installer.is_some() { - prepare("rust-analyzer"); - } - if miri_installer.is_some() { - prepare("miri"); + for tool in &["rls", "rust-analyzer", "miri"] { + if built_tools.contains(tool) { + prepare(tool); + } } - // create an 'uninstall' package builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755); pkgbuild("uninstall"); @@ -1554,14 +1559,10 @@ impl Step for Extended { if rust_demangler_installer.is_some() { prepare("rust-demangler"); } - if rls_installer.is_some() { - prepare("rls"); - } - if rust_analyzer_installer.is_some() { - prepare("rust-analyzer"); - } - if miri_installer.is_some() { - prepare("miri"); + for tool in &["rls", "rust-analyzer", "miri"] { + if built_tools.contains(tool) { + prepare(tool); + } } if target.contains("windows-gnu") { prepare("rust-mingw"); @@ -1640,7 +1641,7 @@ impl Step for Extended { .arg("-out") .arg(exe.join("StdGroup.wxs")), ); - if rls_installer.is_some() { + if built_tools.contains("rls") { builder.run( Command::new(&heat) .current_dir(&exe) @@ -1659,7 +1660,7 @@ impl Step for Extended { .arg(etc.join("msi/remove-duplicates.xsl")), ); } - if rust_analyzer_installer.is_some() { + if built_tools.contains("rust-analyzer") { builder.run( Command::new(&heat) .current_dir(&exe) @@ -1714,7 +1715,7 @@ impl Step for Extended { .arg(etc.join("msi/remove-duplicates.xsl")), ); } - if miri_installer.is_some() { + if built_tools.contains("miri") { builder.run( Command::new(&heat) .current_dir(&exe) @@ -1790,13 +1791,13 @@ impl Step for Extended { if rust_demangler_installer.is_some() { cmd.arg("-dRustDemanglerDir=rust-demangler"); } - if rls_installer.is_some() { + if built_tools.contains("rls") { cmd.arg("-dRlsDir=rls"); } - if rust_analyzer_installer.is_some() { + if built_tools.contains("rust-analyzer") { cmd.arg("-dRustAnalyzerDir=rust-analyzer"); } - if miri_installer.is_some() { + if built_tools.contains("miri") { cmd.arg("-dMiriDir=miri"); } if target.contains("windows-gnu") { @@ -1815,13 +1816,13 @@ impl Step for Extended { if rust_demangler_installer.is_some() { candle("RustDemanglerGroup.wxs".as_ref()); } - if rls_installer.is_some() { + if built_tools.contains("rls") { candle("RlsGroup.wxs".as_ref()); } - if rust_analyzer_installer.is_some() { + if built_tools.contains("rust-analyzer") { candle("RustAnalyzerGroup.wxs".as_ref()); } - if miri_installer.is_some() { + if built_tools.contains("miri") { candle("MiriGroup.wxs".as_ref()); } candle("AnalysisGroup.wxs".as_ref()); @@ -1855,16 +1856,16 @@ impl Step for Extended { .arg("ClippyGroup.wixobj") .current_dir(&exe); - if rls_installer.is_some() { + if built_tools.contains("rls") { cmd.arg("RlsGroup.wixobj"); } - if rust_analyzer_installer.is_some() { + if built_tools.contains("rust-analyzer") { cmd.arg("RustAnalyzerGroup.wixobj"); } if rust_demangler_installer.is_some() { cmd.arg("RustDemanglerGroup.wixobj"); } - if miri_installer.is_some() { + if built_tools.contains("miri") { cmd.arg("MiriGroup.wixobj"); } @@ -1994,7 +1995,8 @@ impl Step for LlvmTools { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("llvm-tools") + let default = should_build_extended_tool(&run.builder, "llvm-tools"); + run.path("llvm-tools").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -2003,7 +2005,6 @@ impl Step for LlvmTools { fn run(self, builder: &Builder<'_>) -> Option { let target = self.target; - assert!(builder.config.extended); /* run only if llvm-config isn't used */ if let Some(config) = builder.config.target_config.get(&target) { From 82faa9caffd766b626c216f895f083f874f2440f Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 19 Jul 2021 15:22:19 +0200 Subject: [PATCH 5/9] refactor gating of demangler --- src/bootstrap/dist.rs | 50 ++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 2237f5e3e0cb5..3b4940e59f4c5 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1284,7 +1284,13 @@ impl Step for RustDemangler { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("rust-demangler") + // While other tools use `should_build_extended_tool` to decide whether to be run by + // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like + // the other ones or if `profiler = true`. Because we don't know the target at this stage + // we run the step by default when only `extended = true`, and decide whether to actually + // run it or not later. + let default = run.builder.config.extended; + run.path("rust-demangler").default_condition(default) } fn make_run(run: RunConfig<'_>) { @@ -1301,11 +1307,11 @@ impl Step for RustDemangler { fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; - assert!(builder.config.extended); // Only build this extended tool if explicitly included in `tools`, or if `profiler = true` - let profiler = builder.config.profiler_enabled(target); - if !builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")) { + let condition = should_build_extended_tool(builder, "rust-demangler") + || builder.config.profiler_enabled(target); + if builder.config.extended && !condition { return None; } @@ -1380,6 +1386,15 @@ impl Step for Extended { tarballs.push(builder.ensure(Docs { host: target })); } + if builder.config.profiler_enabled(target) + || should_build_extended_tool(builder, "rust-demangler") + { + if let Some(tarball) = builder.ensure(RustDemangler { compiler, target }) { + tarballs.push(tarball); + built_tools.insert("rust-demangler"); + } + } + add_tool!("cargo" => Cargo { compiler, target }); add_tool!("rustfmt" => Rustfmt { compiler, target }); add_tool!("rls" => Rls { compiler, target }); @@ -1389,7 +1404,6 @@ impl Step for Extended { add_tool!("miri" => Miri { compiler, target }); add_tool!("analysis" => Analysis { compiler, target }); - let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target }); let mingw_installer = builder.ensure(Mingw { host: target }); let etc = builder.src.join("src/etc/installer"); @@ -1399,8 +1413,6 @@ impl Step for Extended { return; } - tarballs.extend(rust_demangler_installer.clone()); - if target.contains("pc-windows-gnu") { tarballs.push(mingw_installer.unwrap()); } @@ -1447,11 +1459,7 @@ impl Step for Extended { let xform = |p: &Path| { let mut contents = t!(fs::read_to_string(p)); - if rust_demangler_installer.is_none() { - contents = filter(&contents, "rust-demangler"); - } - - for tool in &["rls", "rust-analyzer", "miri", "rustfmt"] { + for tool in &["rust-demangler", "rls", "rust-analyzer", "miri", "rustfmt"] { if !built_tools.contains(tool) { contents = filter(&contents, tool); } @@ -1492,10 +1500,7 @@ impl Step for Extended { prepare("rust-std"); prepare("rust-analysis"); prepare("clippy"); - if rust_demangler_installer.is_some() { - prepare("rust-demangler"); - } - for tool in &["rls", "rust-analyzer", "miri"] { + for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] { if built_tools.contains(tool) { prepare(tool); } @@ -1556,10 +1561,7 @@ impl Step for Extended { prepare("rust-docs"); prepare("rust-std"); prepare("clippy"); - if rust_demangler_installer.is_some() { - prepare("rust-demangler"); - } - for tool in &["rls", "rust-analyzer", "miri"] { + for tool in &["rust-demangler", "rls", "rust-analyzer", "miri"] { if built_tools.contains(tool) { prepare(tool); } @@ -1696,7 +1698,7 @@ impl Step for Extended { .arg("-t") .arg(etc.join("msi/remove-duplicates.xsl")), ); - if rust_demangler_installer.is_some() { + if built_tools.contains("rust-demangler") { builder.run( Command::new(&heat) .current_dir(&exe) @@ -1788,7 +1790,7 @@ impl Step for Extended { .arg(&input); add_env(builder, &mut cmd, target); - if rust_demangler_installer.is_some() { + if built_tools.contains("rust-demangler") { cmd.arg("-dRustDemanglerDir=rust-demangler"); } if built_tools.contains("rls") { @@ -1813,7 +1815,7 @@ impl Step for Extended { candle("CargoGroup.wxs".as_ref()); candle("StdGroup.wxs".as_ref()); candle("ClippyGroup.wxs".as_ref()); - if rust_demangler_installer.is_some() { + if built_tools.contains("rust-demangler") { candle("RustDemanglerGroup.wxs".as_ref()); } if built_tools.contains("rls") { @@ -1862,7 +1864,7 @@ impl Step for Extended { if built_tools.contains("rust-analyzer") { cmd.arg("RustAnalyzerGroup.wixobj"); } - if rust_demangler_installer.is_some() { + if built_tools.contains("rust-demangler") { cmd.arg("RustDemanglerGroup.wixobj"); } if built_tools.contains("miri") { From f4c6d8f77c23aef7fc7a2128e99f8801b8176a09 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 19 Jul 2021 15:26:47 +0200 Subject: [PATCH 6/9] refactor gating of mingw --- src/bootstrap/dist.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 3b4940e59f4c5..9776b097bec07 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1386,6 +1386,10 @@ impl Step for Extended { tarballs.push(builder.ensure(Docs { host: target })); } + if target.contains("windows-gnu") { + tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw")); + } + if builder.config.profiler_enabled(target) || should_build_extended_tool(builder, "rust-demangler") { @@ -1404,8 +1408,6 @@ impl Step for Extended { add_tool!("miri" => Miri { compiler, target }); add_tool!("analysis" => Analysis { compiler, target }); - let mingw_installer = builder.ensure(Mingw { host: target }); - let etc = builder.src.join("src/etc/installer"); // Avoid producing tarballs during a dry run. @@ -1413,10 +1415,6 @@ impl Step for Extended { return; } - if target.contains("pc-windows-gnu") { - tarballs.push(mingw_installer.unwrap()); - } - let tarball = Tarball::new(builder, "rust", &target.triple); let generated = tarball.combine(&tarballs); From 2cfac3f04599ab50f1a7b6e21bb98c02b53bba75 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 23 Jul 2021 15:02:12 +0200 Subject: [PATCH 7/9] don't build extra tools if build.tools is explicitly an empty list --- src/bootstrap/dist.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 9776b097bec07..5290e8453e7be 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -50,12 +50,7 @@ fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool { if !builder.config.extended { return false; } - - if let Some(tools) = &builder.config.tools { - tools.is_empty() || tools.contains(tool) - } else { - true - } + builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool)) } #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] From c15e2480908868024f41ac4de92eabf435f5e911 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 23 Jul 2021 16:51:43 +0200 Subject: [PATCH 8/9] refactor extended tarball generaton to use the new ensure_if_default --- src/bootstrap/builder.rs | 44 +++++++++++++++++++++++++++++++------ src/bootstrap/dist.rs | 47 +++++++++++++++------------------------- src/bootstrap/install.rs | 2 +- 3 files changed, 55 insertions(+), 38 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 56ecc6e68a98c..f2d841cb335ab 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -163,14 +163,8 @@ impl StepDescription { } fn maybe_run(&self, builder: &Builder<'_>, pathset: &PathSet) { - if builder.config.exclude.iter().any(|e| pathset.has(e)) { - eprintln!("Skipping {:?} because it is excluded", pathset); + if self.is_excluded(builder, pathset) { return; - } else if !builder.config.exclude.is_empty() { - eprintln!( - "{:?} not skipped for {:?} -- not in {:?}", - pathset, self.name, builder.config.exclude - ); } // Determine the targets participating in this rule. @@ -182,6 +176,21 @@ impl StepDescription { } } + fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool { + if builder.config.exclude.iter().any(|e| pathset.has(e)) { + eprintln!("Skipping {:?} because it is excluded", pathset); + return true; + } + + if !builder.config.exclude.is_empty() { + eprintln!( + "{:?} not skipped for {:?} -- not in {:?}", + pathset, self.name, builder.config.exclude + ); + } + false + } + fn run(v: &[StepDescription], builder: &Builder<'_>, paths: &[PathBuf]) { let should_runs = v.iter().map(|desc| (desc.should_run)(ShouldRun::new(builder))).collect::>(); @@ -1579,6 +1588,27 @@ impl<'a> Builder<'a> { self.cache.put(step, out.clone()); out } + + /// Ensure that a given step is built *only if it's supposed to be built by default*, returning + /// its output. This will cache the step, so it's safe (and good!) to call this as often as + /// needed to ensure that all dependencies are build. + pub(crate) fn ensure_if_default>>( + &'a self, + step: S, + ) -> S::Output { + let desc = StepDescription::from::(); + let should_run = (desc.should_run)(ShouldRun::new(self)); + + // Avoid running steps contained in --exclude + for pathset in &should_run.paths { + if desc.is_excluded(self, pathset) { + return None; + } + } + + // Only execute if it's supposed to run as default + if desc.default && should_run.is_really_default() { self.ensure(step) } else { None } + } } #[cfg(test)] diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 5290e8453e7be..730bc51acbe1a 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -59,7 +59,7 @@ pub struct Docs { } impl Step for Docs { - type Output = GeneratedTarball; + type Output = Option; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -72,7 +72,7 @@ impl Step for Docs { } /// Builds the `rust-docs` installer component. - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + fn run(self, builder: &Builder<'_>) -> Option { let host = self.host; builder.default_doc(&[]); @@ -82,7 +82,7 @@ impl Step for Docs { tarball.set_product_name("Rust Documentation"); tarball.add_bulk_dir(&builder.doc_out(host), dest); tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); - tarball.generate() + Some(tarball.generate()) } } @@ -1359,13 +1359,11 @@ impl Step for Extended { let mut tarballs = Vec::new(); let mut built_tools = HashSet::new(); - macro_rules! add_tool { + macro_rules! add_component { ($name:expr => $step:expr) => { - if should_build_extended_tool(builder, $name) { - if let Some(tarball) = builder.ensure($step) { - tarballs.push(tarball); - built_tools.insert($name); - } + if let Some(tarball) = builder.ensure_if_default($step) { + tarballs.push(tarball); + built_tools.insert($name); } }; } @@ -1377,31 +1375,20 @@ impl Step for Extended { tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) })); tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std")); - if builder.config.docs { - tarballs.push(builder.ensure(Docs { host: target })); - } - if target.contains("windows-gnu") { tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw")); } - if builder.config.profiler_enabled(target) - || should_build_extended_tool(builder, "rust-demangler") - { - if let Some(tarball) = builder.ensure(RustDemangler { compiler, target }) { - tarballs.push(tarball); - built_tools.insert("rust-demangler"); - } - } - - add_tool!("cargo" => Cargo { compiler, target }); - add_tool!("rustfmt" => Rustfmt { compiler, target }); - add_tool!("rls" => Rls { compiler, target }); - add_tool!("rust-analyzer" => RustAnalyzer { compiler, target }); - add_tool!("llvm-tools" => LlvmTools { target }); - add_tool!("clippy" => Clippy { compiler, target }); - add_tool!("miri" => Miri { compiler, target }); - add_tool!("analysis" => Analysis { compiler, target }); + add_component!("rust-docs" => Docs { host: target }); + add_component!("rust-demangler"=> RustDemangler { compiler, target }); + add_component!("cargo" => Cargo { compiler, target }); + add_component!("rustfmt" => Rustfmt { compiler, target }); + add_component!("rls" => Rls { compiler, target }); + add_component!("rust-analyzer" => RustAnalyzer { compiler, target }); + add_component!("llvm-components" => LlvmTools { target }); + add_component!("clippy" => Clippy { compiler, target }); + add_component!("miri" => Miri { compiler, target }); + add_component!("analysis" => Analysis { compiler, target }); let etc = builder.src.join("src/etc/installer"); diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index 72a9ff2d5a794..06acf1a9a0083 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -139,7 +139,7 @@ macro_rules! install { install!((self, builder, _config), Docs, "src/doc", _config.docs, only_hosts: false, { - let tarball = builder.ensure(dist::Docs { host: self.target }); + let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs"); install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); }; Std, "library/std", true, only_hosts: false, { From 69f712c088b16fa88672efa2d49fe2d6ea4ca06e Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 28 Jul 2021 16:57:25 +0200 Subject: [PATCH 9/9] set all of the optional tools as DEFAULT = true The default is then overridden by `should_run`. --- src/bootstrap/dist.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 730bc51acbe1a..f5ddc07ebb2d5 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -679,6 +679,7 @@ pub struct Analysis { impl Step for Analysis { type Output = Option; + const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { let default = should_build_extended_tool(&run.builder, "analysis"); @@ -958,6 +959,7 @@ pub struct Cargo { impl Step for Cargo { type Output = Option; + const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1014,6 +1016,7 @@ pub struct Rls { impl Step for Rls { type Output = Option; const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { let default = should_build_extended_tool(&run.builder, "rls"); @@ -1059,6 +1062,7 @@ pub struct RustAnalyzer { impl Step for RustAnalyzer { type Output = Option; + const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1114,6 +1118,7 @@ pub struct Clippy { impl Step for Clippy { type Output = Option; + const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1164,6 +1169,7 @@ pub struct Miri { impl Step for Miri { type Output = Option; + const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1223,6 +1229,7 @@ pub struct Rustfmt { impl Step for Rustfmt { type Output = Option; + const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1276,6 +1283,7 @@ pub struct RustDemangler { impl Step for RustDemangler { type Output = Option; + const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1975,6 +1983,7 @@ pub struct LlvmTools { impl Step for LlvmTools { type Output = Option; const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { let default = should_build_extended_tool(&run.builder, "llvm-tools");