Skip to content

Commit

Permalink
Auto merge of #63470 - Mark-Simulacrum:rustc-depdep, r=alexcrichton
Browse files Browse the repository at this point in the history 
Utilize -Zbinary-dep-depinfo in rustbuild

We no longer utilize stamp-file mtimes at all inside rustbuild, and a future PR may be able to entirely eliminate them by eagerly copying to the appropriate sysroot. The only mtime-based dependency tracking left is for documentation because we lie to Cargo about the rustdoc binary, so Cargo does not track changes to the real binary, and codegen-backends because binary-dep-depinfo does not emit that information into the depfiles.

Both of these are fixable in the longer term but this existing patch gives us the following benefits:
 * We no longer delete Cargo target directories manually within a stage. Cross-stage, changes to codegen backends will still clear out target directories. This means that incremental state persists across individual steps (e.g., rebuilding libstd does not clear out librustc incremental state). Fixes #54712.
 * Dependency tracking across steps within a given stage is now fully precise. We will not clear out all codegen backend dependencies due to changes in librustc_driver, for example, only deleting the final librustc_codegen_llvm crate. Fixes #54008, fixes #50481.
 * We properly track codegen backends as a dependency (equivalent to rustc) across changes. Fixes #53284, and fixes #52719.
 * Cross-stage dependency tracking of crates is also much more accurate and reliable. Most likely fixes #49979 (but no reproduction steps in that issue). Fixes #59105.

cc #63012
  • Loading branch information
@bors
bors committed Aug 16, 2019
2 parents 9dd5c19 + 417f9ea commit 9a32ad0
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 100 deletions.
87 changes: 22 additions & 65 deletions src/bootstrap/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -754,76 +754,20 @@ impl<'a> Builder<'a> {
let mut cargo = Command::new(&self.initial_cargo);
let out_dir = self.stage_out(compiler, mode);

// command specific path, we call clear_if_dirty with this
let mut my_out = match cmd {
"build" => self.cargo_out(compiler, mode, target),

// This is the intended out directory for crate documentation.
"doc" | "rustdoc" => self.crate_doc_out(target),

_ => self.stage_out(compiler, mode),
};

// This is for the original compiler, but if we're forced to use stage 1, then
// std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since
// we copy the libs forward.
let cmp = self.compiler_for(compiler.stage, compiler.host, target);

let libstd_stamp = match cmd {
"check" | "clippy" | "fix" => check::libstd_stamp(self, cmp, target),
_ => compile::libstd_stamp(self, cmp, target),
};

let libtest_stamp = match cmd {
"check" | "clippy" | "fix" => check::libtest_stamp(self, cmp, target),
_ => compile::libtest_stamp(self, cmp, target),
};

let librustc_stamp = match cmd {
"check" | "clippy" | "fix" => check::librustc_stamp(self, cmp, target),
_ => compile::librustc_stamp(self, cmp, target),
};
// Codegen backends are not yet tracked by -Zbinary-dep-depinfo,
// so we need to explicitly clear out if they've been updated.
for backend in self.codegen_backends(compiler) {
self.clear_if_dirty(&out_dir, &backend);
}

if cmd == "doc" || cmd == "rustdoc" {
if mode == Mode::Rustc || mode == Mode::ToolRustc || mode == Mode::Codegen {
let my_out = match mode {
// This is the intended out directory for compiler documentation.
my_out = self.compiler_doc_out(target);
}
Mode::Rustc | Mode::ToolRustc | Mode::Codegen => self.compiler_doc_out(target),
_ => self.crate_doc_out(target),
};
let rustdoc = self.rustdoc(compiler);
self.clear_if_dirty(&my_out, &rustdoc);
} else if cmd != "test" {
match mode {
Mode::Std => {
self.clear_if_dirty(&my_out, &self.rustc(compiler));
for backend in self.codegen_backends(compiler) {
self.clear_if_dirty(&my_out, &backend);
}
},
Mode::Test => {
self.clear_if_dirty(&my_out, &libstd_stamp);
},
Mode::Rustc => {
self.clear_if_dirty(&my_out, &self.rustc(compiler));
self.clear_if_dirty(&my_out, &libstd_stamp);
self.clear_if_dirty(&my_out, &libtest_stamp);
},
Mode::Codegen => {
self.clear_if_dirty(&my_out, &librustc_stamp);
},
Mode::ToolBootstrap => { },
Mode::ToolStd => {
self.clear_if_dirty(&my_out, &libstd_stamp);
},
Mode::ToolTest => {
self.clear_if_dirty(&my_out, &libstd_stamp);
self.clear_if_dirty(&my_out, &libtest_stamp);
},
Mode::ToolRustc => {
self.clear_if_dirty(&my_out, &libstd_stamp);
self.clear_if_dirty(&my_out, &libtest_stamp);
self.clear_if_dirty(&my_out, &librustc_stamp);
},
}
}

cargo
Expand Down Expand Up @@ -861,6 +805,19 @@ impl<'a> Builder<'a> {
},
}

// This tells Cargo (and in turn, rustc) to output more complete
// dependency information. Most importantly for rustbuild, this
// includes sysroot artifacts, like libstd, which means that we don't
// need to track those in rustbuild (an error prone process!). This
// feature is currently unstable as there may be some bugs and such, but
// it represents a big improvement in rustbuild's reliability on
// rebuilds, so we're using it here.
//
// For some additional context, see #63470 (the PR originally adding
// this), as well as #63012 which is the tracking issue for this
// feature on the rustc side.
cargo.arg("-Zbinary-dep-depinfo");

cargo.arg("-j").arg(self.jobs().to_string());
// Remove make-related flags to ensure Cargo can correctly set things up
cargo.env_remove("MAKEFLAGS");
Expand Down
1 change: 0 additions & 1 deletion src/bootstrap/check.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,6 @@ impl Step for Rustdoc {
let libdir = builder.sysroot_libdir(compiler, target);
let hostdir = builder.sysroot_libdir(compiler, compiler.host);
add_to_sysroot(&builder, &libdir, &hostdir, &rustdoc_stamp(builder, compiler, target));
builder.cargo(compiler, Mode::ToolRustc, target, "clean");
}
}

Expand Down
35 changes: 1 addition & 34 deletions src/bootstrap/compile.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use std::path::{Path, PathBuf};
use std::process::{Command, Stdio, exit};
use std::str;

use build_helper::{output, mtime, t, up_to_date};
use build_helper::{output, t, up_to_date};
use filetime::FileTime;
use serde::Deserialize;
use serde_json;
Expand Down Expand Up @@ -274,8 +274,6 @@ impl Step for StdLink {
// for reason why the sanitizers are not built in stage0.
copy_apple_sanitizer_dylibs(builder, &builder.native_dir(target), "osx", &libdir);
}

builder.cargo(target_compiler, Mode::ToolStd, target, "clean");
}
}

Expand Down Expand Up @@ -480,8 +478,6 @@ impl Step for TestLink {
&builder.sysroot_libdir(target_compiler, compiler.host),
&libtest_stamp(builder, compiler, target)
);

builder.cargo(target_compiler, Mode::ToolTest, target, "clean");
}
}

Expand Down Expand Up @@ -639,7 +635,6 @@ impl Step for RustcLink {
&builder.sysroot_libdir(target_compiler, compiler.host),
&librustc_stamp(builder, compiler, target)
);
builder.cargo(target_compiler, Mode::ToolRustc, target, "clean");
}
}

Expand Down Expand Up @@ -1202,41 +1197,13 @@ pub fn run_cargo(builder: &Builder<'_>,
deps.push((path_to_add.into(), false));
}

// Now we want to update the contents of the stamp file, if necessary. First
// we read off the previous contents along with its mtime. If our new
// contents (the list of files to copy) is different or if any dep's mtime
// is newer then we rewrite the stamp file.
deps.sort();
let stamp_contents = fs::read(stamp);
let stamp_mtime = mtime(&stamp);
let mut new_contents = Vec::new();
let mut max = None;
let mut max_path = None;
for (dep, proc_macro) in deps.iter() {
let mtime = mtime(dep);
if Some(mtime) > max {
max = Some(mtime);
max_path = Some(dep.clone());
}
new_contents.extend(if *proc_macro { b"h" } else { b"t" });
new_contents.extend(dep.to_str().unwrap().as_bytes());
new_contents.extend(b"\0");
}
let max = max.unwrap();
let max_path = max_path.unwrap();
let contents_equal = stamp_contents
.map(|contents| contents == new_contents)
.unwrap_or_default();
if contents_equal && max <= stamp_mtime {
builder.verbose(&format!("not updating {:?}; contents equal and {:?} <= {:?}",
stamp, max, stamp_mtime));
return deps.into_iter().map(|(d, _)| d).collect()
}
if max > stamp_mtime {
builder.verbose(&format!("updating {:?} as {:?} changed", stamp, max_path));
} else {
builder.verbose(&format!("updating {:?} as deps changed", stamp));
}
t!(fs::write(&stamp, &new_contents));
deps.into_iter().map(|(d, _)| d).collect()
}
Expand Down

0 comments on commit 9a32ad0

Please sign in to comment.