Rollup merge of #77407 - pietroalbini:less-build-manifest, r=Mark-Sim…

…ulacrum

Improve build-manifest to work with the improved promote-release

This PR makes some changes to build-manifest to have it work better with the other improvements I'm making to [promote-release](https://github.com/rust-lang/promote-release).

A new way to invoke the tool was added: `./x.py run src/tools/build-manifest`. The new invocation disables the generation of `.sha256` files and the generation of GPG signatures, as those steps are not tied to the Rust version we're building the manifest of: handling them in `promote-release` will improve the maintenability of our release process. Invocations through the old command (`./x.py dist hash-and-sign`) are referred inside the source code as "legacy". The new invocation also enables internal parallelism, disabled on legacy to avoid overloading our old server.

Improvements were also made on how the checksums included in the manifest are generated:

* The manifest is first generated with placeholder checksums, and then a function walks through the manifes and calculates only the needed hashes. Before this PR, all the hashes were calculated beforehand, including the hashes of unused files.
* Calculating the hashes is now done in parallel with rayon, to better utilize all the available disk bandwidth.
* The `sha2` crate is now used instead of the `sha256sum` CLI tool: this avoids the overhead of calling another process, but more importantly enables hardware acceleration whenever available (the `sha256sum` CLI tool doesn't support it at all).

r? @Mark-Simulacrum
This PR is best reviewed commit-by-commit.

Cargo.lock

@@ -183,7 +183,16 @@ dependencies = [
  "block-padding",
  "byte-tools",
  "byteorder",
- "generic-array",
+ "generic-array 0.12.3",
+]
+
+[[package]]
+name = "block-buffer"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
+dependencies = [
+ "generic-array 0.14.4",
 ]
 
 [[package]]
@@ -233,8 +242,11 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "flate2",
+ "hex 0.4.2",
+ "rayon",
  "serde",
  "serde_json",
+ "sha2",
  "tar",
  "toml",
 ]
@@ -687,6 +699,12 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a21fa21941700a3cd8fcb4091f361a6a712fac632f85d9f487cc892045d55c6"
 
+[[package]]
+name = "cpuid-bool"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
+
 [[package]]
 name = "crates-io"
 version = "0.31.1"
@@ -884,7 +902,16 @@ version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
 dependencies = [
- "generic-array",
+ "generic-array 0.12.3",
+]
+
+[[package]]
+name = "digest"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
+dependencies = [
+ "generic-array 0.14.4",
 ]
 
 [[package]]
@@ -1166,6 +1193,16 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "generic-array"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
 [[package]]
 name = "getopts"
 version = "0.2.21"
@@ -1835,9 +1872,9 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a18af3dcaf2b0219366cdb4e2af65a6101457b415c3d1a5c71dd9c2b7c77b9c8"
 dependencies = [
- "block-buffer",
- "digest",
- "opaque-debug",
+ "block-buffer 0.7.3",
+ "digest 0.8.1",
+ "opaque-debug 0.2.3",
 ]
 
 [[package]]
@@ -2097,6 +2134,12 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
 
+[[package]]
+name = "opaque-debug"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
+
 [[package]]
 name = "open"
 version = "1.4.0"
@@ -4362,10 +4405,23 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df"
 dependencies = [
- "block-buffer",
- "digest",
+ "block-buffer 0.7.3",
+ "digest 0.8.1",
  "fake-simd",
- "opaque-debug",
+ "opaque-debug 0.2.3",
+]
+
+[[package]]
+name = "sha2"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1"
+dependencies = [
+ "block-buffer 0.9.0",
+ "cfg-if",
+ "cpuid-bool",
+ "digest 0.9.0",
+ "opaque-debug 0.3.0",
 ]
 
 [[package]]

src/bootstrap/builder.rs

@@ -477,7 +477,7 @@ impl<'a> Builder<'a> {
                 install::Src,
                 install::Rustc
             ),
-            Kind::Run => describe!(run::ExpandYamlAnchors,),
+            Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest,),
         }
     }
 

src/bootstrap/dist.rs

@@ -46,7 +46,7 @@ pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
     }
 }
 
-fn distdir(builder: &Builder<'_>) -> PathBuf {
+pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
     builder.out.join("dist")
 }
 
@@ -2371,6 +2371,7 @@ impl Step for HashSign {
         cmd.arg(addr);
         cmd.arg(&builder.config.channel);
         cmd.arg(&builder.src);
+        cmd.env("BUILD_MANIFEST_LEGACY", "1");
 
         builder.create_dir(&distdir(builder));
 

src/bootstrap/run.rs

@@ -1,5 +1,7 @@
 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::dist::distdir;
 use crate::tool::Tool;
+use build_helper::output;
 use std::process::Command;
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
@@ -41,3 +43,43 @@ fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool {
     }
     true
 }
+
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct BuildManifest;
+
+impl Step for BuildManifest {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/build-manifest")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(BuildManifest);
+    }
+
+    fn run(self, builder: &Builder<'_>) {
+        // This gets called by `promote-release`
+        // (https://github.com/rust-lang/promote-release).
+        let mut cmd = builder.tool_cmd(Tool::BuildManifest);
+        let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
+            panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
+        });
+        let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
+            panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
+        });
+
+        let today = output(Command::new("date").arg("+%Y-%m-%d"));
+
+        cmd.arg(sign);
+        cmd.arg(distdir(builder));
+        cmd.arg(today.trim());
+        cmd.arg(addr);
+        cmd.arg(&builder.config.channel);
+        cmd.arg(&builder.src);
+
+        builder.create_dir(&distdir(builder));
+        builder.run(&mut cmd);
+    }
+}

src/tools/build-manifest/Cargo.toml

@@ -11,3 +11,6 @@ serde_json = "1.0"
 anyhow = "1.0.32"
 flate2 = "1.0.16"
 tar = "0.4.29"
+sha2 = "0.9.1"
+rayon = "1.3.1"
+hex = "0.4.2"

src/tools/build-manifest/README.md

@@ -20,8 +20,7 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to
 `path/to/output` with:
 
 ```
-$ BUILD_MANIFEST_DISABLE_SIGNING=1 cargo +nightly run \
-    path/to/dist path/to/output 1970-01-01 http://example.com \
+$ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com \
     CHANNEL path/to/rust/repo
 ```