From 4329e4122bcde6a470e72a7ff8a914983882abc0 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Sun, 24 Nov 2024 14:19:25 +0000 Subject: [PATCH 01/10] Allow rendering a BUILD file for a single crate --- crate_universe/src/cli.rs | 8 ++++- crate_universe/src/cli/render.rs | 57 ++++++++++++++++++++++++++++++++ crate_universe/src/main.rs | 4 +++ crate_universe/src/rendering.rs | 2 +- 4 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 crate_universe/src/cli/render.rs diff --git a/crate_universe/src/cli.rs b/crate_universe/src/cli.rs index 9172c97f66..19a1fea833 100644 --- a/crate_universe/src/cli.rs +++ b/crate_universe/src/cli.rs @@ -2,6 +2,7 @@ mod generate; mod query; +mod render; mod splice; mod vendor; @@ -15,12 +16,14 @@ use tracing_subscriber::FmtSubscriber; pub use self::generate::GenerateOptions; pub use self::query::QueryOptions; +pub use self::render::RenderOptions; pub use self::splice::SpliceOptions; pub use self::vendor::VendorOptions; // Entrypoints pub use generate::generate; pub use query::query; +pub use render::render; pub use splice::splice; pub use vendor::vendor; @@ -42,6 +45,9 @@ pub enum Options { /// Vendor BUILD files to the workspace with either repository definitions or `cargo vendor` generated sources. Vendor(VendorOptions), + + /// Render a BUILD file for a single crate. + Render(RenderOptions), } // Convenience wrappers to avoid dependencies in the binary @@ -51,7 +57,7 @@ pub fn parse_args() -> Options { Options::parse() } -const EXPECTED_LOGGER_NAMES: [&str; 4] = ["Generate", "Splice", "Query", "Vendor"]; +const EXPECTED_LOGGER_NAMES: [&str; 5] = ["Generate", "Splice", "Query", "Vendor", "Render"]; /// A wrapper for the tracing-subscriber default [FormatEvent] /// that prepends the name of the active CLI option. diff --git a/crate_universe/src/cli/render.rs b/crate_universe/src/cli/render.rs new file mode 100644 index 0000000000..8a4648b6ed --- /dev/null +++ b/crate_universe/src/cli/render.rs @@ -0,0 +1,57 @@ +use crate::config::RenderConfig; +use crate::context::CrateContext; +use crate::rendering::{Platforms, Renderer}; +use crate::utils::target_triple::TargetTriple; + +use anyhow::{Context, Result}; +use clap::Parser; +use serde::{Deserialize, Serialize}; + +use std::collections::BTreeSet; +use std::path::PathBuf; + +#[derive(Parser, Debug)] +#[clap(about = "Command line options for the `render` subcommand", version)] +pub struct RenderOptions { + #[clap(long)] + options_json: String, + + #[clap(long)] + output_path: PathBuf, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct StructuredRenderOptions { + config: RenderConfig, + + supported_platform_triples: BTreeSet, + + platforms: Platforms, + + crate_context: CrateContext, +} + +pub fn render(opt: RenderOptions) -> Result<()> { + let RenderOptions { + options_json, + output_path, + } = opt; + + let deserialized_options = serde_json::from_str(&options_json) + .with_context(|| format!("Failed to deserialize options_json from '{}'", options_json))?; + + let StructuredRenderOptions { + config, + supported_platform_triples, + platforms, + crate_context, + } = deserialized_options; + + let renderer = Renderer::new(config, supported_platform_triples); + let output = renderer.render_one_build_file(&platforms, &crate_context) + .with_context(|| format!("Failed to render BUILD.bazel file for crate {}", crate_context.name))?; + std::fs::write(&output_path, output.as_bytes()) + .with_context(|| format!("Failed to write BUILD.bazel file to {}", output_path.display()))?; + + Ok(()) +} diff --git a/crate_universe/src/main.rs b/crate_universe/src/main.rs index e1ab0c9daa..473c831066 100644 --- a/crate_universe/src/main.rs +++ b/crate_universe/src/main.rs @@ -25,5 +25,9 @@ fn main() -> cli::Result<()> { cli::init_logging("Vendor", verbose_logging); cli::vendor(opt) } + cli::Options::Render(opt) => { + cli::init_logging("Render", verbose_logging); + cli::render(opt) + } } } diff --git a/crate_universe/src/rendering.rs b/crate_universe/src/rendering.rs index 5abb068e20..dd93052793 100644 --- a/crate_universe/src/rendering.rs +++ b/crate_universe/src/rendering.rs @@ -296,7 +296,7 @@ impl Renderer { .collect() } - fn render_one_build_file(&self, platforms: &Platforms, krate: &CrateContext) -> Result { + pub(crate) fn render_one_build_file(&self, platforms: &Platforms, krate: &CrateContext) -> Result { let mut starlark = Vec::new(); // Banner comment for top of the file. From 7b53a90fc4034bf1e8d5b2b7bc71d98accce9ef3 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 27 Nov 2024 22:35:43 +0000 Subject: [PATCH 02/10] External path dependencies work This allows dependencies of kind path to work, both in the [dependencies] table and the [patches] table. The intention is that this is used for third-party code not first-party code. --- .bazelci/presubmit.yml | 16 ++ crate_universe/extensions.bzl | 27 +++ crate_universe/private/crates_repository.bzl | 7 + crate_universe/private/crates_vendor.bzl | 3 +- crate_universe/private/generate_utils.bzl | 12 + crate_universe/private/local_crate_mirror.bzl | 63 ++++++ crate_universe/private/srcs.bzl | 1 + crate_universe/src/cli/generate.rs | 72 +++++- crate_universe/src/cli/render.rs | 44 ++-- crate_universe/src/cli/vendor.rs | 22 +- crate_universe/src/context.rs | 22 +- crate_universe/src/context/crate_context.rs | 8 + .../src/metadata/metadata_annotation.rs | 104 +++++++-- crate_universe/src/rendering.rs | 166 +++++++++----- .../src/rendering/template_engine.rs | 54 ++++- .../src/rendering/templates/module_bzl.j2 | 12 + examples/.bazelignore | 1 + examples/bzlmod/hello_world/MODULE.bazel.lock | 2 +- examples/crate_universe_local_path/.gitignore | 1 + .../crate_universe_local_path/BUILD.bazel | 12 + .../crate_universe_local_path/MODULE.bazel | 27 +++ .../crate_universe_local_path/WORKSPACE.bazel | 31 +++ .../crates_from_workspace/BUILD.bazel | 15 ++ .../crates_from_workspace/Cargo.lock | 14 ++ .../crates_from_workspace/Cargo.toml | 18 ++ .../crates_from_workspace/test.rs | 19 ++ .../.github/workflows/rust.yml | 60 +++++ .../lazy_static_1.5.0_copy/.gitignore | 5 + .../lazy_static_1.5.0_copy/Cargo.toml | 28 +++ .../lazy_static_1.5.0_copy/LICENSE-APACHE | 201 +++++++++++++++++ .../lazy_static_1.5.0_copy/LICENSE-MIT | 25 +++ .../lazy_static_1.5.0_copy/README.md | 105 +++++++++ .../examples/mutex_map.rs | 23 ++ .../lazy_static_1.5.0_copy/src/core_lazy.rs | 32 +++ .../lazy_static_1.5.0_copy/src/inline_lazy.rs | 48 ++++ .../lazy_static_1.5.0_copy/src/lib.rs | 208 ++++++++++++++++++ .../incorrect_visibility_restriction.rs | 8 + .../incorrect_visibility_restriction.stderr | 10 + .../tests/compile_fail/static_is_private.rs | 14 ++ .../compile_fail/static_is_private.stderr | 14 ++ .../tests/compile_fail/static_is_sized.rs | 9 + .../tests/compile_fail/static_is_sized.stderr | 64 ++++++ .../lazy_static_1.5.0_copy/tests/no_std.rs | 19 ++ .../lazy_static_1.5.0_copy/tests/test.rs | 185 ++++++++++++++++ .../lazy_static_1.5.0_copy/tests/ui.rs | 6 + .../vendor_lazy_static.sh | 28 +++ 46 files changed, 1750 insertions(+), 115 deletions(-) create mode 100644 crate_universe/private/local_crate_mirror.bzl create mode 100644 examples/crate_universe_local_path/.gitignore create mode 100644 examples/crate_universe_local_path/BUILD.bazel create mode 100644 examples/crate_universe_local_path/MODULE.bazel create mode 100644 examples/crate_universe_local_path/WORKSPACE.bazel create mode 100644 examples/crate_universe_local_path/crates_from_workspace/BUILD.bazel create mode 100644 examples/crate_universe_local_path/crates_from_workspace/Cargo.lock create mode 100644 examples/crate_universe_local_path/crates_from_workspace/Cargo.toml create mode 100644 examples/crate_universe_local_path/crates_from_workspace/test.rs create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/.github/workflows/rust.yml create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/.gitignore create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/Cargo.toml create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/LICENSE-APACHE create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/LICENSE-MIT create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/README.md create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/examples/mutex_map.rs create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/src/core_lazy.rs create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/src/inline_lazy.rs create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/src/lib.rs create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/tests/compile_fail/incorrect_visibility_restriction.rs create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/tests/compile_fail/incorrect_visibility_restriction.stderr create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/tests/compile_fail/static_is_private.rs create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/tests/compile_fail/static_is_private.stderr create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/tests/compile_fail/static_is_sized.rs create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/tests/compile_fail/static_is_sized.stderr create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/tests/no_std.rs create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/tests/test.rs create mode 100644 examples/crate_universe_local_path/lazy_static_1.5.0_copy/tests/ui.rs create mode 100755 examples/crate_universe_local_path/vendor_lazy_static.sh diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 8958eb37d5..e3d4674590 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -533,6 +533,22 @@ tasks: test_targets: - "//..." build_flags: *aspects_flags + crate_universe_local_path_external: + name: Crate Universe Local Path External + platform: ubuntu2004 + working_directory: examples/crate_universe_local_path + run_targets: + - "//:vendor_lazy_static_out_of_tree" + test_targets: + - "//..." + crate_universe_local_path_in_tree: + name: Crate Universe Local Path In Tree + platform: ubuntu2004 + working_directory: examples/crate_universe_local_path + run_targets: + - "//:vendor_lazy_static_in_tree" + test_targets: + - "//..." # See https://github.com/bazelbuild/rules_rust/issues/2186 about re-enabling these. # crate_universe_examples_windows: # name: Crate Universe Examples diff --git a/crate_universe/extensions.bzl b/crate_universe/extensions.bzl index 943fb73976..01594df3ae 100644 --- a/crate_universe/extensions.bzl +++ b/crate_universe/extensions.bzl @@ -6,6 +6,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//crate_universe/private:crates_vendor.bzl", "CRATES_VENDOR_ATTRS", "generate_config_file", "generate_splicing_manifest") load("//crate_universe/private:generate_utils.bzl", "CARGO_BAZEL_GENERATOR_SHA256", "CARGO_BAZEL_GENERATOR_URL", "GENERATOR_ENV_VARS", "render_config") +load("//crate_universe/private:local_crate_mirror.bzl", "local_crate_mirror") load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS") load("//rust/platform:triple.bzl", "get_host_triple") load("//rust/platform:triple_mappings.bzl", "system_to_binary_ext") @@ -133,6 +134,8 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo lockfile_path = tag_path.get_child("lockfile.json") module_ctx.file(lockfile_path, "") + paths_to_track_file = module_ctx.path("paths-to-track") + cargo_bazel([ "generate", "--cargo-lockfile", @@ -148,8 +151,16 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo "--repin", "--lockfile", lockfile_path, + "--nonhermetic-root-bazel-workspace-dir", + module_ctx.path(Label("@@//:MODULE.bazel")).dirname, + "--paths-to-track", + paths_to_track_file, ]) + paths_to_track = json.decode(module_ctx.read(paths_to_track_file)) + for path in paths_to_track: + module_ctx.read(path) + crates_dir = tag_path.get_child(cfg.name) _generate_repo( name = cfg.name, @@ -212,6 +223,22 @@ def _generate_hub_and_spokes(*, module_ctx, cargo_bazel, cfg, annotations, cargo strip_prefix = repo.get("strip_prefix", None), **kwargs ) + elif "Path" in repo: + options = { + "config": rendering_config, + "crate_context": crate, + "platform_conditions": contents["conditions"], + "supported_platform_triples": cfg.supported_platform_triples, + } + kwargs = {} + if len(CARGO_BAZEL_URLS) == 0: + kwargs["generator"] = "@cargo_bazel_bootstrap//:cargo-bazel" + local_crate_mirror( + name = crate_repo_name, + options_json = json.encode(options), + path = repo["Path"]["path"], + **kwargs + ) else: fail("Invalid repo: expected Http or Git to exist for crate %s-%s, got %s" % (name, version, repo)) diff --git a/crate_universe/private/crates_repository.bzl b/crate_universe/private/crates_repository.bzl index 7924711372..9514748619 100644 --- a/crate_universe/private/crates_repository.bzl +++ b/crate_universe/private/crates_repository.bzl @@ -71,6 +71,8 @@ def _crates_repository_impl(repository_ctx): "metadata": metadata_path, }) + paths_to_track_file = repository_ctx.path("paths-to-track") + # Run the generator execute_generator( repository_ctx = repository_ctx, @@ -82,10 +84,15 @@ def _crates_repository_impl(repository_ctx): repository_dir = repository_ctx.path("."), cargo = cargo_path, rustc = rustc_path, + paths_to_track_file = paths_to_track_file, # sysroot = tools.sysroot, **kwargs ) + paths_to_track = json.decode(repository_ctx.read(paths_to_track_file)) + for path in paths_to_track: + repository_ctx.read(path) + # Determine the set of reproducible values attrs = {attr: getattr(repository_ctx.attr, attr) for attr in dir(repository_ctx.attr)} exclude = ["to_json", "to_proto"] diff --git a/crate_universe/private/crates_vendor.bzl b/crate_universe/private/crates_vendor.bzl index b8d95627bf..a195f8c20f 100644 --- a/crate_universe/private/crates_vendor.bzl +++ b/crate_universe/private/crates_vendor.bzl @@ -34,6 +34,7 @@ ${{_ENVIRON[@]}} \\ {env} \\ "{bin}" \\ {args} \\ + --nonhermetic-root-bazel-workspace-dir="${{BUILD_WORKSPACE_DIRECTORY}}" \\ "$@" """ @@ -42,7 +43,7 @@ _WINDOWS_WRAPPER = """\ set RUNTIME_PWD=%CD% {env} -{bin} {args} %* +{bin} {args} --nonhermetic-root-bazel-workspace-dir=%BUILD_WORKSPACE_DIRECTORY% %* exit %ERRORLEVEL% """ diff --git a/crate_universe/private/generate_utils.bzl b/crate_universe/private/generate_utils.bzl index 94dcdd2360..87743e561f 100644 --- a/crate_universe/private/generate_utils.bzl +++ b/crate_universe/private/generate_utils.bzl @@ -433,6 +433,7 @@ def execute_generator( repository_dir, cargo, rustc, + paths_to_track_file, metadata = None): """Execute the `cargo-bazel` binary to produce `BUILD` and `.bzl` files. @@ -446,6 +447,7 @@ def execute_generator( repository_dir (path): The output path for the Bazel module and BUILD files. cargo (path): The path of a Cargo binary. rustc (path): The path of a Rustc binary. + paths_to_track_file (path): Path to file where generator should write which files should trigger re-generating as a JSON list. metadata (path, optional): The path to a Cargo metadata json file. If this is set, it indicates to the generator that repinning is required. This file must be adjacent to a `Cargo.toml` and `Cargo.lock` file. @@ -470,8 +472,18 @@ def execute_generator( cargo, "--rustc", rustc, + "--nonhermetic-root-bazel-workspace-dir", + repository_ctx.workspace_root, + "--paths-to-track", + paths_to_track_file, ] + if repository_ctx.attr.generator: + args.extend([ + "--generator", + repository_ctx.attr.generator, + ]) + if lockfile_path: args.extend([ "--lockfile", diff --git a/crate_universe/private/local_crate_mirror.bzl b/crate_universe/private/local_crate_mirror.bzl new file mode 100644 index 0000000000..444d7054a3 --- /dev/null +++ b/crate_universe/private/local_crate_mirror.bzl @@ -0,0 +1,63 @@ +"""`local_crate_mirror` rule implementation. + +This is a private implementation detail of crate_universe, and should not be relied on in manually written code. + +This is effectively a `local_repository` rule impementation, but where the BUILD.bazel file is generated using the `cargo-bazel render` command. +""" + +load("//crate_universe/private:common_utils.bzl", "execute") +load("//crate_universe/private:generate_utils.bzl", "get_generator") +load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS") +load("//rust/platform:triple.bzl", "get_host_triple") + +def _local_crate_mirror_impl(repository_ctx): + path = repository_ctx.path(repository_ctx.attr.path) + + host_triple = get_host_triple(repository_ctx) + + generator, _generator_sha256 = get_generator(repository_ctx, host_triple.str) + + execute(repository_ctx, ["bash", "-c", "cp -r {}/* {}/".format(path, repository_ctx.path("."))]) + + paths_to_track = execute(repository_ctx, ["bash", "-c", "find {} -type f".format(path)]).stdout.strip().split("\n") + for path_to_track in paths_to_track: + if path_to_track: + repository_ctx.read(path_to_track) + + execute(repository_ctx, [generator, "render", "--options-json", repository_ctx.attr.options_json, "--output-path", repository_ctx.path("BUILD.bazel")]) + + repository_ctx.file("WORKSPACE", "") + +local_crate_mirror = repository_rule( + implementation = _local_crate_mirror_impl, + attrs = { + "generator": attr.string( + doc = ( + "The absolute label of a generator. Eg. `@cargo_bazel_bootstrap//:cargo-bazel`. " + + "This is typically used when bootstrapping" + ), + ), + "generator_sha256s": attr.string_dict( + doc = "Dictionary of `host_triple` -> `sha256` for a `cargo-bazel` binary.", + default = CARGO_BAZEL_SHA256S, + ), + "generator_urls": attr.string_dict( + doc = ( + "URL template from which to download the `cargo-bazel` binary. `{host_triple}` and will be " + + "filled in according to the host platform." + ), + default = CARGO_BAZEL_URLS, + ), + "options_json": attr.string( + doc = "JSON serialized instance of a crate_universe::context::SingleBuildFileRenderContext", + ), + "path": attr.string( + # TODO: Verify what happens if this is not an absolute path. + doc = "Absolute path to the BUILD.bazel file to generate.", + ), + "quiet": attr.bool( + doc = "If stdout and stderr should not be printed to the terminal.", + default = True, + ), + }, +) diff --git a/crate_universe/private/srcs.bzl b/crate_universe/private/srcs.bzl index a96e424466..bea5cc28f9 100644 --- a/crate_universe/private/srcs.bzl +++ b/crate_universe/private/srcs.bzl @@ -11,6 +11,7 @@ CARGO_BAZEL_SRCS = [ Label("//crate_universe:src/cli.rs"), Label("//crate_universe:src/cli/generate.rs"), Label("//crate_universe:src/cli/query.rs"), + Label("//crate_universe:src/cli/render.rs"), Label("//crate_universe:src/cli/splice.rs"), Label("//crate_universe:src/cli/vendor.rs"), Label("//crate_universe:src/config.rs"), diff --git a/crate_universe/src/cli/generate.rs b/crate_universe/src/cli/generate.rs index 2178f276cb..5d974139c2 100644 --- a/crate_universe/src/cli/generate.rs +++ b/crate_universe/src/cli/generate.rs @@ -2,18 +2,21 @@ use std::fs; use std::path::{Path, PathBuf}; +use std::sync::Arc; use anyhow::{bail, Context as AnyhowContext, Result}; +use camino::Utf8PathBuf; use cargo_lock::Lockfile; use clap::Parser; use crate::config::Config; use crate::context::Context; use crate::lockfile::{lock_context, write_lockfile}; -use crate::metadata::{load_metadata, Annotations, Cargo}; +use crate::metadata::{load_metadata, Annotations, Cargo, SourceAnnotation}; use crate::rendering::{write_outputs, Renderer}; use crate::splicing::SplicingManifest; use crate::utils::normalize_cargo_file_paths; +use crate::utils::starlark::Label; /// Command line options for the `generate` subcommand #[derive(Parser, Debug)] @@ -63,6 +66,23 @@ pub struct GenerateOptions { /// If true, outputs will be printed instead of written to disk. #[clap(long)] pub dry_run: bool, + + /// The path to the Bazel root workspace (i.e. the directory containing the WORKSPACE.bazel file or similar). + /// BE CAREFUL with this value. We never want to include it in a lockfile hash (to keep lockfiles portable), + /// which means you also should not use it anywhere that _should_ be guarded by a lockfile hash. + /// You basically never want to use this value. + #[clap(long)] + pub nonhermetic_root_bazel_workspace_dir: Utf8PathBuf, + + #[clap(long)] + pub paths_to_track: PathBuf, + + /// The label of this binary, if it was built in bootstrap mode. + /// BE CAREFUL with this value. We never want to include it in a lockfile hash (to keep lockfiles portable), + /// which means you also should not use it anywhere that _should_ be guarded by a lockfile hash. + /// You basically never want to use this value. + #[clap(long)] + pub(crate) generator: Option