From 004db4728bc6ccc62416f93ba6473b144def7fe8 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Mon, 14 Aug 2023 13:00:28 +1000 Subject: [PATCH] Add test suite `coverage-map` to test coverage mappings emitted by LLVM We compile each test file to LLVM IR assembly, and then pass that IR to a dedicated program that can decode LLVM coverage maps and print them in a more human-readable format. We can then check that output against known-good snapshots. This test suite has some advantages over the existing `run-coverage` tests: - We can test coverage instrumentation without needing to run target binaries. - We can observe subtle improvements/regressions in the underlying coverage mappings that don't make a visible difference to coverage reports. --- src/bootstrap/builder.rs | 1 + src/bootstrap/test.rs | 14 +++ src/tools/compiletest/src/common.rs | 6 + src/tools/compiletest/src/lib.rs | 2 + src/tools/compiletest/src/runtest.rs | 71 ++++++++++- tests/coverage-map/if.cov-map | 15 +++ tests/coverage-map/if.rs | 9 ++ tests/coverage-map/long_and_wide.cov-map | 32 +++++ tests/coverage-map/long_and_wide.rs | 150 +++++++++++++++++++++++ tests/coverage-map/trivial.cov-map | 8 ++ tests/coverage-map/trivial.rs | 3 + 11 files changed, 306 insertions(+), 5 deletions(-) create mode 100644 tests/coverage-map/if.cov-map create mode 100644 tests/coverage-map/if.rs create mode 100644 tests/coverage-map/long_and_wide.cov-map create mode 100644 tests/coverage-map/long_and_wide.rs create mode 100644 tests/coverage-map/trivial.cov-map create mode 100644 tests/coverage-map/trivial.rs diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 73c0a192cef94..a24a6a4636d48 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -726,6 +726,7 @@ impl<'a> Builder<'a> { test::Tidy, test::Ui, test::RunPassValgrind, + test::CoverageMap, test::RunCoverage, test::MirOpt, test::Codegen, diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index d1018978f78c9..d78e0deda69f0 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1340,6 +1340,12 @@ host_test!(RunMakeFullDeps { default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" }); +default_test!(CoverageMap { + path: "tests/coverage-map", + mode: "coverage-map", + suite: "coverage-map" +}); + host_test!(RunCoverage { path: "tests/run-coverage", mode: "run-coverage", suite: "run-coverage" }); host_test!(RunCoverageRustdoc { path: "tests/run-coverage-rustdoc", @@ -1545,6 +1551,14 @@ note: if you're sure you want to do this, please open an issue as to why. In the .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); } + if mode == "coverage-map" { + let coverage_dump = builder.ensure(tool::CoverageDump { + compiler: compiler.with_stage(0), + target: compiler.host, + }); + cmd.arg("--coverage-dump-path").arg(coverage_dump); + } + if mode == "run-make" || mode == "run-coverage" { let rust_demangler = builder .ensure(tool::RustDemangler { diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index 7c17e92d0dfe2..b91d5a958bb62 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -66,6 +66,7 @@ string_enum! { JsDocTest => "js-doc-test", MirOpt => "mir-opt", Assembly => "assembly", + CoverageMap => "coverage-map", RunCoverage => "run-coverage", } } @@ -161,6 +162,9 @@ pub struct Config { /// The rust-demangler executable. pub rust_demangler_path: Option, + /// The coverage-dump executable. + pub coverage_dump_path: Option, + /// The Python executable to use for LLDB and htmldocck. pub python: String, @@ -639,6 +643,7 @@ pub const UI_EXTENSIONS: &[&str] = &[ UI_STDERR_32, UI_STDERR_16, UI_COVERAGE, + UI_COVERAGE_MAP, ]; pub const UI_STDERR: &str = "stderr"; pub const UI_STDOUT: &str = "stdout"; @@ -649,6 +654,7 @@ pub const UI_STDERR_64: &str = "64bit.stderr"; pub const UI_STDERR_32: &str = "32bit.stderr"; pub const UI_STDERR_16: &str = "16bit.stderr"; pub const UI_COVERAGE: &str = "coverage"; +pub const UI_COVERAGE_MAP: &str = "cov-map"; /// Absolute path to the directory where all output for all tests in the given /// `relative_dir` group should reside. Example: diff --git a/src/tools/compiletest/src/lib.rs b/src/tools/compiletest/src/lib.rs index 1a765477fe501..619ff9b322114 100644 --- a/src/tools/compiletest/src/lib.rs +++ b/src/tools/compiletest/src/lib.rs @@ -48,6 +48,7 @@ pub fn parse_config(args: Vec) -> Config { .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH") .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH") .optopt("", "rust-demangler-path", "path to rust-demangler to use in tests", "PATH") + .optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH") .reqopt("", "python", "path to python to use for doc tests", "PATH") .optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH") .optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH") @@ -218,6 +219,7 @@ pub fn parse_config(args: Vec) -> Config { rustc_path: opt_path(matches, "rustc-path"), rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from), rust_demangler_path: matches.opt_str("rust-demangler-path").map(PathBuf::from), + coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from), python: matches.opt_str("python").unwrap(), jsondocck_path: matches.opt_str("jsondocck-path"), jsondoclint_path: matches.opt_str("jsondoclint-path"), diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index dd4c59fdff513..670441aacbd08 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -6,8 +6,8 @@ use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, RustdocJs use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc}; use crate::common::{CompareMode, FailMode, PassMode}; use crate::common::{Config, TestPaths}; -use crate::common::{Pretty, RunCoverage, RunPassValgrind}; -use crate::common::{UI_COVERAGE, UI_RUN_STDERR, UI_RUN_STDOUT}; +use crate::common::{CoverageMap, Pretty, RunCoverage, RunPassValgrind}; +use crate::common::{UI_COVERAGE, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT}; use crate::compute_diff::{write_diff, write_filtered_diff}; use crate::errors::{self, Error, ErrorKind}; use crate::header::TestProps; @@ -254,6 +254,7 @@ impl<'test> TestCx<'test> { MirOpt => self.run_mir_opt_test(), Assembly => self.run_assembly_test(), JsDocTest => self.run_js_doc_test(), + CoverageMap => self.run_coverage_map_test(), RunCoverage => self.run_coverage_test(), } } @@ -467,6 +468,46 @@ impl<'test> TestCx<'test> { } } + fn run_coverage_map_test(&self) { + let Some(coverage_dump_path) = &self.config.coverage_dump_path else { + self.fatal("missing --coverage-dump"); + }; + + let proc_res = self.compile_test_and_save_ir(); + if !proc_res.status.success() { + self.fatal_proc_rec("compilation failed!", &proc_res); + } + drop(proc_res); + + let llvm_ir_path = self.output_base_name().with_extension("ll"); + + let mut dump_command = Command::new(coverage_dump_path); + dump_command.arg(llvm_ir_path); + let proc_res = self.run_command_to_procres(&mut dump_command); + if !proc_res.status.success() { + self.fatal_proc_rec("coverage-dump failed!", &proc_res); + } + + let kind = UI_COVERAGE_MAP; + + let expected_coverage_dump = self.load_expected_output(kind); + let actual_coverage_dump = self.normalize_output(&proc_res.stdout, &[]); + + let coverage_dump_errors = self.compare_output( + kind, + &actual_coverage_dump, + &expected_coverage_dump, + self.props.compare_output_lines_by_subset, + ); + + if coverage_dump_errors > 0 { + self.fatal_proc_rec( + &format!("{coverage_dump_errors} errors occurred comparing coverage output."), + &proc_res, + ); + } + } + fn run_coverage_test(&self) { let should_run = self.run_if_enabled(); let proc_res = self.compile_test(should_run, Emit::None); @@ -650,6 +691,10 @@ impl<'test> TestCx<'test> { let mut cmd = Command::new(tool_path); configure_cmd_fn(&mut cmd); + self.run_command_to_procres(&mut cmd) + } + + fn run_command_to_procres(&self, cmd: &mut Command) -> ProcRes { let output = cmd.output().unwrap_or_else(|_| panic!("failed to exec `{cmd:?}`")); let proc_res = ProcRes { @@ -2321,9 +2366,11 @@ impl<'test> TestCx<'test> { } } DebugInfo => { /* debuginfo tests must be unoptimized */ } - RunCoverage => { - // Coverage reports are affected by optimization level, and - // the current snapshots assume no optimization by default. + CoverageMap | RunCoverage => { + // Coverage mappings and coverage reports are affected by + // optimization level, so they ignore the optimize-tests + // setting and set an optimization level in their mode's + // compile flags (below) or in per-test `compile-flags`. } _ => { rustc.arg("-O"); @@ -2392,8 +2439,22 @@ impl<'test> TestCx<'test> { rustc.arg(dir_opt); } + CoverageMap => { + rustc.arg("-Cinstrument-coverage"); + // These tests only compile to MIR, so they don't need the + // profiler runtime to be present. + rustc.arg("-Zno-profiler-runtime"); + // Coverage mappings are sensitive to MIR optimizations, and + // the current snapshots assume `opt-level=2` unless overridden + // by `compile-flags`. + rustc.arg("-Copt-level=2"); + } RunCoverage => { rustc.arg("-Cinstrument-coverage"); + // Coverage reports are sometimes sensitive to optimizations, + // and the current snapshots assume no optimization unless + // overridden by `compile-flags`. + rustc.arg("-Copt-level=0"); } RunPassValgrind | Pretty | DebugInfo | Codegen | Rustdoc | RustdocJson | RunMake | CodegenUnits | JsDocTest | Assembly => { diff --git a/tests/coverage-map/if.cov-map b/tests/coverage-map/if.cov-map new file mode 100644 index 0000000000000..3cedb5ffbecb1 --- /dev/null +++ b/tests/coverage-map/if.cov-map @@ -0,0 +1,15 @@ +Function name: if::main +Raw bytes (28): 0x[01, 01, 02, 01, 05, 05, 02, 04, 01, 03, 01, 02, 0c, 05, 02, 0d, 02, 06, 02, 02, 06, 00, 07, 07, 01, 05, 01, 02] +Number of files: 1 +- file 0 => global file 1 +Number of expressions: 2 +- expression 0 operands: lhs = Counter(0), rhs = Counter(1) +- expression 1 operands: lhs = Counter(1), rhs = Expression(0, Sub) +Number of file 0 mappings: 4 +- Code(Counter(0)) at (prev + 3, 1) to (start + 2, 12) +- Code(Counter(1)) at (prev + 2, 13) to (start + 2, 6) +- Code(Expression(0, Sub)) at (prev + 2, 6) to (start + 0, 7) + = (c0 - c1) +- Code(Expression(1, Add)) at (prev + 1, 5) to (start + 1, 2) + = (c1 + (c0 - c1)) + diff --git a/tests/coverage-map/if.rs b/tests/coverage-map/if.rs new file mode 100644 index 0000000000000..ed3f69bdc98d2 --- /dev/null +++ b/tests/coverage-map/if.rs @@ -0,0 +1,9 @@ +// compile-flags: --edition=2021 + +fn main() { + let cond = std::env::args().len() == 1; + if cond { + println!("true"); + } + println!("done"); +} diff --git a/tests/coverage-map/long_and_wide.cov-map b/tests/coverage-map/long_and_wide.cov-map new file mode 100644 index 0000000000000..97aebf9b18ab0 --- /dev/null +++ b/tests/coverage-map/long_and_wide.cov-map @@ -0,0 +1,32 @@ +Function name: long_and_wide::far_function +Raw bytes (10): 0x[01, 01, 00, 01, 01, 96, 01, 01, 00, 15] +Number of files: 1 +- file 0 => global file 1 +Number of expressions: 0 +Number of file 0 mappings: 1 +- Code(Counter(0)) at (prev + 150, 1) to (start + 0, 21) + +Function name: long_and_wide::long_function +Raw bytes (10): 0x[01, 01, 00, 01, 01, 10, 01, 84, 01, 02] +Number of files: 1 +- file 0 => global file 1 +Number of expressions: 0 +Number of file 0 mappings: 1 +- Code(Counter(0)) at (prev + 16, 1) to (start + 132, 2) + +Function name: long_and_wide::main +Raw bytes (9): 0x[01, 01, 00, 01, 01, 07, 01, 04, 02] +Number of files: 1 +- file 0 => global file 1 +Number of expressions: 0 +Number of file 0 mappings: 1 +- Code(Counter(0)) at (prev + 7, 1) to (start + 4, 2) + +Function name: long_and_wide::wide_function +Raw bytes (10): 0x[01, 01, 00, 01, 01, 0e, 01, 00, 8b, 01] +Number of files: 1 +- file 0 => global file 1 +Number of expressions: 0 +Number of file 0 mappings: 1 +- Code(Counter(0)) at (prev + 14, 1) to (start + 0, 139) + diff --git a/tests/coverage-map/long_and_wide.rs b/tests/coverage-map/long_and_wide.rs new file mode 100644 index 0000000000000..a7cbcd4802791 --- /dev/null +++ b/tests/coverage-map/long_and_wide.rs @@ -0,0 +1,150 @@ +// compile-flags: --edition=2021 +// ignore-tidy-linelength + +// This file deliberately contains line and column numbers larger than 127, +// to verify that `coverage-dump`'s ULEB128 parser can handle them. + +fn main() { + wide_function(); + long_function(); + far_function(); +} + +#[rustfmt::skip] +fn wide_function() { /* */ (); } + +fn long_function() { + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // + // +} + +fn far_function() {} diff --git a/tests/coverage-map/trivial.cov-map b/tests/coverage-map/trivial.cov-map new file mode 100644 index 0000000000000..874e294a1c498 --- /dev/null +++ b/tests/coverage-map/trivial.cov-map @@ -0,0 +1,8 @@ +Function name: trivial::main +Raw bytes (9): 0x[01, 01, 00, 01, 01, 03, 01, 00, 0d] +Number of files: 1 +- file 0 => global file 1 +Number of expressions: 0 +Number of file 0 mappings: 1 +- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 13) + diff --git a/tests/coverage-map/trivial.rs b/tests/coverage-map/trivial.rs new file mode 100644 index 0000000000000..d0a9b44fb3605 --- /dev/null +++ b/tests/coverage-map/trivial.rs @@ -0,0 +1,3 @@ +// compile-flags: --edition=2021 + +fn main() {}