Add convenient way to test stdin

.vscode/settings.json

@@ -4,5 +4,7 @@
         "./tests/integrations/basic-bin/Cargo.toml",
         "./tests/integrations/basic-fail/Cargo.toml",
         "./tests/integrations/basic-fail-mode/Cargo.toml",
-    ]
+        "./tests/integrations/cargo-run/Cargo.toml"
+    ],
+    "nixEnvSelector.nixFile": "${workspaceRoot}/shell.nix"
 }

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ui_test"
-version = "0.20.0"
+version = "0.20.1"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "A test framework for testing rustc diagnostics output"

README.md

@@ -6,6 +6,7 @@ A smaller version of compiletest-rs
   So if you have any slow tests, prepend them with a small integral number to make them get run first, taking advantage of parallelism as much as possible (instead of waiting for the slow tests at the end).
 * `cargo test --test your_test_name -- --help` lists the commands you can specify for filtering, blessing and making your tests less verbose.
     * Since `cargo test` on its own runs all tests, using `cargo test -- --check` will not work on its own, but `cargo test -- --quiet` and `cargo test -- some_test_name` will work just fine, as the CLI matches.
+* if there is a `.stdin` file with the same filename as your test, it will be piped as standard input to your program.
 
 ## Supported magic comment annotations
 
@@ -53,6 +54,8 @@ their command specifies, or the test will fail without even being run.
 * `//@aux-build: filename` looks for a file in the `auxiliary` directory (within the directory of the test), compiles it as a library and links the current crate against it. This allows you import the crate with `extern crate` or just via `use` statements. This will automatically detect aux files that are proc macros and build them as proc macros.
 * `//@run` compiles the test and runs the resulting binary. The resulting binary must exit successfully. Stdout and stderr are taken from the resulting binary. Any warnings during compilation are ignored.
     * You can also specify a different exit code/status that is expected via e.g. `//@run: 1` or `//@run: 101` (the latter is the standard Rust exit code for panics).
+    * run tests collect the run output into `.run.stderr` and `.run.stdout` respectively.
+    * if a `.run.stdin` file exists, it will be piped as standard input to your test's execution.
 
 [rustfix]: https://github.com/rust-lang/rustfix
 

shell.nix

@@ -0,0 +1,18 @@
+let
+  pkgs = import <nixpkgs> {};
+in
+pkgs.mkShell rec {
+  name = "rustc";
+  buildInputs = with pkgs; [
+    rustup
+    pkg-config
+    alsaLib
+    libGL
+    xorg.libX11
+    xorg.libXi
+    python39
+  ];
+  RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}";
+  LD_LIBRARY_PATH = "${pkgs.lib.makeLibraryPath buildInputs}";
+}
+

src/config.rs

@@ -166,12 +166,14 @@ impl Config {
     }
 
     /// Replace all occurrences of a path in stderr/stdout with a byte string.
+    #[track_caller]
     pub fn path_filter(&mut self, path: &Path, replacement: &'static (impl AsRef<[u8]> + ?Sized)) {
         self.path_stderr_filter(path, replacement);
         self.path_stdout_filter(path, replacement);
     }
 
     /// Replace all occurrences of a path in stderr with a byte string.
+    #[track_caller]
     pub fn path_stderr_filter(
         &mut self,
         path: &Path,
@@ -183,6 +185,7 @@ impl Config {
     }
 
     /// Replace all occurrences of a path in stdout with a byte string.
+    #[track_caller]
     pub fn path_stdout_filter(
         &mut self,
         path: &Path,
@@ -194,12 +197,14 @@ impl Config {
     }
 
     /// Replace all occurrences of a regex pattern in stderr/stdout with a byte string.
+    #[track_caller]
     pub fn filter(&mut self, pattern: &str, replacement: &'static (impl AsRef<[u8]> + ?Sized)) {
         self.stderr_filter(pattern, replacement);
         self.stdout_filter(pattern, replacement);
     }
 
     /// Replace all occurrences of a regex pattern in stderr with a byte string.
+    #[track_caller]
     pub fn stderr_filter(
         &mut self,
         pattern: &str,
@@ -210,6 +215,7 @@ impl Config {
     }
 
     /// Replace all occurrences of a regex pattern in stdout with a byte string.
+    #[track_caller]
     pub fn stdout_filter(
         &mut self,
         pattern: &str,

src/lib.rs

@@ -627,23 +627,45 @@ impl dyn TestStatus {
             build_manager,
         )?;
 
-        let mut cmd = build_command(path, config, revision, comments)?;
+        let mut config = config.clone();
+
+        // Put aux builds into a separate directory per path so that multiple aux files
+        // from different directories (but with the same file name) don't collide.
+        let relative = strip_path_prefix(path.parent().unwrap(), &config.out_dir);
+
+        config.out_dir.extend(relative);
+
+        let mut cmd = build_command(path, &config, revision, comments)?;
         cmd.args(&extra_args);
+        let stdin = path.with_extension("stdin");
+        if stdin.exists() {
+            cmd.stdin(std::fs::File::open(stdin).unwrap());
+        }
 
         let (cmd, status, stderr, stdout) = self.run_command(cmd)?;
 
         let mode = config.mode.maybe_override(comments, revision)?;
+        let cmd = check_test_result(
+            cmd,
+            match *mode {
+                Mode::Run { .. } => Mode::Pass,
+                _ => *mode,
+            },
+            path,
+            &config,
+            revision,
+            comments,
+            status,
+            &stdout,
+            &stderr,
+        )?;
 
         if let Mode::Run { .. } = *mode {
-            if Mode::Pass.ok(status).is_ok() {
-                return run_test_binary(mode, path, revision, comments, cmd, config);
-            }
+            return run_test_binary(mode, path, revision, comments, cmd, &config);
         }
-        check_test_result(
-            cmd, *mode, path, config, revision, comments, status, &stdout, &stderr,
-        )?;
+
         run_rustfix(
-            &stderr, &stdout, path, comments, revision, config, *mode, extra_args,
+            &stderr, &stdout, path, comments, revision, &config, *mode, extra_args,
         )?;
         Ok(TestOk::Ok)
     }
@@ -739,6 +761,11 @@ fn run_test_binary(
     mut cmd: Command,
     config: &Config,
 ) -> TestResult {
+    let revision = if revision.is_empty() {
+        "run".to_string()
+    } else {
+        format!("run.{revision}")
+    };
     cmd.arg("--print").arg("file-names");
     let output = cmd.output().unwrap();
     assert!(output.status.success());
@@ -747,16 +774,22 @@ fn run_test_binary(
     let file = files.next().unwrap();
     assert_eq!(files.next(), None);
     let file = std::str::from_utf8(file).unwrap();
-    let exe = config.out_dir.join(file);
-    let mut exe = Command::new(exe);
-    let output = exe.output().unwrap();
+    let exe_file = config.out_dir.join(file);
+    let mut exe = Command::new(&exe_file);
+    let stdin = path.with_extension(format!("{revision}.stdin"));
+    if stdin.exists() {
+        exe.stdin(std::fs::File::open(stdin).unwrap());
+    }
+    let output = exe
+        .output()
+        .unwrap_or_else(|err| panic!("exe file: {}: {err}", exe_file.display()));
 
     let mut errors = vec![];
 
     check_test_output(
         path,
         &mut errors,
-        revision,
+        &revision,
         config,
         comments,
         &output.stdout,
@@ -948,7 +981,7 @@ fn check_test_result(
     status: ExitStatus,
     stdout: &[u8],
     stderr: &[u8],
-) -> Result<(), Errored> {
+) -> Result<Command, Errored> {
     let mut errors = vec![];
     errors.extend(mode.ok(status).err());
     // Always remove annotation comments from stderr.
@@ -973,7 +1006,7 @@ fn check_test_result(
         comments,
     )?;
     if errors.is_empty() {
-        Ok(())
+        Ok(command)
     } else {
         Err(Errored {
             command,

tests/integration.rs

@@ -33,9 +33,10 @@ fn main() -> Result<()> {
 
     config.stdout_filter("in ([0-9]m )?[0-9\\.]+s", "");
     config.stdout_filter(r#""--out-dir"(,)? "[^"]+""#, r#""--out-dir"$1 "$$TMP"#);
+    config.filter("\\.exe", b"");
     config.filter(
-        "( *process didn't exit successfully: `[^-]+)-[0-9a-f]+",
-        "$1-HASH",
+        "( *process didn't exit successfully: `.*)-[0-9a-f]+`",
+        "$1-HASH`",
     );
     // Windows io::Error uses "exit code".
     config.filter("exit code", "exit status");
@@ -54,7 +55,6 @@ fn main() -> Result<()> {
     config
         .stdout_filters
         .insert(0, (Match::Exact(b"\\\\".to_vec()), b"\\"));
-    config.filter("\\.exe", b"");
     config.stdout_filter(r#"(panic.*)\.rs:[0-9]+:[0-9]+"#, "$1.rs");
     config.filter("(\\+)? +[0-9]+: .*\n", "");
     config.filter("(\\+)?                +at /.*\n", "");
@@ -69,6 +69,7 @@ fn main() -> Result<()> {
     config.filter("(src/.*?\\.rs):[0-9]+:[0-9]+", "$1:LL:CC");
     config.filter("program not found", "No such file or directory");
     config.filter(" \\(os error [0-9]+\\)", "");
+    config.filter("note: rustc 1\\..*", "");
 
     let text = ui_test::status_emitter::Text::from(args.format);
 

tests/integrations/basic-fail/Cargo.stderr

@@ -5,7 +5,7 @@ Location:
 error: test failed, to rerun pass `--test ui_tests`
 
 Caused by:
-  process didn't exit successfully: `$DIR/target/ui/debug/deps/ui_tests-HASH` (exit status: 1)
+  process didn't exit successfully: `$DIR/target/ui/$DIR/debug/deps/ui_tests-HASH` (exit status: 1)
 thread 'main' panicked at 'invalid mode/result combo: yolo: Err(tests failed
 
 Location:
@@ -21,15 +21,15 @@ Location:
 error: test failed, to rerun pass `--test ui_tests_invalid_program`
 
 Caused by:
-  process didn't exit successfully: `$DIR/target/ui/debug/deps/ui_tests_invalid_program-HASH` (exit status: 1)
+  process didn't exit successfully: `$DIR/target/ui/$DIR/debug/deps/ui_tests_invalid_program-HASH` (exit status: 1)
 Error: tests failed
 
 Location:
     $DIR/src/lib.rs:LL:CC
 error: test failed, to rerun pass `--test ui_tests_invalid_program2`
 
 Caused by:
-  process didn't exit successfully: `$DIR/target/ui/debug/deps/ui_tests_invalid_program2-HASH` (exit status: 1)
+  process didn't exit successfully: `$DIR/target/ui/$DIR/debug/deps/ui_tests_invalid_program2-HASH` (exit status: 1)
 error: 4 targets failed:
     `--test ui_tests`
     `--test ui_tests_bless`

tests/integrations/basic-fail/Cargo.stdout

@@ -58,8 +58,8 @@ tests/actual_tests/executable.rs FAILED:
 command: "$CMD"
 
 actual output differed from expected
-Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/executable.stdout` to the actual output
---- tests/actual_tests/executable.stdout
+Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/executable.run.stdout` to the actual output
+--- tests/actual_tests/executable.run.stdout
 +++ <stdout output>
 -69
 +42
@@ -74,7 +74,7 @@ full stdout:
 tests/actual_tests/executable_compile_err.rs FAILED:
 command: "rustc" "--error-format=json" "--extern" "basic_fail=$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug" "--out-dir" "$TMP "tests/actual_tests/executable_compile_err.rs" "--edition" "2021"
 
-run(0) test got exit status: 1, but expected 0
+pass test got exit status: 1, but expected 0
 
 actual output differed from expected
 Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/executable_compile_err.stderr` to the actual output
@@ -235,25 +235,25 @@ command: "rustc" "--error-format=json" "--extern" "basic_fail=$DIR/$DIR/../../..
 fail test got exit status: 101, but expected 1
 
 error: substring `mismatched types` not found in stderr output
-  --> tests/actual_tests/rustc_ice.rs:10:17
+  --> tests/actual_tests/rustc_ice.rs:11:17
    |
-10 |     //~^ ERROR: mismatched types
+11 |     //~^ ERROR: mismatched types
    |                 ^^^^^^^^^^^^^^^^ expected because of this pattern
    |
 
 error: There were 1 unmatched diagnostics
- --> tests/actual_tests/rustc_ice.rs:9:5
-  |
-9 |     add("42", 3);
-  |     ^^^^^^^^^^^^ Ice: no errors reported for args
-  |
+  --> tests/actual_tests/rustc_ice.rs:10:5
+   |
+10 |     add("42", 3);
+   |     ^^^^^^^^^^^^ Ice: no errors reported for args
+   |
 
 full stderr:
 error: internal compiler error: no errors reported for args
- --> tests/actual_tests/rustc_ice.rs:9:5
-  |
-9 |     add("42", 3);
-  |     ^^^^^^^^^^^^
+  --> tests/actual_tests/rustc_ice.rs:10:5
+   |
+10 |     add("42", 3);
+   |     ^^^^^^^^^^^^
 
 thread 'rustc' panicked at 'aborting due to `-Z treat-err-as-bug=1`', compiler/rustc_errors/src/lib.rs
 stack backtrace:
@@ -262,7 +262,7 @@ error: the compiler unexpectedly panicked. this is a bug.
 
 note: we would appreciate a bug report: https://github.com/rust-lang/rust/issues/new?labels=C-bug%2C+I-ICE%2C+T-compiler&template=ice.md
 
-note: rustc 1.72.0 (5680fa18f 2023-08-23)
+
 
 note: compiler flags: -Z treat-err-as-bug
 

tests/integrations/basic-fail/tests/actual_tests/rustc_ice.rs

@@ -3,6 +3,7 @@
 //@normalize-stderr-test: " +[0-9]+: .*\n" -> ""
 //@normalize-stderr-test: "                +at /.*\n" -> ""
 //@normalize-stderr-test: " running on .*" -> ""
+//@normalize-stderr-test: "note: rustc 1\..*" -> ""
 use basic_fail::add;
 
 fn main() {

tests/integrations/basic-fail/tests/actual_tests/rustc_ice.stderr

@@ -1,8 +1,8 @@
 error: internal compiler error: no errors reported for args
- --> $DIR/rustc_ice.rs:9:5
-  |
-9 |     add("42", 3);
-  |     ^^^^^^^^^^^^
+  --> $DIR/rustc_ice.rs:10:5
+   |
+10 |     add("42", 3);
+   |     ^^^^^^^^^^^^
 
 thread 'rustc' panicked at 'aborting due to `-Z treat-err-as-bug=1`', compiler/rustc_errors/src/lib.rs:1712:30
 stack backtrace:
@@ -11,7 +11,7 @@ error: the compiler unexpectedly panicked. this is a bug.
 
 note: we would appreciate a bug report: https://github.com/rust-lang/rust/issues/new?labels=C-bug%2C+I-ICE%2C+T-compiler&template=ice.md
 
-note: rustc 1.72.0 (5680fa18f 2023-08-23)
+
 
 note: compiler flags: -Z treat-err-as-bug