From b6d8ee611f20b11883bc807150c2a54c934ff9fd Mon Sep 17 00:00:00 2001 From: Oliver Scherer Date: Thu, 14 Sep 2023 21:07:08 +0200 Subject: [PATCH 01/11] Strip compiler version and hash from test output --- tests/integration.rs | 1 + tests/integrations/basic-fail/Cargo.stdout | 24 +++++++++---------- .../tests/actual_tests/rustc_ice.rs | 1 + .../tests/actual_tests/rustc_ice.stderr | 10 ++++---- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/tests/integration.rs b/tests/integration.rs index 59384079..d2259557 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -69,6 +69,7 @@ fn main() -> Result<()> { config.filter("(src/.*?\\.rs):[0-9]+:[0-9]+", "$1:LL:CC"); config.filter("program not found", "No such file or directory"); config.filter(" \\(os error [0-9]+\\)", ""); + config.filter("note: rustc 1\\..*", ""); let text = ui_test::status_emitter::Text::from(args.format); diff --git a/tests/integrations/basic-fail/Cargo.stdout b/tests/integrations/basic-fail/Cargo.stdout index f763391d..d55cc280 100644 --- a/tests/integrations/basic-fail/Cargo.stdout +++ b/tests/integrations/basic-fail/Cargo.stdout @@ -235,25 +235,25 @@ command: "rustc" "--error-format=json" "--extern" "basic_fail=$DIR/$DIR/../../.. fail test got exit status: 101, but expected 1 error: substring `mismatched types` not found in stderr output - --> tests/actual_tests/rustc_ice.rs:10:17 + --> tests/actual_tests/rustc_ice.rs:11:17 | -10 | //~^ ERROR: mismatched types +11 | //~^ ERROR: mismatched types | ^^^^^^^^^^^^^^^^ expected because of this pattern | error: There were 1 unmatched diagnostics - --> tests/actual_tests/rustc_ice.rs:9:5 - | -9 | add("42", 3); - | ^^^^^^^^^^^^ Ice: no errors reported for args - | + --> tests/actual_tests/rustc_ice.rs:10:5 + | +10 | add("42", 3); + | ^^^^^^^^^^^^ Ice: no errors reported for args + | full stderr: error: internal compiler error: no errors reported for args - --> tests/actual_tests/rustc_ice.rs:9:5 - | -9 | add("42", 3); - | ^^^^^^^^^^^^ + --> tests/actual_tests/rustc_ice.rs:10:5 + | +10 | add("42", 3); + | ^^^^^^^^^^^^ thread 'rustc' panicked at 'aborting due to `-Z treat-err-as-bug=1`', compiler/rustc_errors/src/lib.rs stack backtrace: @@ -262,7 +262,7 @@ error: the compiler unexpectedly panicked. this is a bug. note: we would appreciate a bug report: https://github.com/rust-lang/rust/issues/new?labels=C-bug%2C+I-ICE%2C+T-compiler&template=ice.md -note: rustc 1.72.0 (5680fa18f 2023-08-23) + note: compiler flags: -Z treat-err-as-bug diff --git a/tests/integrations/basic-fail/tests/actual_tests/rustc_ice.rs b/tests/integrations/basic-fail/tests/actual_tests/rustc_ice.rs index 3008a372..576192e7 100644 --- a/tests/integrations/basic-fail/tests/actual_tests/rustc_ice.rs +++ b/tests/integrations/basic-fail/tests/actual_tests/rustc_ice.rs @@ -3,6 +3,7 @@ //@normalize-stderr-test: " +[0-9]+: .*\n" -> "" //@normalize-stderr-test: " +at /.*\n" -> "" //@normalize-stderr-test: " running on .*" -> "" +//@normalize-stderr-test: "note: rustc 1\..*" -> "" use basic_fail::add; fn main() { diff --git a/tests/integrations/basic-fail/tests/actual_tests/rustc_ice.stderr b/tests/integrations/basic-fail/tests/actual_tests/rustc_ice.stderr index 0f02f80a..87281596 100644 --- a/tests/integrations/basic-fail/tests/actual_tests/rustc_ice.stderr +++ b/tests/integrations/basic-fail/tests/actual_tests/rustc_ice.stderr @@ -1,8 +1,8 @@ error: internal compiler error: no errors reported for args - --> $DIR/rustc_ice.rs:9:5 - | -9 | add("42", 3); - | ^^^^^^^^^^^^ + --> $DIR/rustc_ice.rs:10:5 + | +10 | add("42", 3); + | ^^^^^^^^^^^^ thread 'rustc' panicked at 'aborting due to `-Z treat-err-as-bug=1`', compiler/rustc_errors/src/lib.rs:1712:30 stack backtrace: @@ -11,7 +11,7 @@ error: the compiler unexpectedly panicked. this is a bug. note: we would appreciate a bug report: https://github.com/rust-lang/rust/issues/new?labels=C-bug%2C+I-ICE%2C+T-compiler&template=ice.md -note: rustc 1.72.0 (5680fa18f 2023-08-23) + note: compiler flags: -Z treat-err-as-bug From c3cbd7b4361fbcda7bb1ea196d50fc6ef896ada6 Mon Sep 17 00:00:00 2001 From: Oliver Scherer Date: Thu, 14 Sep 2023 21:11:32 +0200 Subject: [PATCH 02/11] Add mode to automatically read `.stdin` files and pass them to the binary --- .vscode/settings.json | 1 + src/lib.rs | 4 + tests/integrations/cargo-run/Cargo.lock | 902 ++++++++++++++++++ tests/integrations/cargo-run/Cargo.stdout | 13 + tests/integrations/cargo-run/Cargo.toml | 14 + tests/integrations/cargo-run/src/main.rs | 13 + .../tests/actual_tests/empty_no_stdin.rs | 0 .../tests/actual_tests/empty_no_stdin.stderr | 2 + .../tests/actual_tests/matching_stdin.rs | 2 + .../tests/actual_tests/matching_stdin.stderr | 2 + .../tests/actual_tests/matching_stdin.stdin | 2 + .../tests/actual_tests/mismatching_stdin.rs | 2 + .../actual_tests/mismatching_stdin.stderr | 4 + .../actual_tests/mismatching_stdin.stdin | 1 + .../cargo-run/tests/actual_tests/no_stdin.rs | 2 + .../tests/actual_tests/no_stdin.stderr | 2 + .../integrations/cargo-run/tests/ui_tests.rs | 25 + 17 files changed, 991 insertions(+) create mode 100644 tests/integrations/cargo-run/Cargo.lock create mode 100644 tests/integrations/cargo-run/Cargo.stdout create mode 100644 tests/integrations/cargo-run/Cargo.toml create mode 100644 tests/integrations/cargo-run/src/main.rs create mode 100644 tests/integrations/cargo-run/tests/actual_tests/empty_no_stdin.rs create mode 100644 tests/integrations/cargo-run/tests/actual_tests/empty_no_stdin.stderr create mode 100644 tests/integrations/cargo-run/tests/actual_tests/matching_stdin.rs create mode 100644 tests/integrations/cargo-run/tests/actual_tests/matching_stdin.stderr create mode 100644 tests/integrations/cargo-run/tests/actual_tests/matching_stdin.stdin create mode 100644 tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.rs create mode 100644 tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.stderr create mode 100644 tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.stdin create mode 100644 tests/integrations/cargo-run/tests/actual_tests/no_stdin.rs create mode 100644 tests/integrations/cargo-run/tests/actual_tests/no_stdin.stderr create mode 100644 tests/integrations/cargo-run/tests/ui_tests.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index eed12a79..e11c8984 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,5 +4,6 @@ "./tests/integrations/basic-bin/Cargo.toml", "./tests/integrations/basic-fail/Cargo.toml", "./tests/integrations/basic-fail-mode/Cargo.toml", + "./tests/integrations/cargo-run/Cargo.toml" ] } diff --git a/src/lib.rs b/src/lib.rs index c6f8fd1a..2a16a49e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -629,6 +629,10 @@ impl dyn TestStatus { let mut cmd = build_command(path, config, revision, comments)?; cmd.args(&extra_args); + let stdin = path.with_extension("stdin"); + if stdin.exists() { + cmd.stdin(std::fs::File::open(stdin).unwrap()); + } let (cmd, status, stderr, stdout) = self.run_command(cmd)?; diff --git a/tests/integrations/cargo-run/Cargo.lock b/tests/integrations/cargo-run/Cargo.lock new file mode 100644 index 00000000..2ae30b52 --- /dev/null +++ b/tests/integrations/cargo-run/Cargo.lock @@ -0,0 +1,902 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "annotate-snippets" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b9d411ecbaf79885c6df4d75fff75858d5995ff25385657a28af47e82f9c36" +dependencies = [ + "unicode-width", + "yansi-term", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "basic_bin" +version = "0.1.0" +dependencies = [ + "tempfile", + "ui_test", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + +[[package]] +name = "bstr" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "color-eyre" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + +[[package]] +name = "colored" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" +dependencies = [ + "is-terminal", + "lazy_static", + "windows-sys 0.48.0", +] + +[[package]] +name = "comma" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335" + +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "eyre" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "indicatif" +version = "0.17.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b297dc40733f23a0e52728a58fa9489a5b7638a324932de16b41adc3ef80730" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix 0.38.4", + "windows-sys 0.48.0", +] + +[[package]] +name = "itoa" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "levenshtein" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "linux-raw-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" + +[[package]] +name = "log" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + +[[package]] +name = "pad" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ad9b889f1b12e0b9ee24db044b5129150d5eada288edc800f789928dc8c0e3" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" + +[[package]] +name = "portable-atomic" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc55135a600d700580e406b4de0d59cb9ad25e344a3a091a97ded2622ec4ec6" + +[[package]] +name = "prettydiff" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ff1fec61082821f8236cf6c0c14e8172b62ce8a72a0eedc30d3b247bb68dc11" +dependencies = [ + "ansi_term", + "pad", +] + +[[package]] +name = "proc-macro2" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +dependencies = [ + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustfix" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd2853d9e26988467753bd9912c3a126f642d05d229a4b53f5752ee36c56481" +dependencies = [ + "anyhow", + "log", + "serde", + "serde_json", +] + +[[package]] +name = "rustix" +version = "0.37.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +dependencies = [ + "bitflags 2.3.3", + "errno", + "libc", + "linux-raw-sys 0.4.3", + "windows-sys 0.48.0", +] + +[[package]] +name = "ryu" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" + +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +dependencies = [ + "serde", +] + +[[package]] +name = "serde" +version = "1.0.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.171" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "syn" +version = "2.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +dependencies = [ + "autocfg", + "cfg-if", + "fastrand", + "redox_syscall", + "rustix 0.37.23", + "windows-sys 0.48.0", +] + +[[package]] +name = "thiserror" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + +[[package]] +name = "ui_test" +version = "0.20.0" +dependencies = [ + "annotate-snippets", + "anyhow", + "bstr", + "cargo-platform", + "cargo_metadata", + "color-eyre", + "colored", + "comma", + "crossbeam-channel", + "indicatif", + "lazy_static", + "levenshtein", + "prettydiff", + "regex", + "rustc_version", + "rustfix", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "unicode-ident" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.1", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "yansi-term" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1" +dependencies = [ + "winapi", +] diff --git a/tests/integrations/cargo-run/Cargo.stdout b/tests/integrations/cargo-run/Cargo.stdout new file mode 100644 index 00000000..043e9e39 --- /dev/null +++ b/tests/integrations/cargo-run/Cargo.stdout @@ -0,0 +1,13 @@ + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished + +Building dependencies ... ok +tests/actual_tests/empty_no_stdin.rs ... ok +tests/actual_tests/matching_stdin.rs ... ok +tests/actual_tests/mismatching_stdin.rs ... ok +tests/actual_tests/no_stdin.rs ... ok + +test result: ok. 4 passed; + diff --git a/tests/integrations/cargo-run/Cargo.toml b/tests/integrations/cargo-run/Cargo.toml new file mode 100644 index 00000000..7474d9c4 --- /dev/null +++ b/tests/integrations/cargo-run/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "basic_bin" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dev-dependencies] +ui_test = { path = "../../.."} +tempfile = "3.3.0" + +[[test]] +name = "ui_tests" +harness = false diff --git a/tests/integrations/cargo-run/src/main.rs b/tests/integrations/cargo-run/src/main.rs new file mode 100644 index 00000000..7d4a5aa2 --- /dev/null +++ b/tests/integrations/cargo-run/src/main.rs @@ -0,0 +1,13 @@ +fn main() { + let arg = std::env::args().nth(1).unwrap(); + let input = std::fs::read_to_string(arg).unwrap(); + let mut b = std::io::stdin().lines(); + for (line, a ) in input.lines().enumerate() { + let b = b.next().expect(&format!(".stdin file is missing line {}", line + 1)).unwrap(); + assert_eq!(a, b); + } + for b in b { + panic!("{}", b.unwrap()); + } + panic!("done"); +} diff --git a/tests/integrations/cargo-run/tests/actual_tests/empty_no_stdin.rs b/tests/integrations/cargo-run/tests/actual_tests/empty_no_stdin.rs new file mode 100644 index 00000000..e69de29b diff --git a/tests/integrations/cargo-run/tests/actual_tests/empty_no_stdin.stderr b/tests/integrations/cargo-run/tests/actual_tests/empty_no_stdin.stderr new file mode 100644 index 00000000..bb416ba7 --- /dev/null +++ b/tests/integrations/cargo-run/tests/actual_tests/empty_no_stdin.stderr @@ -0,0 +1,2 @@ +thread 'main' panicked at 'done', src/main.rs:12:5 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace diff --git a/tests/integrations/cargo-run/tests/actual_tests/matching_stdin.rs b/tests/integrations/cargo-run/tests/actual_tests/matching_stdin.rs new file mode 100644 index 00000000..5f5521fa --- /dev/null +++ b/tests/integrations/cargo-run/tests/actual_tests/matching_stdin.rs @@ -0,0 +1,2 @@ +abc +def diff --git a/tests/integrations/cargo-run/tests/actual_tests/matching_stdin.stderr b/tests/integrations/cargo-run/tests/actual_tests/matching_stdin.stderr new file mode 100644 index 00000000..bb416ba7 --- /dev/null +++ b/tests/integrations/cargo-run/tests/actual_tests/matching_stdin.stderr @@ -0,0 +1,2 @@ +thread 'main' panicked at 'done', src/main.rs:12:5 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace diff --git a/tests/integrations/cargo-run/tests/actual_tests/matching_stdin.stdin b/tests/integrations/cargo-run/tests/actual_tests/matching_stdin.stdin new file mode 100644 index 00000000..5f5521fa --- /dev/null +++ b/tests/integrations/cargo-run/tests/actual_tests/matching_stdin.stdin @@ -0,0 +1,2 @@ +abc +def diff --git a/tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.rs b/tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.rs new file mode 100644 index 00000000..0bf4a5ab --- /dev/null +++ b/tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.rs @@ -0,0 +1,2 @@ +cake +lie diff --git a/tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.stderr b/tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.stderr new file mode 100644 index 00000000..c5c8748b --- /dev/null +++ b/tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.stderr @@ -0,0 +1,4 @@ +thread 'main' panicked at 'assertion failed: `(left == right)` + left: `"cake"`, + right: `"asdfasdf"`', src/main.rs:7:9 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace diff --git a/tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.stdin b/tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.stdin new file mode 100644 index 00000000..f13588b9 --- /dev/null +++ b/tests/integrations/cargo-run/tests/actual_tests/mismatching_stdin.stdin @@ -0,0 +1 @@ +asdfasdf diff --git a/tests/integrations/cargo-run/tests/actual_tests/no_stdin.rs b/tests/integrations/cargo-run/tests/actual_tests/no_stdin.rs new file mode 100644 index 00000000..cba17e97 --- /dev/null +++ b/tests/integrations/cargo-run/tests/actual_tests/no_stdin.rs @@ -0,0 +1,2 @@ +asdf +uiop diff --git a/tests/integrations/cargo-run/tests/actual_tests/no_stdin.stderr b/tests/integrations/cargo-run/tests/actual_tests/no_stdin.stderr new file mode 100644 index 00000000..c9fa1491 --- /dev/null +++ b/tests/integrations/cargo-run/tests/actual_tests/no_stdin.stderr @@ -0,0 +1,2 @@ +thread 'main' panicked at '.stdin file is missing line 1', src/main.rs:6:26 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace diff --git a/tests/integrations/cargo-run/tests/ui_tests.rs b/tests/integrations/cargo-run/tests/ui_tests.rs new file mode 100644 index 00000000..0613148a --- /dev/null +++ b/tests/integrations/cargo-run/tests/ui_tests.rs @@ -0,0 +1,25 @@ +use ui_test::*; + +fn main() -> ui_test::color_eyre::Result<()> { + let mut config = Config { + output_conflict_handling: if std::env::var_os("BLESS").is_some() { + OutputConflictHandling::Bless + } else { + OutputConflictHandling::Error("cargo test".to_string()) + }, + mode: Mode::Panic, + ..Config::cargo("tests/actual_tests") + }; + + config.program.args = vec!["run".into(), "--quiet".into()]; + config.program.input_file_flag = Some("--".into()); + config.program.out_dir_flag = None; + + run_tests_generic( + vec![config], + default_file_filter, + |_config, _path, _content| {}, + // Avoid github actions, as these would end up showing up in `Cargo.stderr` + status_emitter::Text::verbose(), + ) +} From 0fa3a92c3a7e8e59639f95718a688bccf91927bf Mon Sep 17 00:00:00 2001 From: Oliver Scherer Date: Thu, 14 Sep 2023 21:32:51 +0200 Subject: [PATCH 03/11] Separate run output from build output --- src/lib.rs | 33 ++++++++++++++----- tests/integrations/basic-fail/Cargo.stdout | 6 ++-- ...xecutable.stdout => executable.run.stdout} | 0 ...e.stderr => failing_executable.run.stderr} | 0 ...isioned_executable_panic.run.panic.stderr} | 0 ...evisioned_executable_panic.run.run.stderr} | 0 ...{run_panic.stderr => run_panic.run.stderr} | 0 ...xecutable.stdout => executable.run.stdout} | 0 8 files changed, 27 insertions(+), 12 deletions(-) rename tests/integrations/basic-fail/tests/actual_tests/{executable.stdout => executable.run.stdout} (100%) rename tests/integrations/basic-fail/tests/actual_tests_bless/{failing_executable.stderr => failing_executable.run.stderr} (100%) rename tests/integrations/basic-fail/tests/actual_tests_bless/{revisioned_executable_panic.panic.stderr => revisioned_executable_panic.run.panic.stderr} (100%) rename tests/integrations/basic-fail/tests/actual_tests_bless/{revisioned_executable_panic.run.stderr => revisioned_executable_panic.run.run.stderr} (100%) rename tests/integrations/basic-fail/tests/actual_tests_bless/{run_panic.stderr => run_panic.run.stderr} (100%) rename tests/integrations/basic/tests/actual_tests/{executable.stdout => executable.run.stdout} (100%) diff --git a/src/lib.rs b/src/lib.rs index 2a16a49e..075a28c8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -637,15 +637,25 @@ impl dyn TestStatus { let (cmd, status, stderr, stdout) = self.run_command(cmd)?; let mode = config.mode.maybe_override(comments, revision)?; + let cmd = check_test_result( + cmd, + match *mode { + Mode::Run { .. } => Mode::Pass, + _ => *mode, + }, + path, + config, + revision, + comments, + status, + &stdout, + &stderr, + )?; if let Mode::Run { .. } = *mode { - if Mode::Pass.ok(status).is_ok() { - return run_test_binary(mode, path, revision, comments, cmd, config); - } + return run_test_binary(mode, path, revision, comments, cmd, config); } - check_test_result( - cmd, *mode, path, config, revision, comments, status, &stdout, &stderr, - )?; + run_rustfix( &stderr, &stdout, path, comments, revision, config, *mode, extra_args, )?; @@ -743,6 +753,11 @@ fn run_test_binary( mut cmd: Command, config: &Config, ) -> TestResult { + let revision = if revision.is_empty() { + "run".to_string() + } else { + format!("run.{revision}") + }; cmd.arg("--print").arg("file-names"); let output = cmd.output().unwrap(); assert!(output.status.success()); @@ -760,7 +775,7 @@ fn run_test_binary( check_test_output( path, &mut errors, - revision, + &revision, config, comments, &output.stdout, @@ -952,7 +967,7 @@ fn check_test_result( status: ExitStatus, stdout: &[u8], stderr: &[u8], -) -> Result<(), Errored> { +) -> Result { let mut errors = vec![]; errors.extend(mode.ok(status).err()); // Always remove annotation comments from stderr. @@ -977,7 +992,7 @@ fn check_test_result( comments, )?; if errors.is_empty() { - Ok(()) + Ok(command) } else { Err(Errored { command, diff --git a/tests/integrations/basic-fail/Cargo.stdout b/tests/integrations/basic-fail/Cargo.stdout index d55cc280..7b13acd2 100644 --- a/tests/integrations/basic-fail/Cargo.stdout +++ b/tests/integrations/basic-fail/Cargo.stdout @@ -58,8 +58,8 @@ tests/actual_tests/executable.rs FAILED: command: "$CMD" actual output differed from expected -Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/executable.stdout` to the actual output ---- tests/actual_tests/executable.stdout +Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/executable.run.stdout` to the actual output +--- tests/actual_tests/executable.run.stdout +++ -69 +42 @@ -74,7 +74,7 @@ full stdout: tests/actual_tests/executable_compile_err.rs FAILED: command: "rustc" "--error-format=json" "--extern" "basic_fail=$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug/libbasic_fail.rlib" "--extern" "basic_fail=$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug/libbasic_fail-$HASH.rmeta" "-L" "$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug" "-L" "$DIR/$DIR/../../../target/$TMP/$TRIPLE/debug" "--out-dir" "$TMP "tests/actual_tests/executable_compile_err.rs" "--edition" "2021" -run(0) test got exit status: 1, but expected 0 +pass test got exit status: 1, but expected 0 actual output differed from expected Execute `DO NOT BLESS. These are meant to fail` to update `tests/actual_tests/executable_compile_err.stderr` to the actual output diff --git a/tests/integrations/basic-fail/tests/actual_tests/executable.stdout b/tests/integrations/basic-fail/tests/actual_tests/executable.run.stdout similarity index 100% rename from tests/integrations/basic-fail/tests/actual_tests/executable.stdout rename to tests/integrations/basic-fail/tests/actual_tests/executable.run.stdout diff --git a/tests/integrations/basic-fail/tests/actual_tests_bless/failing_executable.stderr b/tests/integrations/basic-fail/tests/actual_tests_bless/failing_executable.run.stderr similarity index 100% rename from tests/integrations/basic-fail/tests/actual_tests_bless/failing_executable.stderr rename to tests/integrations/basic-fail/tests/actual_tests_bless/failing_executable.run.stderr diff --git a/tests/integrations/basic-fail/tests/actual_tests_bless/revisioned_executable_panic.panic.stderr b/tests/integrations/basic-fail/tests/actual_tests_bless/revisioned_executable_panic.run.panic.stderr similarity index 100% rename from tests/integrations/basic-fail/tests/actual_tests_bless/revisioned_executable_panic.panic.stderr rename to tests/integrations/basic-fail/tests/actual_tests_bless/revisioned_executable_panic.run.panic.stderr diff --git a/tests/integrations/basic-fail/tests/actual_tests_bless/revisioned_executable_panic.run.stderr b/tests/integrations/basic-fail/tests/actual_tests_bless/revisioned_executable_panic.run.run.stderr similarity index 100% rename from tests/integrations/basic-fail/tests/actual_tests_bless/revisioned_executable_panic.run.stderr rename to tests/integrations/basic-fail/tests/actual_tests_bless/revisioned_executable_panic.run.run.stderr diff --git a/tests/integrations/basic-fail/tests/actual_tests_bless/run_panic.stderr b/tests/integrations/basic-fail/tests/actual_tests_bless/run_panic.run.stderr similarity index 100% rename from tests/integrations/basic-fail/tests/actual_tests_bless/run_panic.stderr rename to tests/integrations/basic-fail/tests/actual_tests_bless/run_panic.run.stderr diff --git a/tests/integrations/basic/tests/actual_tests/executable.stdout b/tests/integrations/basic/tests/actual_tests/executable.run.stdout similarity index 100% rename from tests/integrations/basic/tests/actual_tests/executable.stdout rename to tests/integrations/basic/tests/actual_tests/executable.run.stdout From cd57202f56772b036a771e685d3eb960f6e5a700 Mon Sep 17 00:00:00 2001 From: Oliver Scherer Date: Thu, 14 Sep 2023 21:37:18 +0200 Subject: [PATCH 04/11] Allow run tests to have their own stdin --- src/lib.rs | 4 ++++ tests/integrations/basic/Cargo.stdout | 3 ++- tests/integrations/basic/tests/actual_tests/stdin.rs | 10 ++++++++++ .../basic/tests/actual_tests/stdin.run.stderr | 6 ++++++ .../basic/tests/actual_tests/stdin.run.stdin | 6 ++++++ 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/integrations/basic/tests/actual_tests/stdin.rs create mode 100644 tests/integrations/basic/tests/actual_tests/stdin.run.stderr create mode 100644 tests/integrations/basic/tests/actual_tests/stdin.run.stdin diff --git a/src/lib.rs b/src/lib.rs index 075a28c8..35031c29 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -768,6 +768,10 @@ fn run_test_binary( let file = std::str::from_utf8(file).unwrap(); let exe = config.out_dir.join(file); let mut exe = Command::new(exe); + let stdin = path.with_extension(format!("{revision}.stdin")); + if stdin.exists() { + exe.stdin(std::fs::File::open(stdin).unwrap()); + } let output = exe.output().unwrap(); let mut errors = vec![]; diff --git a/tests/integrations/basic/Cargo.stdout b/tests/integrations/basic/Cargo.stdout index 5cf2900d..10461098 100644 --- a/tests/integrations/basic/Cargo.stdout +++ b/tests/integrations/basic/Cargo.stdout @@ -17,11 +17,12 @@ tests/actual_tests/executable.rs ... ok tests/actual_tests/foomp-rustfix.rs ... ok tests/actual_tests/foomp.rs ... ok tests/actual_tests/no_rustfix.rs ... ok +tests/actual_tests/stdin.rs ... ok tests/actual_tests/unicode.rs ... ok tests/actual_tests/windows_paths.rs ... ok tests/actual_tests/subdir/aux_proc_macro.rs ... ok -test result: ok. 9 passed; +test result: ok. 10 passed; running 0 tests diff --git a/tests/integrations/basic/tests/actual_tests/stdin.rs b/tests/integrations/basic/tests/actual_tests/stdin.rs new file mode 100644 index 00000000..ddbddaee --- /dev/null +++ b/tests/integrations/basic/tests/actual_tests/stdin.rs @@ -0,0 +1,10 @@ +//@run + +fn main() { + let mut n = 0; + for line in std::io::stdin().lines() { + n += 1; + eprintln!("{}", line.unwrap()); + } + assert_eq!(n, 6); +} diff --git a/tests/integrations/basic/tests/actual_tests/stdin.run.stderr b/tests/integrations/basic/tests/actual_tests/stdin.run.stderr new file mode 100644 index 00000000..f39001d7 --- /dev/null +++ b/tests/integrations/basic/tests/actual_tests/stdin.run.stderr @@ -0,0 +1,6 @@ +foo + +bar + +baz + diff --git a/tests/integrations/basic/tests/actual_tests/stdin.run.stdin b/tests/integrations/basic/tests/actual_tests/stdin.run.stdin new file mode 100644 index 00000000..f39001d7 --- /dev/null +++ b/tests/integrations/basic/tests/actual_tests/stdin.run.stdin @@ -0,0 +1,6 @@ +foo + +bar + +baz + From 3532429c66498eadf397015a9d3e1d25881e09cc Mon Sep 17 00:00:00 2001 From: Oliver Scherer Date: Thu, 14 Sep 2023 21:39:35 +0200 Subject: [PATCH 05/11] Document .stdin --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 40dcc5ec..cd827d09 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ A smaller version of compiletest-rs So if you have any slow tests, prepend them with a small integral number to make them get run first, taking advantage of parallelism as much as possible (instead of waiting for the slow tests at the end). * `cargo test --test your_test_name -- --help` lists the commands you can specify for filtering, blessing and making your tests less verbose. * Since `cargo test` on its own runs all tests, using `cargo test -- --check` will not work on its own, but `cargo test -- --quiet` and `cargo test -- some_test_name` will work just fine, as the CLI matches. +* if there is a `.stdin` file with the same filename as your test, it will be piped as standard input to your program. ## Supported magic comment annotations @@ -53,6 +54,8 @@ their command specifies, or the test will fail without even being run. * `//@aux-build: filename` looks for a file in the `auxiliary` directory (within the directory of the test), compiles it as a library and links the current crate against it. This allows you import the crate with `extern crate` or just via `use` statements. This will automatically detect aux files that are proc macros and build them as proc macros. * `//@run` compiles the test and runs the resulting binary. The resulting binary must exit successfully. Stdout and stderr are taken from the resulting binary. Any warnings during compilation are ignored. * You can also specify a different exit code/status that is expected via e.g. `//@run: 1` or `//@run: 101` (the latter is the standard Rust exit code for panics). + * run tests collect the run output into `.run.stderr` and `.run.stdout` respectively. + * if a `.run.stdin` file exists, it will be piped as standard input to your test's execution. [rustfix]: https://github.com/rust-lang/rustfix From b3c9cfe113d6a4832381fef3f5ff8f6b0f3c8c4d Mon Sep 17 00:00:00 2001 From: Oliver Scherer Date: Thu, 14 Sep 2023 21:57:37 +0200 Subject: [PATCH 06/11] Maybe the tests are overwriting each other? --- tests/integrations/cargo-run/tests/ui_tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integrations/cargo-run/tests/ui_tests.rs b/tests/integrations/cargo-run/tests/ui_tests.rs index 0613148a..aa03b4bf 100644 --- a/tests/integrations/cargo-run/tests/ui_tests.rs +++ b/tests/integrations/cargo-run/tests/ui_tests.rs @@ -13,7 +13,6 @@ fn main() -> ui_test::color_eyre::Result<()> { config.program.args = vec!["run".into(), "--quiet".into()]; config.program.input_file_flag = Some("--".into()); - config.program.out_dir_flag = None; run_tests_generic( vec![config], From d16ddab01063c3be1cdccdbbdd48c8f50eb5c0b7 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Fri, 15 Sep 2023 19:50:15 +0200 Subject: [PATCH 07/11] Add nix support --- .vscode/settings.json | 3 ++- shell.nix | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 shell.nix diff --git a/.vscode/settings.json b/.vscode/settings.json index e11c8984..73f6be6f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,5 +5,6 @@ "./tests/integrations/basic-fail/Cargo.toml", "./tests/integrations/basic-fail-mode/Cargo.toml", "./tests/integrations/cargo-run/Cargo.toml" - ] + ], + "nixEnvSelector.nixFile": "${workspaceRoot}/shell.nix" } diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000..8511dcca --- /dev/null +++ b/shell.nix @@ -0,0 +1,18 @@ +let + pkgs = import {}; +in +pkgs.mkShell rec { + name = "rustc"; + buildInputs = with pkgs; [ + rustup + pkg-config + alsaLib + libGL + xorg.libX11 + xorg.libXi + python39 + ]; + RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}"; + LD_LIBRARY_PATH = "${pkgs.lib.makeLibraryPath buildInputs}"; +} + From bbabefdedfcf430ef2a9161b67559cfa64386526 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Sat, 16 Sep 2023 12:14:18 +0200 Subject: [PATCH 08/11] Make it easier to see where the wrong regex is wrong --- src/config.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/config.rs b/src/config.rs index ef72de9b..c3837923 100644 --- a/src/config.rs +++ b/src/config.rs @@ -166,12 +166,14 @@ impl Config { } /// Replace all occurrences of a path in stderr/stdout with a byte string. + #[track_caller] pub fn path_filter(&mut self, path: &Path, replacement: &'static (impl AsRef<[u8]> + ?Sized)) { self.path_stderr_filter(path, replacement); self.path_stdout_filter(path, replacement); } /// Replace all occurrences of a path in stderr with a byte string. + #[track_caller] pub fn path_stderr_filter( &mut self, path: &Path, @@ -183,6 +185,7 @@ impl Config { } /// Replace all occurrences of a path in stdout with a byte string. + #[track_caller] pub fn path_stdout_filter( &mut self, path: &Path, @@ -194,12 +197,14 @@ impl Config { } /// Replace all occurrences of a regex pattern in stderr/stdout with a byte string. + #[track_caller] pub fn filter(&mut self, pattern: &str, replacement: &'static (impl AsRef<[u8]> + ?Sized)) { self.stderr_filter(pattern, replacement); self.stdout_filter(pattern, replacement); } /// Replace all occurrences of a regex pattern in stderr with a byte string. + #[track_caller] pub fn stderr_filter( &mut self, pattern: &str, @@ -210,6 +215,7 @@ impl Config { } /// Replace all occurrences of a regex pattern in stdout with a byte string. + #[track_caller] pub fn stdout_filter( &mut self, pattern: &str, From 56513650253ca371b28d7f3e9d48e33a65c2c84a Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Sat, 16 Sep 2023 12:15:46 +0200 Subject: [PATCH 09/11] Ensure independence of individual tests --- src/lib.rs | 24 +++++++++++++++------- tests/integration.rs | 4 ++-- tests/integrations/basic-fail/Cargo.stderr | 6 +++--- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 35031c29..e7fefa99 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -627,7 +627,15 @@ impl dyn TestStatus { build_manager, )?; - let mut cmd = build_command(path, config, revision, comments)?; + let mut config = config.clone(); + + // Put aux builds into a separate directory per path so that multiple aux files + // from different directories (but with the same file name) don't collide. + let relative = strip_path_prefix(path.parent().unwrap(), &config.out_dir); + + config.out_dir.extend(relative); + + let mut cmd = build_command(path, &config, revision, comments)?; cmd.args(&extra_args); let stdin = path.with_extension("stdin"); if stdin.exists() { @@ -644,7 +652,7 @@ impl dyn TestStatus { _ => *mode, }, path, - config, + &config, revision, comments, status, @@ -653,11 +661,11 @@ impl dyn TestStatus { )?; if let Mode::Run { .. } = *mode { - return run_test_binary(mode, path, revision, comments, cmd, config); + return run_test_binary(mode, path, revision, comments, cmd, &config); } run_rustfix( - &stderr, &stdout, path, comments, revision, config, *mode, extra_args, + &stderr, &stdout, path, comments, revision, &config, *mode, extra_args, )?; Ok(TestOk::Ok) } @@ -766,13 +774,15 @@ fn run_test_binary( let file = files.next().unwrap(); assert_eq!(files.next(), None); let file = std::str::from_utf8(file).unwrap(); - let exe = config.out_dir.join(file); - let mut exe = Command::new(exe); + let exe_file = config.out_dir.join(file); + let mut exe = Command::new(&exe_file); let stdin = path.with_extension(format!("{revision}.stdin")); if stdin.exists() { exe.stdin(std::fs::File::open(stdin).unwrap()); } - let output = exe.output().unwrap(); + let output = exe + .output() + .unwrap_or_else(|err| panic!("exe file: {}: {err}", exe_file.display())); let mut errors = vec![]; diff --git a/tests/integration.rs b/tests/integration.rs index d2259557..7e188287 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -34,8 +34,8 @@ fn main() -> Result<()> { config.stdout_filter("in ([0-9]m )?[0-9\\.]+s", ""); config.stdout_filter(r#""--out-dir"(,)? "[^"]+""#, r#""--out-dir"$1 "$$TMP"#); config.filter( - "( *process didn't exit successfully: `[^-]+)-[0-9a-f]+", - "$1-HASH", + "( *process didn't exit successfully: `.*)-[0-9a-f]+`", + "$1-HASH`", ); // Windows io::Error uses "exit code". config.filter("exit code", "exit status"); diff --git a/tests/integrations/basic-fail/Cargo.stderr b/tests/integrations/basic-fail/Cargo.stderr index f6e7319d..5dcec8ab 100644 --- a/tests/integrations/basic-fail/Cargo.stderr +++ b/tests/integrations/basic-fail/Cargo.stderr @@ -5,7 +5,7 @@ Location: error: test failed, to rerun pass `--test ui_tests` Caused by: - process didn't exit successfully: `$DIR/target/ui/debug/deps/ui_tests-HASH` (exit status: 1) + process didn't exit successfully: `$DIR/target/ui/$DIR/debug/deps/ui_tests-HASH` (exit status: 1) thread 'main' panicked at 'invalid mode/result combo: yolo: Err(tests failed Location: @@ -21,7 +21,7 @@ Location: error: test failed, to rerun pass `--test ui_tests_invalid_program` Caused by: - process didn't exit successfully: `$DIR/target/ui/debug/deps/ui_tests_invalid_program-HASH` (exit status: 1) + process didn't exit successfully: `$DIR/target/ui/$DIR/debug/deps/ui_tests_invalid_program-HASH` (exit status: 1) Error: tests failed Location: @@ -29,7 +29,7 @@ Location: error: test failed, to rerun pass `--test ui_tests_invalid_program2` Caused by: - process didn't exit successfully: `$DIR/target/ui/debug/deps/ui_tests_invalid_program2-HASH` (exit status: 1) + process didn't exit successfully: `$DIR/target/ui/$DIR/debug/deps/ui_tests_invalid_program2-HASH` (exit status: 1) error: 4 targets failed: `--test ui_tests` `--test ui_tests_bless` From 1a356b2673dbaa9979e557b1fb598dedf1f401e9 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Sat, 16 Sep 2023 14:55:51 +0200 Subject: [PATCH 10/11] Filter away `.exe` line endings before running other filters --- tests/integration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration.rs b/tests/integration.rs index 7e188287..a39a15ee 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -33,6 +33,7 @@ fn main() -> Result<()> { config.stdout_filter("in ([0-9]m )?[0-9\\.]+s", ""); config.stdout_filter(r#""--out-dir"(,)? "[^"]+""#, r#""--out-dir"$1 "$$TMP"#); + config.filter("\\.exe", b""); config.filter( "( *process didn't exit successfully: `.*)-[0-9a-f]+`", "$1-HASH`", @@ -54,7 +55,6 @@ fn main() -> Result<()> { config .stdout_filters .insert(0, (Match::Exact(b"\\\\".to_vec()), b"\\")); - config.filter("\\.exe", b""); config.stdout_filter(r#"(panic.*)\.rs:[0-9]+:[0-9]+"#, "$1.rs"); config.filter("(\\+)? +[0-9]+: .*\n", ""); config.filter("(\\+)? +at /.*\n", ""); From ad972c3014fbed9f07e3123aedf6eb5b35427b92 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Sat, 16 Sep 2023 14:55:58 +0200 Subject: [PATCH 11/11] Version bump --- Cargo.lock | 2 +- Cargo.toml | 2 +- tests/integrations/basic-bin/Cargo.lock | 2 +- tests/integrations/basic-fail-mode/Cargo.lock | 2 +- tests/integrations/basic-fail/Cargo.lock | 2 +- tests/integrations/basic/Cargo.lock | 2 +- tests/integrations/cargo-run/Cargo.lock | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 529fe3e1..9da8ba5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -651,7 +651,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.20.0" +version = "0.20.1" dependencies = [ "annotate-snippets", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index d499df74..d2d4961c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ui_test" -version = "0.20.0" +version = "0.20.1" edition = "2021" license = "MIT OR Apache-2.0" description = "A test framework for testing rustc diagnostics output" diff --git a/tests/integrations/basic-bin/Cargo.lock b/tests/integrations/basic-bin/Cargo.lock index 2ae30b52..43a4ce8d 100644 --- a/tests/integrations/basic-bin/Cargo.lock +++ b/tests/integrations/basic-bin/Cargo.lock @@ -697,7 +697,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.20.0" +version = "0.20.1" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic-fail-mode/Cargo.lock b/tests/integrations/basic-fail-mode/Cargo.lock index 8542b0f8..7e4c8404 100644 --- a/tests/integrations/basic-fail-mode/Cargo.lock +++ b/tests/integrations/basic-fail-mode/Cargo.lock @@ -697,7 +697,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.20.0" +version = "0.20.1" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic-fail/Cargo.lock b/tests/integrations/basic-fail/Cargo.lock index 42e8b316..ad542c14 100644 --- a/tests/integrations/basic-fail/Cargo.lock +++ b/tests/integrations/basic-fail/Cargo.lock @@ -697,7 +697,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.20.0" +version = "0.20.1" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/basic/Cargo.lock b/tests/integrations/basic/Cargo.lock index 9699eb27..ffe002fe 100644 --- a/tests/integrations/basic/Cargo.lock +++ b/tests/integrations/basic/Cargo.lock @@ -697,7 +697,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.20.0" +version = "0.20.1" dependencies = [ "annotate-snippets", "anyhow", diff --git a/tests/integrations/cargo-run/Cargo.lock b/tests/integrations/cargo-run/Cargo.lock index 2ae30b52..43a4ce8d 100644 --- a/tests/integrations/cargo-run/Cargo.lock +++ b/tests/integrations/cargo-run/Cargo.lock @@ -697,7 +697,7 @@ dependencies = [ [[package]] name = "ui_test" -version = "0.20.0" +version = "0.20.1" dependencies = [ "annotate-snippets", "anyhow",