From d77f636c6339d97ef3fa790f974d56b182cd59d1 Mon Sep 17 00:00:00 2001 From: Augie Fackler Date: Fri, 21 Apr 2023 12:48:16 -0400 Subject: [PATCH 1/3] libtest: add tests for junit output format I'm about to make some changes here, and it was making me uneasy to modify the output format without test coverage. --- tests/run-make/libtest-junit/Makefile | 19 +++++++++++++++ tests/run-make/libtest-junit/f.rs | 23 +++++++++++++++++++ .../run-make/libtest-junit/output-default.xml | 1 + .../libtest-junit/output-stdout-success.xml | 1 + .../run-make/libtest-junit/validate_junit.py | 12 ++++++++++ 5 files changed, 56 insertions(+) create mode 100644 tests/run-make/libtest-junit/Makefile create mode 100644 tests/run-make/libtest-junit/f.rs create mode 100644 tests/run-make/libtest-junit/output-default.xml create mode 100644 tests/run-make/libtest-junit/output-stdout-success.xml create mode 100755 tests/run-make/libtest-junit/validate_junit.py diff --git a/tests/run-make/libtest-junit/Makefile b/tests/run-make/libtest-junit/Makefile new file mode 100644 index 0000000000000..d97cafccf1fd4 --- /dev/null +++ b/tests/run-make/libtest-junit/Makefile @@ -0,0 +1,19 @@ +# ignore-cross-compile +include ../tools.mk + +# Test expected libtest's junit output + +OUTPUT_FILE_DEFAULT := $(TMPDIR)/libtest-junit-output-default.xml +OUTPUT_FILE_STDOUT_SUCCESS := $(TMPDIR)/libtest-junit-output-stdout-success.xml + +all: f.rs validate_junit.py output-default.xml output-stdout-success.xml + $(RUSTC) --test f.rs + RUST_BACKTRACE=0 $(call RUN,f) -Z unstable-options --test-threads=1 --format=junit > $(OUTPUT_FILE_DEFAULT) || true + RUST_BACKTRACE=0 $(call RUN,f) -Z unstable-options --test-threads=1 --format=junit --show-output > $(OUTPUT_FILE_STDOUT_SUCCESS) || true + + cat $(OUTPUT_FILE_DEFAULT) | "$(PYTHON)" validate_junit.py + cat $(OUTPUT_FILE_STDOUT_SUCCESS) | "$(PYTHON)" validate_junit.py + + # Normalize the actual output and compare to expected output file + cat $(OUTPUT_FILE_DEFAULT) | sed 's/time="[0-9.]*"/time="$$TIME"/g' | diff output-default.xml - + cat $(OUTPUT_FILE_STDOUT_SUCCESS) | sed 's/time="[0-9.]*"/time="$$TIME"/g' | diff output-stdout-success.xml - diff --git a/tests/run-make/libtest-junit/f.rs b/tests/run-make/libtest-junit/f.rs new file mode 100644 index 0000000000000..d360d77317d78 --- /dev/null +++ b/tests/run-make/libtest-junit/f.rs @@ -0,0 +1,23 @@ +#[test] +fn a() { + println!("print from successful test"); + // Should pass +} + +#[test] +fn b() { + println!("print from failing test"); + assert!(false); +} + +#[test] +#[should_panic] +fn c() { + assert!(false); +} + +#[test] +#[ignore = "msg"] +fn d() { + assert!(false); +} diff --git a/tests/run-make/libtest-junit/output-default.xml b/tests/run-make/libtest-junit/output-default.xml new file mode 100644 index 0000000000000..ea61562a33ccc --- /dev/null +++ b/tests/run-make/libtest-junit/output-default.xml @@ -0,0 +1 @@ + diff --git a/tests/run-make/libtest-junit/output-stdout-success.xml b/tests/run-make/libtest-junit/output-stdout-success.xml new file mode 100644 index 0000000000000..ea61562a33ccc --- /dev/null +++ b/tests/run-make/libtest-junit/output-stdout-success.xml @@ -0,0 +1 @@ + diff --git a/tests/run-make/libtest-junit/validate_junit.py b/tests/run-make/libtest-junit/validate_junit.py new file mode 100755 index 0000000000000..47a8e70ccc38c --- /dev/null +++ b/tests/run-make/libtest-junit/validate_junit.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +import sys +import xml.etree.ElementTree as ET + +# Try to decode line in order to ensure it is a valid XML document +for line in sys.stdin: + try: + ET.fromstring(line) + except ET.ParseError as pe: + print("Invalid xml: %r" % line) + raise From 610f82726172c26d0a9f72fee3d72caad9bdb514 Mon Sep 17 00:00:00 2001 From: Augie Fackler Date: Fri, 21 Apr 2023 10:45:17 -0400 Subject: [PATCH 2/3] junit: also include per-case stdout in xml By placing the stdout in a CDATA block we avoid almost all escaping, as there's only two byte sequences you can't sneak into a CDATA and you can handle that with some only slightly regrettable CDATA-splitting. I've done this in at least two other implementations of the junit xml format over the years and it's always worked out. The only quirk new to this (for me) is smuggling newlines as to avoid literal newlines in the output. --- library/test/src/formatters/junit.rs | 40 ++++++++++++++++--- .../run-make/libtest-junit/output-default.xml | 2 +- .../libtest-junit/output-stdout-success.xml | 2 +- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/library/test/src/formatters/junit.rs b/library/test/src/formatters/junit.rs index 2e07ce3c09923..65883dd36a169 100644 --- a/library/test/src/formatters/junit.rs +++ b/library/test/src/formatters/junit.rs @@ -11,7 +11,7 @@ use crate::{ pub struct JunitFormatter { out: OutputLocation, - results: Vec<(TestDesc, TestResult, Duration)>, + results: Vec<(TestDesc, TestResult, Duration, Vec)>, } impl JunitFormatter { @@ -26,6 +26,18 @@ impl JunitFormatter { } } +fn str_to_cdata(s: &str) -> String { + // Drop the stdout in a cdata. Unfortunately, you can't put either of `]]>` or + // `", "]]]]>"); + let escaped_output = escaped_output.replace(" ", ""); + format!("", escaped_output) +} + impl OutputFormatter for JunitFormatter { fn write_discovery_start(&mut self) -> io::Result<()> { Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!")) @@ -63,14 +75,14 @@ impl OutputFormatter for JunitFormatter { desc: &TestDesc, result: &TestResult, exec_time: Option<&time::TestExecTime>, - _stdout: &[u8], + stdout: &[u8], _state: &ConsoleTestState, ) -> io::Result<()> { // Because the testsuite node holds some of the information as attributes, we can't write it // until all of the tests have finished. Instead of writing every result as they come in, we add // them to a Vec and write them all at once when run is complete. let duration = exec_time.map(|t| t.0).unwrap_or_default(); - self.results.push((desc.clone(), result.clone(), duration)); + self.results.push((desc.clone(), result.clone(), duration, stdout.to_vec())); Ok(()) } fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { @@ -85,7 +97,7 @@ impl OutputFormatter for JunitFormatter { >", state.failed, state.total, state.ignored ))?; - for (desc, result, duration) in std::mem::take(&mut self.results) { + for (desc, result, duration, stdout) in std::mem::take(&mut self.results) { let (class_name, test_name) = parse_class_name(&desc); match result { TestResult::TrIgnored => { /* no-op */ } @@ -98,6 +110,11 @@ impl OutputFormatter for JunitFormatter { duration.as_secs_f64() ))?; self.write_message("")?; + if !stdout.is_empty() { + self.write_message("")?; + self.write_message(&str_to_cdata(&String::from_utf8_lossy(&stdout)))?; + self.write_message("")?; + } self.write_message("")?; } @@ -110,6 +127,11 @@ impl OutputFormatter for JunitFormatter { duration.as_secs_f64() ))?; self.write_message(&format!(""))?; + if !stdout.is_empty() { + self.write_message("")?; + self.write_message(&str_to_cdata(&String::from_utf8_lossy(&stdout)))?; + self.write_message("")?; + } self.write_message("")?; } @@ -136,11 +158,19 @@ impl OutputFormatter for JunitFormatter { TestResult::TrOk => { self.write_message(&format!( "", + name=\"{}\" time=\"{}\"", class_name, test_name, duration.as_secs_f64() ))?; + if stdout.is_empty() { + self.write_message("/>")?; + } else { + self.write_message(">")?; + self.write_message(&str_to_cdata(&String::from_utf8_lossy(&stdout)))?; + self.write_message("")?; + self.write_message("")?; + } } } } diff --git a/tests/run-make/libtest-junit/output-default.xml b/tests/run-make/libtest-junit/output-default.xml index ea61562a33ccc..0c300611e1f76 100644 --- a/tests/run-make/libtest-junit/output-default.xml +++ b/tests/run-make/libtest-junit/output-default.xml @@ -1 +1 @@ - + diff --git a/tests/run-make/libtest-junit/output-stdout-success.xml b/tests/run-make/libtest-junit/output-stdout-success.xml index ea61562a33ccc..0c300611e1f76 100644 --- a/tests/run-make/libtest-junit/output-stdout-success.xml +++ b/tests/run-make/libtest-junit/output-stdout-success.xml @@ -1 +1 @@ - + From 58537cde0607eb3ca4076e00b8e2cfac52827ca1 Mon Sep 17 00:00:00 2001 From: Augie Fackler Date: Fri, 28 Apr 2023 18:37:26 -0400 Subject: [PATCH 3/3] junit: fix typo in comment and don't include output for passes when not requested --- library/test/src/formatters/junit.rs | 4 ++-- tests/run-make/libtest-junit/output-default.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/library/test/src/formatters/junit.rs b/library/test/src/formatters/junit.rs index 65883dd36a169..9f5bf24367eaa 100644 --- a/library/test/src/formatters/junit.rs +++ b/library/test/src/formatters/junit.rs @@ -31,7 +31,7 @@ fn str_to_cdata(s: &str) -> String { // `", "]]]]>"); let escaped_output = escaped_output.replace(" ", ""); @@ -163,7 +163,7 @@ impl OutputFormatter for JunitFormatter { test_name, duration.as_secs_f64() ))?; - if stdout.is_empty() { + if stdout.is_empty() || !state.options.display_output { self.write_message("/>")?; } else { self.write_message(">")?; diff --git a/tests/run-make/libtest-junit/output-default.xml b/tests/run-make/libtest-junit/output-default.xml index 0c300611e1f76..d59e07b8ad89d 100644 --- a/tests/run-make/libtest-junit/output-default.xml +++ b/tests/run-make/libtest-junit/output-default.xml @@ -1 +1 @@ - +