Skip to content

Commit

Permalink
twister: add option to ignore skipped tests in reports
Browse files Browse the repository at this point in the history 
With --no-skipped-report, twister will skip over all tests marked as
skipped (filtered) in the junit report.

This is useful for CI where have 1000s of filtered tests that appear in
the report and in some cases cause tools parsing the output to fail or
provide incomplete results.

Fixes #38179

Signed-off-by: Anas Nashif <[email protected]>
  • Loading branch information
@nashif
nashif committed Mar 3, 2022
1 parent 923e491 commit dd651c0
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 20 deletions.
38 changes: 19 additions & 19 deletions scripts/pylib/twister/twisterlib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2980,7 +2980,7 @@ def summary(self, results, unrecognized_sections):
logger.info(f"{Fore.GREEN}{run}{Fore.RESET} test configurations executed on platforms, \
{Fore.RED}{results.total - run - results.skipped_configs}{Fore.RESET} test configurations were only built.")

def save_reports(self, name, suffix, report_dir, no_update, release, only_failed, platform_reports, json_report):
def save_reports(self, name, suffix, report_dir, no_update, release, only_failed, platform_reports, json_report, report_skipped):
if not self.instances:
return

Expand All @@ -3003,9 +3003,9 @@ def save_reports(self, name, suffix, report_dir, no_update, release, only_failed

if not no_update:
self.xunit_report(filename + ".xml", full_report=False,
append=only_failed, version=self.version)
append=only_failed, version=self.version, report_skipped=report_skipped)
self.xunit_report(filename + "_report.xml", full_report=True,
append=only_failed, version=self.version)
append=only_failed, version=self.version, report_skipped=report_skipped)
self.csv_report(filename + ".csv")

if json_report:
Expand Down Expand Up @@ -3588,15 +3588,15 @@ def discard_report(self, filename):
"reason": reason}
cw.writerow(rowdict)

def target_report(self, outdir, suffix, append=False):
def target_report(self, outdir, suffix, append=False, report_skipped=True):
platforms = {inst.platform.name for _, inst in self.instances.items()}
for platform in platforms:
if suffix:
filename = os.path.join(outdir,"{}_{}.xml".format(platform, suffix))
else:
filename = os.path.join(outdir,"{}.xml".format(platform))
self.xunit_report(filename, platform, full_report=True,
append=append, version=self.version)
append=append, version=self.version, report_skipped=report_skipped)


@staticmethod
Expand All @@ -3610,7 +3610,7 @@ def process_log(log_file):
return filtered_string


def xunit_report(self, filename, platform=None, full_report=False, append=False, version="NA"):
def xunit_report(self, filename, platform=None, full_report=False, append=False, version="NA", report_skipped=True):
total = 0
fails = passes = errors = skips = 0
if platform:
Expand Down Expand Up @@ -3670,6 +3670,8 @@ def xunit_report(self, filename, platform=None, full_report=False, append=False,

run = p
eleTestsuite = None
if not report_skipped and total == skips:
continue

# When we re-run the tests, we re-use the results and update only with
# the newly run tests.
Expand All @@ -3687,23 +3689,21 @@ def xunit_report(self, filename, platform=None, full_report=False, append=False,
tests="%d" % (total),
failures="%d" % fails,
errors="%d" % (errors), skipped="%s" % (skips))
eleTSPropetries = ET.SubElement(eleTestsuite, 'properties')
# Multiple 'property' can be added to 'properties'
# differing by name and value
ET.SubElement(eleTSPropetries, 'property', name="version", value=version)

else:
eleTestsuite = ET.SubElement(eleTestsuites, 'testsuite',
name=run, time="%f" % duration,
tests="%d" % (total),
failures="%d" % fails,
errors="%d" % (errors), skipped="%s" % (skips))
eleTSPropetries = ET.SubElement(eleTestsuite, 'properties')
# Multiple 'property' can be added to 'properties'
# differing by name and value
ET.SubElement(eleTSPropetries, 'property', name="version", value=version)
name=run, time="%f" % duration,
tests="%d" % (total),
failures="%d" % fails,
errors="%d" % (errors), skipped="%s" % (skips))

eleTSPropetries = ET.SubElement(eleTestsuite, 'properties')
# Multiple 'property' can be added to 'properties'
# differing by name and value
ET.SubElement(eleTSPropetries, 'property', name="version", value=version)

for _, instance in inst.items():
if instance.status == 'skipped' and not report_skipped:
continue
if full_report:
tname = os.path.basename(instance.testcase.name)
else:
Expand Down
8 changes: 7 additions & 1 deletion scripts/twister
View file
Original file line number Diff line number Diff line change
Expand Up @@ -577,6 +577,11 @@ structure in the main Zephyr tree: boards/<arch>/<board_name>/""")
help="Re-use the outdir before building. Will result in "
"faster compilation since builds will be incremental.")

parser.add_argument(
"--no-skipped-report", action="store_true",
help="""Do not report skipped test cases in junit output. [Experimental]
""")

parser.add_argument(
"-O", "--outdir",
default=os.path.join(os.getcwd(), "twister-out"),
Expand Down Expand Up @@ -1359,7 +1364,8 @@ def main():
options.release,
options.only_failed,
options.platform_reports,
options.json_report
options.json_report,
not options.no_skipped_report
)

# FIXME: remove later
Expand Down

0 comments on commit dd651c0

Please sign in to comment.