Skip to content

Commit

Permalink
Enhance "Total test files" report
Browse files Browse the repository at this point in the history
Count "rerun" tests rather than "failed" tests. Also add the number
of tests in "run=count/total".
  • Loading branch information
vstinner committed Sep 3, 2023
1 parent dda3b23 commit e132cde
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 14 deletions.
27 changes: 18 additions & 9 deletions Lib/test/libregrtest/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def __init__(self):
# tests
self.tests = []
self.selected = []
self.all_runtests: list[RunTests] = []

# test results
self.good = []
Expand Down Expand Up @@ -350,6 +351,7 @@ def _rerun_failed_tests(self, need_rerun):
# Re-run failed tests
self.log(f"Re-running {len(tests)} failed tests in verbose mode in subprocesses")
runtests = RunTests(tests, match_tests=match_tests, rerun=True)
self.all_runtests.append(runtests)
self._run_tests_mp(runtests)

def rerun_failed_tests(self, need_rerun):
Expand Down Expand Up @@ -621,6 +623,7 @@ def run_tests(self):
tests = self.selected
self.set_tests(tests)
runtests = RunTests(tests, forever=self.ns.forever)
self.all_runtests.append(runtests)
if self.ns.use_mp:
self._run_tests_mp(runtests)
else:
Expand All @@ -646,6 +649,9 @@ def finalize(self):

def display_summary(self):
duration = time.perf_counter() - self.start_time
first_runtests = self.all_runtests[0]
# re-run tests disables forever
forever = first_runtests.forever

# Total duration
print()
Expand All @@ -661,22 +667,25 @@ def display_summary(self):
print(f"Total tests: {' '.join(stats)}")

# Total test files
all_tests = [self.good, self.bad, self.rerun_bad,
all_tests = [self.good, self.bad, self.rerun,
self.skipped,
self.environment_changed, self.run_no_tests]
run = sum(map(len, all_tests))
report = [f'run={run}']
bad = len(self.bad) + len(self.rerun_bad)
text = f'run={run}'
if not forever:
text = f"{text}/{len(self.tests)}"
report = [text]
bad = len(self.bad)
if bad:
report.append(f'failed={bad}')
if self.environment_changed:
report.append(f'env_changed={len(self.environment_changed)}')
skipped = len(self.skipped)
if skipped:
report.append(f'skipped={skipped}')
resource_denied = len(self.resource_denied)
if resource_denied:
report.append(f'resource_denied={resource_denied}')
if self.skipped:
report.append(f'skipped={self.skipped}')
if self.resource_denied:
report.append(f'resource_denied={len(self.resource_denied)}')
if self.rerun:
report.append(f'rerun={len(self.rerun)}')
if self.run_no_tests:
report.append(f'run_no_tests={len(self.run_no_tests)}')
print(f"Total test files: {' '.join(report)}")
Expand Down
22 changes: 17 additions & 5 deletions Lib/test/test_regrtest.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ def check_executed_tests(self, output, tests, skipped=(), failed=(),
resource_denied=(),
randomize=False, interrupted=False,
fail_env_changed=False,
*, stats):
*, stats, forever=False):
if isinstance(tests, str):
tests = [tests]
if isinstance(skipped, str):
Expand Down Expand Up @@ -536,10 +536,18 @@ def list_regex(line_format, tests):

# Total test files
run = len(total_tests) - len(resource_denied)
if rerun is not None:
total_failed = len(rerun_failed)
total_rerun = 1
else:
total_failed = len(failed)
total_rerun = 0
if interrupted:
run = 0
report = [f'run={run}']
total_failed = len(failed) + len(rerun_failed)
text = f'run={run}'
if not forever:
text = f'{text}/{len(tests)}'
report = [text]
if total_failed:
report.append(f'failed={total_failed}')
if env_changed:
Expand All @@ -548,6 +556,8 @@ def list_regex(line_format, tests):
report.append(f'skipped={len(skipped)}')
if resource_denied:
report.append(f'resource_denied={len(resource_denied)}')
if total_rerun:
report.append(f'rerun={total_rerun}')
if run_no_tests:
report.append(f'run_no_tests={len(run_no_tests)}')
line = fr'Total test files: {" ".join(report)}'
Expand Down Expand Up @@ -958,15 +968,17 @@ def test_run(self):
# --forever
output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, [test]*3, failed=test,
stats=TestStats(3, 1))
stats=TestStats(3, 1),
forever=True)

# --forever --rerun
output = self.run_tests('--forever', '--rerun', test, exitcode=0)
self.check_executed_tests(output, [test]*3,
rerun=Rerun(test,
match='test_run',
success=True),
stats=TestStats(4, 1))
stats=TestStats(4, 1),
forever=True)

def check_leak(self, code, what):
test = self.create_test('huntrleaks', code=code)
Expand Down

0 comments on commit e132cde

Please sign in to comment.