From 5075146a1aab6966f166bbf57540de2523a600ab Mon Sep 17 00:00:00 2001 From: Zhenyu Yao <111329301+zhenyu-ms@users.noreply.github.com> Date: Thu, 20 Jun 2024 11:58:08 +0800 Subject: [PATCH] Fix/multitest parting revision (#1097) change parting to case-level round-robin --- .../2755_changed.multitest_parting.rst | 1 + testplan/testing/multitest/base.py | 59 +++++++++++-------- .../testing/multitest/test_multitest_parts.py | 47 ++++++++++++--- .../testplan/testing/test_filtering.py | 8 +-- .../testplan/testing/test_listing.py | 12 ++-- 5 files changed, 86 insertions(+), 41 deletions(-) create mode 100644 doc/newsfragments/2755_changed.multitest_parting.rst diff --git a/doc/newsfragments/2755_changed.multitest_parting.rst b/doc/newsfragments/2755_changed.multitest_parting.rst new file mode 100644 index 000000000..bb2f3a847 --- /dev/null +++ b/doc/newsfragments/2755_changed.multitest_parting.rst @@ -0,0 +1 @@ +Testcases of ``MultiTest`` can now be distributed more evenly across parts. \ No newline at end of file diff --git a/testplan/testing/multitest/base.py b/testplan/testing/multitest/base.py index 6d1317937..05c041f7f 100644 --- a/testplan/testing/multitest/base.py +++ b/testplan/testing/multitest/base.py @@ -392,6 +392,11 @@ def get_test_context(self): ctx = [] sorted_suites = self.cfg.test_sorter.sorted_testsuites(self.cfg.suites) + if hasattr(self.cfg, "xfail_tests") and self.cfg.xfail_tests: + xfail_data = self.cfg.xfail_tests + else: + xfail_data = {} + for suite in sorted_suites: testcases = suite.get_testcases() @@ -402,13 +407,6 @@ def get_test_context(self): else self.cfg.test_sorter.sorted_testcases(suite, testcases) ) - if self.cfg.part: - sorted_testcases = [ - testcase - for (idx, testcase) in enumerate(sorted_testcases) - if (idx) % self.cfg.part[1] == self.cfg.part[0] - ] - testcases_to_run = [ case for case in sorted_testcases @@ -425,26 +423,39 @@ def get_test_context(self): testcases_to_run = sorted_testcases if testcases_to_run: - if hasattr(self.cfg, "xfail_tests") and self.cfg.xfail_tests: - for testcase in testcases_to_run: - testcase_instance = ":".join( - [ - self.name, - suite.name, - testcase.name, - ] - ) - data = self.cfg.xfail_tests.get( - testcase_instance, None - ) - if data is not None: - testcase.__func__.__xfail__ = { - "reason": data["reason"], - "strict": data["strict"], - } + for testcase in testcases_to_run: + testcase_instance = ":".join( + [ + self.name, + suite.name, + testcase.name, + ] + ) + data = xfail_data.get(testcase_instance, None) + if data is not None: + testcase.__func__.__xfail__ = { + "reason": data["reason"], + "strict": data["strict"], + } ctx.append((suite, testcases_to_run)) + if self.cfg.part: + # round-robin at testcase level + numer, denom = self.cfg.part + ofst = 0 + ctx_ = [] + for suite, cases in ctx: + cases_ = [ + case + for idx, case in enumerate(cases) + if (idx + ofst) % denom == numer + ] + ofst = (ofst + len(cases)) % denom + if cases_: + ctx_.append((suite, cases_)) + return ctx_ + return ctx def _dry_run_testsuites(self): diff --git a/tests/functional/testplan/testing/multitest/test_multitest_parts.py b/tests/functional/testplan/testing/multitest/test_multitest_parts.py index 530fe347b..18d27120d 100644 --- a/tests/functional/testplan/testing/multitest/test_multitest_parts.py +++ b/tests/functional/testplan/testing/multitest/test_multitest_parts.py @@ -1,11 +1,11 @@ -import itertools - -from testplan.testing.multitest import MultiTest, testsuite, testcase +from itertools import chain, cycle, repeat +from operator import eq from testplan import TestplanMock +from testplan.report import Status from testplan.runners.pools.base import Pool as ThreadPool from testplan.runners.pools.tasks import Task -from testplan.report import Status +from testplan.testing.multitest import MultiTest, testcase, testsuite @testsuite @@ -44,16 +44,19 @@ def _post_run_checks(self, start_threads, start_procs): raise RuntimeError("Deliberately raises") -uid_gen = itertools.cycle([i for i in range(10)]) - - def get_mtest(part_tuple=None): return MultiTest( name="MTest", suites=[Suite1(), Suite2()], part=part_tuple ) +uid_gen = cycle([i for i in range(10)]) + + def get_mtest_with_custom_uid(part_tuple=None): + # XXX: abolish multi_part_uid, may rename it to multi_part_report_name? + # XXX: or we may still accept customised uids, but we need to rewrite + # XXX: current filters return MultiTest( name="MTest", suites=[Suite1(), Suite2()], @@ -224,3 +227,33 @@ def test_multi_parts_not_successfully_executed(): assert plan.report.status == Status.ERROR # Testplan result assert plan.report.entries[0].status == Status.ERROR # 1st part raised assert "Deliberately raises" in plan.report.entries[0].logs[0]["message"] + + +def test_even_parts(): + plan = TestplanMock(name="plan") + for i in range(8): + plan.add( + MultiTest( + name="MTest", + suites=[Suite1(), Suite2(), Suite3()], + part=(i, 8), + ) + ) + + assert plan.run().run is True + assert all( + map( + eq, + map(lambda e: e.counter["total"], plan.report.entries), + chain(repeat(2, 7), repeat(1)), + ) + ) + assert all( + map(eq, map(len, plan.report.entries), [1, 1, 2, 2, 2, 2, 2, 1]) + ) + for i in range(8): + assert plan.report.entries[i].entries[0].name == "Suite1" + for i in range(2, 5): + assert plan.report.entries[i].entries[1].name == "Suite2" + for i in range(5, 7): + assert plan.report.entries[i].entries[1].name == "Suite3" diff --git a/tests/functional/testplan/testing/test_filtering.py b/tests/functional/testplan/testing/test_filtering.py index 9551d54fc..09b6464ac 100644 --- a/tests/functional/testplan/testing/test_filtering.py +++ b/tests/functional/testplan/testing/test_filtering.py @@ -219,12 +219,12 @@ def test_programmatic_filtering(filter_obj, report_ctx): filtering.Pattern("XXX - part([012]/*):Alpha") | filtering.Pattern("XXX:Beta:test_three"), [ - ("XXX - part(0/3)", [("Alpha", ["test_one"])]), - ("XXX - part(1/3)", [("Alpha", ["test_two"])]), ( - "XXX - part(2/3)", - [("Alpha", ["test_three"]), ("Beta", ["test_three"])], + "XXX - part(0/3)", + [("Alpha", ["test_one"]), ("Beta", ["test_three"])], ), + ("XXX - part(1/3)", [("Alpha", ["test_two"])]), + ("XXX - part(2/3)", [("Alpha", ["test_three"])]), ], ), # Case 4, ill-formed part diff --git a/tests/functional/testplan/testing/test_listing.py b/tests/functional/testplan/testing/test_listing.py index a058a6220..04896a9a1 100644 --- a/tests/functional/testplan/testing/test_listing.py +++ b/tests/functional/testplan/testing/test_listing.py @@ -85,13 +85,13 @@ def test_a(self, env, result): " Primary - part(0/2):Beta:test_c", " Primary - part(0/2):Beta:test_a --tags color=red", " Primary - part(0/2):Alpha", - " Primary - part(0/2):Alpha:test_c", - " Primary - part(0/2):Alpha:test_a --tags color=green", + " Primary - part(0/2):Alpha:test_b --tags bar foo", "Primary - part(1/2)", " Primary - part(1/2):Beta --tags color=yellow", " Primary - part(1/2):Beta:test_b --tags foo", " Primary - part(1/2):Alpha", - " Primary - part(1/2):Alpha:test_b --tags bar foo", + " Primary - part(1/2):Alpha:test_c", + " Primary - part(1/2):Alpha:test_a --tags color=green", "Secondary", " Secondary:Gamma", " Secondary:Gamma:test_c", @@ -248,11 +248,11 @@ def test_programmatic_listing( " Beta", " test_c", " Alpha", - " test_c", - " test_a", + " test_b", "Primary - part(1/2)", " Alpha", - " test_b", + " test_c", + " test_a", ), ), ],