Fix/multitest parting revision (#1097)

change parting to case-level round-robin

doc/newsfragments/2755_changed.multitest_parting.rst

@@ -0,0 +1 @@
+Testcases of ``MultiTest`` can now be distributed more evenly across parts.

testplan/testing/multitest/base.py

@@ -392,6 +392,11 @@ def get_test_context(self):
         ctx = []
         sorted_suites = self.cfg.test_sorter.sorted_testsuites(self.cfg.suites)
 
+        if hasattr(self.cfg, "xfail_tests") and self.cfg.xfail_tests:
+            xfail_data = self.cfg.xfail_tests
+        else:
+            xfail_data = {}
+
         for suite in sorted_suites:
             testcases = suite.get_testcases()
 
@@ -402,13 +407,6 @@ def get_test_context(self):
                 else self.cfg.test_sorter.sorted_testcases(suite, testcases)
             )
 
-            if self.cfg.part:
-                sorted_testcases = [
-                    testcase
-                    for (idx, testcase) in enumerate(sorted_testcases)
-                    if (idx) % self.cfg.part[1] == self.cfg.part[0]
-                ]
-
             testcases_to_run = [
                 case
                 for case in sorted_testcases
@@ -425,26 +423,39 @@ def get_test_context(self):
                 testcases_to_run = sorted_testcases
 
             if testcases_to_run:
-                if hasattr(self.cfg, "xfail_tests") and self.cfg.xfail_tests:
-                    for testcase in testcases_to_run:
-                        testcase_instance = ":".join(
-                            [
-                                self.name,
-                                suite.name,
-                                testcase.name,
-                            ]
-                        )
-                        data = self.cfg.xfail_tests.get(
-                            testcase_instance, None
-                        )
-                        if data is not None:
-                            testcase.__func__.__xfail__ = {
-                                "reason": data["reason"],
-                                "strict": data["strict"],
-                            }
+                for testcase in testcases_to_run:
+                    testcase_instance = ":".join(
+                        [
+                            self.name,
+                            suite.name,
+                            testcase.name,
+                        ]
+                    )
+                    data = xfail_data.get(testcase_instance, None)
+                    if data is not None:
+                        testcase.__func__.__xfail__ = {
+                            "reason": data["reason"],
+                            "strict": data["strict"],
+                        }
 
                 ctx.append((suite, testcases_to_run))
 
+        if self.cfg.part:
+            # round-robin at testcase level
+            numer, denom = self.cfg.part
+            ofst = 0
+            ctx_ = []
+            for suite, cases in ctx:
+                cases_ = [
+                    case
+                    for idx, case in enumerate(cases)
+                    if (idx + ofst) % denom == numer
+                ]
+                ofst = (ofst + len(cases)) % denom
+                if cases_:
+                    ctx_.append((suite, cases_))
+            return ctx_
+
         return ctx
 
     def _dry_run_testsuites(self):

tests/functional/testplan/testing/multitest/test_multitest_parts.py

@@ -1,11 +1,11 @@
-import itertools
-
-from testplan.testing.multitest import MultiTest, testsuite, testcase
+from itertools import chain, cycle, repeat
+from operator import eq
 
 from testplan import TestplanMock
+from testplan.report import Status
 from testplan.runners.pools.base import Pool as ThreadPool
 from testplan.runners.pools.tasks import Task
-from testplan.report import Status
+from testplan.testing.multitest import MultiTest, testcase, testsuite
 
 
 @testsuite
@@ -44,16 +44,19 @@ def _post_run_checks(self, start_threads, start_procs):
             raise RuntimeError("Deliberately raises")
 
 
-uid_gen = itertools.cycle([i for i in range(10)])
-
-
 def get_mtest(part_tuple=None):
     return MultiTest(
         name="MTest", suites=[Suite1(), Suite2()], part=part_tuple
     )
 
 
+uid_gen = cycle([i for i in range(10)])
+
+
 def get_mtest_with_custom_uid(part_tuple=None):
+    # XXX: abolish multi_part_uid, may rename it to multi_part_report_name?
+    # XXX: or we may still accept customised uids, but we need to rewrite
+    # XXX: current filters
     return MultiTest(
         name="MTest",
         suites=[Suite1(), Suite2()],
@@ -224,3 +227,33 @@ def test_multi_parts_not_successfully_executed():
     assert plan.report.status == Status.ERROR  # Testplan result
     assert plan.report.entries[0].status == Status.ERROR  # 1st part raised
     assert "Deliberately raises" in plan.report.entries[0].logs[0]["message"]
+
+
+def test_even_parts():
+    plan = TestplanMock(name="plan")
+    for i in range(8):
+        plan.add(
+            MultiTest(
+                name="MTest",
+                suites=[Suite1(), Suite2(), Suite3()],
+                part=(i, 8),
+            )
+        )
+
+    assert plan.run().run is True
+    assert all(
+        map(
+            eq,
+            map(lambda e: e.counter["total"], plan.report.entries),
+            chain(repeat(2, 7), repeat(1)),
+        )
+    )
+    assert all(
+        map(eq, map(len, plan.report.entries), [1, 1, 2, 2, 2, 2, 2, 1])
+    )
+    for i in range(8):
+        assert plan.report.entries[i].entries[0].name == "Suite1"
+    for i in range(2, 5):
+        assert plan.report.entries[i].entries[1].name == "Suite2"
+    for i in range(5, 7):
+        assert plan.report.entries[i].entries[1].name == "Suite3"

tests/functional/testplan/testing/test_filtering.py

@@ -219,12 +219,12 @@ def test_programmatic_filtering(filter_obj, report_ctx):
             filtering.Pattern("XXX - part([012]/*):Alpha")
             | filtering.Pattern("XXX:Beta:test_three"),
             [
-                ("XXX - part(0/3)", [("Alpha", ["test_one"])]),
-                ("XXX - part(1/3)", [("Alpha", ["test_two"])]),
                 (
-                    "XXX - part(2/3)",
-                    [("Alpha", ["test_three"]), ("Beta", ["test_three"])],
+                    "XXX - part(0/3)",
+                    [("Alpha", ["test_one"]), ("Beta", ["test_three"])],
                 ),
+                ("XXX - part(1/3)", [("Alpha", ["test_two"])]),
+                ("XXX - part(2/3)", [("Alpha", ["test_three"])]),
             ],
         ),
         # Case 4, ill-formed part

tests/functional/testplan/testing/test_listing.py

@@ -85,13 +85,13 @@ def test_a(self, env, result):
     "    Primary - part(0/2):Beta:test_c",
     "    Primary - part(0/2):Beta:test_a  --tags color=red",
     "  Primary - part(0/2):Alpha",
-    "    Primary - part(0/2):Alpha:test_c",
-    "    Primary - part(0/2):Alpha:test_a  --tags color=green",
+    "    Primary - part(0/2):Alpha:test_b  --tags bar foo",
     "Primary - part(1/2)",
     "  Primary - part(1/2):Beta  --tags color=yellow",
     "    Primary - part(1/2):Beta:test_b  --tags foo",
     "  Primary - part(1/2):Alpha",
-    "    Primary - part(1/2):Alpha:test_b  --tags bar foo",
+    "    Primary - part(1/2):Alpha:test_c",
+    "    Primary - part(1/2):Alpha:test_a  --tags color=green",
     "Secondary",
     "  Secondary:Gamma",
     "    Secondary:Gamma:test_c",
@@ -248,11 +248,11 @@ def test_programmatic_listing(
                 "  Beta",
                 "    test_c",
                 "  Alpha",
-                "    test_c",
-                "    test_a",
+                "    test_b",
                 "Primary - part(1/2)",
                 "  Alpha",
-                "    test_b",
+                "    test_c",
+                "    test_a",
             ),
         ),
     ],