Merge pull request #483 from raoyitao/param_report_lookup

Parametrized testcases appear at the end of testsuite report

testplan/testing/multitest/base.py

@@ -349,7 +349,7 @@ def skip_step(self, step):
         if step in (self.resources.start, self.resources.stop):
             return False
         elif self.resources.start_exceptions or self.resources.stop_exceptions:
-            self.logger.critical("Skipping step %s", step.__name__)
+            self.logger.critical('Skipping step "%s"', step.__name__)
             return True
         return False
 
@@ -492,8 +492,8 @@ def _testcase_reports(self, testsuite, testcases, status):
         Generate a list of reports for testcases, including parametrization
         groups.
         """
-        parametrization_reports = {}
         testcase_reports = []
+        parametrization_reports = {}
 
         for testcase in testcases:
             testcase_report = self._new_testcase_report(testcase)
@@ -564,6 +564,7 @@ def _new_parametrized_group_report(self, param_template, param_method):
         # avoid duplication with the generated testcases.
         return testplan.report.TestGroupReport(
             name=param_template,
+            description=strings.get_docstring(param_method),
             category=testplan.report.ReportCategories.PARAMETRIZATION,
             uid=param_template,
             tags=param_method.__tags__,
@@ -635,10 +636,8 @@ def _run_suite(self, testsuite, testcases):
 
     def _run_serial_testcases(self, testsuite, testcases):
         """Run testcases serially and return a list of test reports."""
-        parametrization_reports = self._parametrization_reports(
-            testsuite, testcases
-        )
         testcase_reports = []
+        parametrization_reports = {}
         pre_testcase = getattr(testsuite, "pre_testcase", None)
         post_testcase = getattr(testsuite, "post_testcase", None)
 
@@ -654,24 +653,25 @@ def _run_serial_testcases(self, testsuite, testcases):
                 testcase, "_parametrization_template", None
             )
             if param_template:
+                if param_template not in parametrization_reports:
+                    param_method = getattr(testsuite, param_template)
+                    param_report = self._new_parametrized_group_report(
+                        param_template, param_method
+                    )
+                    parametrization_reports[param_template] = param_report
+                    testcase_reports.append(param_report)
                 parametrization_reports[param_template].append(testcase_report)
             else:
                 testcase_reports.append(testcase_report)
 
             if testcase_report.status == testplan.report.Status.ERROR:
                 if self.cfg.stop_on_error:
                     self.logger.debug(
-                        "Stopping exeucution of testsuite %s due to error.",
+                        'Stopping exeucution of testsuite "%s" due to error',
                         mtest_suite.get_testsuite_name(testsuite),
                     )
                     break
 
-        # Add all non-empty parametrization reports into the list of returned
-        # testcase reports, to be added to the suite report.
-        for param_report in parametrization_reports.values():
-            if param_report.entries:
-                testcase_reports.append(param_report)
-
         return testcase_reports
 
     def _run_parallel_testcases(self, testsuite, execution_groups):
@@ -690,7 +690,7 @@ def _run_parallel_testcases(self, testsuite, execution_groups):
         post_testcase = getattr(testsuite, "post_testcase", None)
 
         for exec_group in execution_groups:
-            self.logger.debug("Running execution group %s", exec_group)
+            self.logger.debug('Running execution group "%s"', exec_group)
             results = [
                 self._thread_pool.submit(
                     self._run_testcase, testcase, pre_testcase, post_testcase
@@ -725,7 +725,7 @@ def _run_parallel_testcases(self, testsuite, execution_groups):
 
             if should_stop:
                 self.logger.debug(
-                    "Stopping execution of testsuite %s due to error.",
+                    'Stopping execution of testsuite "%s" due to error',
                     self.cfg.name,
                 )
                 break
@@ -808,9 +808,13 @@ def _run_case_related(self, method, testcase, case_result):
         interface.check_signature(method, ["self", "name", "env", "result"])
         method(testcase.__name__, self.resources, case_result)
 
-    def _run_testcase(self, testcase, pre_testcase, post_testcase):
+    def _run_testcase(
+        self, testcase, pre_testcase, post_testcase, testcase_report=None
+    ):
         """Runs a testcase method and returns its report."""
-        testcase_report = self._new_testcase_report(testcase)
+        testcase_report = testcase_report or self._new_testcase_report(
+            testcase
+        )
         testcase_report.runtime_status = testplan.report.RuntimeStatus.RUNNING
         case_result = self.cfg.result(
             stdout_style=self.stdout_style, _scratch=self.scratch

tests/functional/testplan/runnable/interactive/test_api.py

@@ -334,7 +334,7 @@ def plan(tmpdir):
                     "total": 3,
                     "unknown": 3,
                 },
-                "description": None,
+                "description": "Parametrized testcase.",
                 "entry_uids": [
                     "test_parametrized__val_1",
                     "test_parametrized__val_2",

tests/functional/testplan/testing/multitest/test_parametrization.py

@@ -26,10 +26,9 @@
 LOGGER = logging.getLogger()
 
 
-def check_parametrization(mockplan, suite_kls, parametrization_group):
-
+def check_parametrization(mockplan, suite_kls, report_list, tag_dict=None):
+    tag_dict = tag_dict or {}
     multitest = MultiTest(name="MyMultitest", suites=[suite_kls()])
-
     mockplan.add(multitest)
 
     with log_propagation_disabled(TESTPLAN_LOGGER):
@@ -44,8 +43,9 @@ def check_parametrization(mockplan, suite_kls, parametrization_group):
                 entries=[
                     TestGroupReport(
                         name="MySuite",
+                        tags=tag_dict,
                         category=ReportCategories.TESTSUITE,
-                        entries=[parametrization_group],
+                        entries=report_list,
                     )
                 ],
             )
@@ -65,6 +65,7 @@ def test_add(self, env, result, a, b=1, expected=0):
 
     parametrization_group = TestGroupReport(
         name="test_add",
+        description="Simple docstring",
         category=ReportCategories.PARAMETRIZATION,
         entries=[
             TestCaseReport(
@@ -90,7 +91,7 @@ def test_add(self, env, result, a, b=1, expected=0):
         ],
     )
 
-    check_parametrization(mockplan, MySuite, parametrization_group)
+    check_parametrization(mockplan, MySuite, [parametrization_group])
 
 
 def test_combinatorial_parametrization(mockplan):
@@ -123,7 +124,7 @@ def test_sample(self, env, result, a, b):
         ],
     )
 
-    check_parametrization(mockplan, MySuite, parametrization_group)
+    check_parametrization(mockplan, MySuite, [parametrization_group])
 
 
 @pytest.mark.parametrize(
@@ -202,7 +203,7 @@ def sample_test(self, env, result, a, b):
         ],
     )
 
-    check_parametrization(mockplan, MySuite, parametrization_group)
+    check_parametrization(mockplan, MySuite, [parametrization_group])
 
 
 @pytest.mark.parametrize(
@@ -251,7 +252,7 @@ def sample_test(self, env, result, foo):
         ],
     )
 
-    check_parametrization(mockplan, MySuite, parametrization_group)
+    check_parametrization(mockplan, MySuite, [parametrization_group])
 
 
 @pytest.mark.parametrize(
@@ -282,6 +283,29 @@ def sample_test(self, env, result, a):
         pytest.fail(msg)
 
 
+def test_same_name_func():
+    """`name_func` should be used for generating method names."""
+
+    @testsuite
+    class MySuite(object):
+        @testcase(
+            parameters=(("foo", "bar"), ("alpha", "beta")),
+            name_func=lambda func_name, kwargs: "same_name",
+        )
+        def sample_test(self, env, result, a, b):
+            pass
+
+        @testcase(
+            parameters=(("foo", "bar"), ("alpha", "beta")),
+            name_func=lambda func_name, kwargs: "same_name",
+        )
+        def other_test(self, env, result, a, b):
+            pass
+
+    with pytest.raises(SchemaError):
+        MultiTest(name="abc", suites=[MySuite()])
+
+
 def test_custom_wrapper():
     """Custom wrappers should be applied to each generated testcase."""
 
@@ -351,7 +375,7 @@ def adder_test(self, env, result, product, category):
 )
 def test_docstring_func(docstring_func, expected_docstring):
     @testsuite
-    class DummySuite(object):
+    class MySuite(object):
         @testcase(
             parameters=(("foo", "bar"),),
             name_func=lambda func_name, kwargs: "dummy_name",
@@ -361,17 +385,17 @@ def adder_test(self, env, result, first, second):
             """Original docstring"""
             pass
 
-    assert DummySuite.dummy_name.__doc__ == expected_docstring
+    assert MySuite.dummy_name.__doc__ == expected_docstring
 
 
 def test_parametrization_tagging(mockplan):
     """
-        Parametrization report group should include tags
-        generated by `tag_func` and native suite tags in `tag_index` attribute.
+    Parametrization report group should include tags generated by
+    `tag_func` and native suite tags in `tag_index` attribute.
     """
 
     @testsuite(tags="foo")
-    class DummySuite(object):
+    class MySuite(object):
         @testcase(
             parameters=("red", "blue", "green"),
             tags="alpha",
@@ -397,52 +421,73 @@ def dummy_test(self, env, result, color):
         ],
     )
 
-    multitest = MultiTest(name="MyMultitest", suites=[DummySuite()])
-
-    mockplan.add(multitest)
-
-    with log_propagation_disabled(TESTPLAN_LOGGER):
-        mockplan.run()
-
-    expected_report = TestReport(
-        name="plan",
-        entries=[
-            TestGroupReport(
-                name="MyMultitest",
-                category=ReportCategories.MULTITEST,
-                entries=[
-                    TestGroupReport(
-                        name="DummySuite",
-                        tags={"simple": {"foo"}},
-                        category=ReportCategories.TESTSUITE,
-                        entries=[parametrization_group],
-                    )
-                ],
-            )
-        ],
+    check_parametrization(
+        mockplan,
+        MySuite,
+        [parametrization_group],
+        tag_dict={"simple": {"foo"}},
     )
 
-    check_report(expected_report, mockplan.report)
 
-
-def test_same_name_func():
-    """`name_func` should be used for generating method names."""
+def test_order_of_parametrization_report(mockplan):
+    """
+    In test suite report, parametrization report group should be
+    placed at the correct position according to the order
+     it is defined in source code.
+    """
 
     @testsuite
     class MySuite(object):
-        @testcase(
-            parameters=(("foo", "bar"), ("alpha", "beta")),
-            name_func=lambda func_name, kwargs: "same_name",
-        )
-        def sample_test(self, env, result, a, b):
+        @testcase(parameters=("red", "blue", "green"))
+        def dummy_test_1(self, env, result, color):
             pass
 
-        @testcase(
-            parameters=(("foo", "bar"), ("alpha", "beta")),
-            name_func=lambda func_name, kwargs: "same_name",
-        )
-        def other_test(self, env, result, a, b):
+        @testcase
+        def dummy_test_2(self, env, result):
             pass
 
-    with pytest.raises(SchemaError):
-        MultiTest(name="abc", suites=[MySuite()])
+        @testcase(parameters=("circle", "squre", "triangle"))
+        def dummy_test_3(self, env, result, shape):
+            pass
+
+        @testcase
+        def dummy_test_4(self, env, result):
+            pass
+
+        @testcase(parameters=("fragrant", "stinky", "musty"))
+        def dummy_test_5(self, env, result, smell):
+            pass
+
+    suite_report_entries = [
+        TestGroupReport(
+            name="dummy_test_1",
+            category=ReportCategories.PARAMETRIZATION,
+            entries=[
+                TestCaseReport(name="dummy_test_1__color_red"),
+                TestCaseReport(name="dummy_test_1__color_blue"),
+                TestCaseReport(name="dummy_test_1__color_green"),
+            ],
+        ),
+        TestCaseReport(name="dummy_test_2"),
+        TestGroupReport(
+            name="dummy_test_3",
+            category=ReportCategories.PARAMETRIZATION,
+            entries=[
+                TestCaseReport(name="dummy_test_3__shape_circle"),
+                TestCaseReport(name="dummy_test_3__shape_squre"),
+                TestCaseReport(name="dummy_test_3__shape_triangle"),
+            ],
+        ),
+        TestCaseReport(name="dummy_test_4"),
+        TestGroupReport(
+            name="dummy_test_5",
+            category=ReportCategories.PARAMETRIZATION,
+            entries=[
+                TestCaseReport(name="dummy_test_5__smell_fragrant"),
+                TestCaseReport(name="dummy_test_5__smell_stinky"),
+                TestCaseReport(name="dummy_test_5__smell_musty"),
+            ],
+        ),
+    ]
+
+    check_parametrization(mockplan, MySuite, suite_report_entries)

tests/unit/testplan/testing/multitest/test_multitest.py

@@ -135,6 +135,7 @@ def parametrized(self, env, result, val):
                 ),
                 report.TestGroupReport(
                     name="parametrized",
+                    description="Parametrized testcase.",
                     category=report.ReportCategories.PARAMETRIZATION,
                     uid="parametrized",
                     parent_uids=["MTest", "Suite"],