From 348be1f08276db3290276dafb5fd535b02104332 Mon Sep 17 00:00:00 2001 From: raoyitao Date: Thu, 27 Aug 2020 18:33:16 +0800 Subject: [PATCH] Parametrized testcases appear at the end of testsuite report * Parametrized testcases should be placed according to the order it is defined in the source file. * Doc string of parametrization template function is resserved as description of that parametrization group report. --- testplan/testing/multitest/base.py | 36 +++-- .../testplan/runnable/interactive/test_api.py | 2 +- .../testing/multitest/test_parametrization.py | 149 ++++++++++++------ .../testing/multitest/test_multitest.py | 1 + 4 files changed, 119 insertions(+), 69 deletions(-) diff --git a/testplan/testing/multitest/base.py b/testplan/testing/multitest/base.py index 12e52c988..65d2dfcd4 100644 --- a/testplan/testing/multitest/base.py +++ b/testplan/testing/multitest/base.py @@ -349,7 +349,7 @@ def skip_step(self, step): if step in (self.resources.start, self.resources.stop): return False elif self.resources.start_exceptions or self.resources.stop_exceptions: - self.logger.critical("Skipping step %s", step.__name__) + self.logger.critical('Skipping step "%s"', step.__name__) return True return False @@ -492,8 +492,8 @@ def _testcase_reports(self, testsuite, testcases, status): Generate a list of reports for testcases, including parametrization groups. """ - parametrization_reports = {} testcase_reports = [] + parametrization_reports = {} for testcase in testcases: testcase_report = self._new_testcase_report(testcase) @@ -564,6 +564,7 @@ def _new_parametrized_group_report(self, param_template, param_method): # avoid duplication with the generated testcases. return testplan.report.TestGroupReport( name=param_template, + description=strings.get_docstring(param_method), category=testplan.report.ReportCategories.PARAMETRIZATION, uid=param_template, tags=param_method.__tags__, @@ -635,10 +636,8 @@ def _run_suite(self, testsuite, testcases): def _run_serial_testcases(self, testsuite, testcases): """Run testcases serially and return a list of test reports.""" - parametrization_reports = self._parametrization_reports( - testsuite, testcases - ) testcase_reports = [] + parametrization_reports = {} pre_testcase = getattr(testsuite, "pre_testcase", None) post_testcase = getattr(testsuite, "post_testcase", None) @@ -654,6 +653,13 @@ def _run_serial_testcases(self, testsuite, testcases): testcase, "_parametrization_template", None ) if param_template: + if param_template not in parametrization_reports: + param_method = getattr(testsuite, param_template) + param_report = self._new_parametrized_group_report( + param_template, param_method + ) + parametrization_reports[param_template] = param_report + testcase_reports.append(param_report) parametrization_reports[param_template].append(testcase_report) else: testcase_reports.append(testcase_report) @@ -661,17 +667,11 @@ def _run_serial_testcases(self, testsuite, testcases): if testcase_report.status == testplan.report.Status.ERROR: if self.cfg.stop_on_error: self.logger.debug( - "Stopping exeucution of testsuite %s due to error.", + 'Stopping exeucution of testsuite "%s" due to error', mtest_suite.get_testsuite_name(testsuite), ) break - # Add all non-empty parametrization reports into the list of returned - # testcase reports, to be added to the suite report. - for param_report in parametrization_reports.values(): - if param_report.entries: - testcase_reports.append(param_report) - return testcase_reports def _run_parallel_testcases(self, testsuite, execution_groups): @@ -690,7 +690,7 @@ def _run_parallel_testcases(self, testsuite, execution_groups): post_testcase = getattr(testsuite, "post_testcase", None) for exec_group in execution_groups: - self.logger.debug("Running execution group %s", exec_group) + self.logger.debug('Running execution group "%s"', exec_group) results = [ self._thread_pool.submit( self._run_testcase, testcase, pre_testcase, post_testcase @@ -725,7 +725,7 @@ def _run_parallel_testcases(self, testsuite, execution_groups): if should_stop: self.logger.debug( - "Stopping execution of testsuite %s due to error.", + 'Stopping execution of testsuite "%s" due to error', self.cfg.name, ) break @@ -808,9 +808,13 @@ def _run_case_related(self, method, testcase, case_result): interface.check_signature(method, ["self", "name", "env", "result"]) method(testcase.__name__, self.resources, case_result) - def _run_testcase(self, testcase, pre_testcase, post_testcase): + def _run_testcase( + self, testcase, pre_testcase, post_testcase, testcase_report=None + ): """Runs a testcase method and returns its report.""" - testcase_report = self._new_testcase_report(testcase) + testcase_report = testcase_report or self._new_testcase_report( + testcase + ) testcase_report.runtime_status = testplan.report.RuntimeStatus.RUNNING case_result = self.cfg.result( stdout_style=self.stdout_style, _scratch=self.scratch diff --git a/tests/functional/testplan/runnable/interactive/test_api.py b/tests/functional/testplan/runnable/interactive/test_api.py index 138e082ed..111ad3c60 100644 --- a/tests/functional/testplan/runnable/interactive/test_api.py +++ b/tests/functional/testplan/runnable/interactive/test_api.py @@ -334,7 +334,7 @@ def plan(tmpdir): "total": 3, "unknown": 3, }, - "description": None, + "description": "Parametrized testcase.", "entry_uids": [ "test_parametrized__val_1", "test_parametrized__val_2", diff --git a/tests/functional/testplan/testing/multitest/test_parametrization.py b/tests/functional/testplan/testing/multitest/test_parametrization.py index 5bf1c1ff5..57840c6f4 100644 --- a/tests/functional/testplan/testing/multitest/test_parametrization.py +++ b/tests/functional/testplan/testing/multitest/test_parametrization.py @@ -26,10 +26,9 @@ LOGGER = logging.getLogger() -def check_parametrization(mockplan, suite_kls, parametrization_group): - +def check_parametrization(mockplan, suite_kls, report_list, tag_dict=None): + tag_dict = tag_dict or {} multitest = MultiTest(name="MyMultitest", suites=[suite_kls()]) - mockplan.add(multitest) with log_propagation_disabled(TESTPLAN_LOGGER): @@ -44,8 +43,9 @@ def check_parametrization(mockplan, suite_kls, parametrization_group): entries=[ TestGroupReport( name="MySuite", + tags=tag_dict, category=ReportCategories.TESTSUITE, - entries=[parametrization_group], + entries=report_list, ) ], ) @@ -65,6 +65,7 @@ def test_add(self, env, result, a, b=1, expected=0): parametrization_group = TestGroupReport( name="test_add", + description="Simple docstring", category=ReportCategories.PARAMETRIZATION, entries=[ TestCaseReport( @@ -90,7 +91,7 @@ def test_add(self, env, result, a, b=1, expected=0): ], ) - check_parametrization(mockplan, MySuite, parametrization_group) + check_parametrization(mockplan, MySuite, [parametrization_group]) def test_combinatorial_parametrization(mockplan): @@ -123,7 +124,7 @@ def test_sample(self, env, result, a, b): ], ) - check_parametrization(mockplan, MySuite, parametrization_group) + check_parametrization(mockplan, MySuite, [parametrization_group]) @pytest.mark.parametrize( @@ -202,7 +203,7 @@ def sample_test(self, env, result, a, b): ], ) - check_parametrization(mockplan, MySuite, parametrization_group) + check_parametrization(mockplan, MySuite, [parametrization_group]) @pytest.mark.parametrize( @@ -251,7 +252,7 @@ def sample_test(self, env, result, foo): ], ) - check_parametrization(mockplan, MySuite, parametrization_group) + check_parametrization(mockplan, MySuite, [parametrization_group]) @pytest.mark.parametrize( @@ -282,6 +283,29 @@ def sample_test(self, env, result, a): pytest.fail(msg) +def test_same_name_func(): + """`name_func` should be used for generating method names.""" + + @testsuite + class MySuite(object): + @testcase( + parameters=(("foo", "bar"), ("alpha", "beta")), + name_func=lambda func_name, kwargs: "same_name", + ) + def sample_test(self, env, result, a, b): + pass + + @testcase( + parameters=(("foo", "bar"), ("alpha", "beta")), + name_func=lambda func_name, kwargs: "same_name", + ) + def other_test(self, env, result, a, b): + pass + + with pytest.raises(SchemaError): + MultiTest(name="abc", suites=[MySuite()]) + + def test_custom_wrapper(): """Custom wrappers should be applied to each generated testcase.""" @@ -351,7 +375,7 @@ def adder_test(self, env, result, product, category): ) def test_docstring_func(docstring_func, expected_docstring): @testsuite - class DummySuite(object): + class MySuite(object): @testcase( parameters=(("foo", "bar"),), name_func=lambda func_name, kwargs: "dummy_name", @@ -361,17 +385,17 @@ def adder_test(self, env, result, first, second): """Original docstring""" pass - assert DummySuite.dummy_name.__doc__ == expected_docstring + assert MySuite.dummy_name.__doc__ == expected_docstring def test_parametrization_tagging(mockplan): """ - Parametrization report group should include tags - generated by `tag_func` and native suite tags in `tag_index` attribute. + Parametrization report group should include tags generated by + `tag_func` and native suite tags in `tag_index` attribute. """ @testsuite(tags="foo") - class DummySuite(object): + class MySuite(object): @testcase( parameters=("red", "blue", "green"), tags="alpha", @@ -397,52 +421,73 @@ def dummy_test(self, env, result, color): ], ) - multitest = MultiTest(name="MyMultitest", suites=[DummySuite()]) - - mockplan.add(multitest) - - with log_propagation_disabled(TESTPLAN_LOGGER): - mockplan.run() - - expected_report = TestReport( - name="plan", - entries=[ - TestGroupReport( - name="MyMultitest", - category=ReportCategories.MULTITEST, - entries=[ - TestGroupReport( - name="DummySuite", - tags={"simple": {"foo"}}, - category=ReportCategories.TESTSUITE, - entries=[parametrization_group], - ) - ], - ) - ], + check_parametrization( + mockplan, + MySuite, + [parametrization_group], + tag_dict={"simple": {"foo"}}, ) - check_report(expected_report, mockplan.report) - -def test_same_name_func(): - """`name_func` should be used for generating method names.""" +def test_order_of_parametrization_report(mockplan): + """ + In test suite report, parametrization report group should be + placed at the correct position according to the order + it is defined in source code. + """ @testsuite class MySuite(object): - @testcase( - parameters=(("foo", "bar"), ("alpha", "beta")), - name_func=lambda func_name, kwargs: "same_name", - ) - def sample_test(self, env, result, a, b): + @testcase(parameters=("red", "blue", "green")) + def dummy_test_1(self, env, result, color): pass - @testcase( - parameters=(("foo", "bar"), ("alpha", "beta")), - name_func=lambda func_name, kwargs: "same_name", - ) - def other_test(self, env, result, a, b): + @testcase + def dummy_test_2(self, env, result): pass - with pytest.raises(SchemaError): - MultiTest(name="abc", suites=[MySuite()]) + @testcase(parameters=("circle", "squre", "triangle")) + def dummy_test_3(self, env, result, shape): + pass + + @testcase + def dummy_test_4(self, env, result): + pass + + @testcase(parameters=("fragrant", "stinky", "musty")) + def dummy_test_5(self, env, result, smell): + pass + + suite_report_entries = [ + TestGroupReport( + name="dummy_test_1", + category=ReportCategories.PARAMETRIZATION, + entries=[ + TestCaseReport(name="dummy_test_1__color_red"), + TestCaseReport(name="dummy_test_1__color_blue"), + TestCaseReport(name="dummy_test_1__color_green"), + ], + ), + TestCaseReport(name="dummy_test_2"), + TestGroupReport( + name="dummy_test_3", + category=ReportCategories.PARAMETRIZATION, + entries=[ + TestCaseReport(name="dummy_test_3__shape_circle"), + TestCaseReport(name="dummy_test_3__shape_squre"), + TestCaseReport(name="dummy_test_3__shape_triangle"), + ], + ), + TestCaseReport(name="dummy_test_4"), + TestGroupReport( + name="dummy_test_5", + category=ReportCategories.PARAMETRIZATION, + entries=[ + TestCaseReport(name="dummy_test_5__smell_fragrant"), + TestCaseReport(name="dummy_test_5__smell_stinky"), + TestCaseReport(name="dummy_test_5__smell_musty"), + ], + ), + ] + + check_parametrization(mockplan, MySuite, suite_report_entries) diff --git a/tests/unit/testplan/testing/multitest/test_multitest.py b/tests/unit/testplan/testing/multitest/test_multitest.py index 2646df14d..eb6b20d39 100644 --- a/tests/unit/testplan/testing/multitest/test_multitest.py +++ b/tests/unit/testplan/testing/multitest/test_multitest.py @@ -135,6 +135,7 @@ def parametrized(self, env, result, val): ), report.TestGroupReport( name="parametrized", + description="Parametrized testcase.", category=report.ReportCategories.PARAMETRIZATION, uid="parametrized", parent_uids=["MTest", "Suite"],