Skip to content

Commit

Permalink
Parametrized testcases appear at the end of testsuite report
Browse files Browse the repository at this point in the history 
* Parametrized testcases should be placed according to the order it is
  defined in the source file.
  • Loading branch information
@raoyitao
raoyitao committed Aug 27, 2020
1 parent d484694 commit 73f4b79
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 67 deletions.
34 changes: 18 additions & 16 deletions testplan/testing/multitest/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,7 @@ def skip_step(self, step):
if step in (self.resources.start, self.resources.stop):
return False
elif self.resources.start_exceptions or self.resources.stop_exceptions:
self.logger.critical("Skipping step %s", step.__name__)
self.logger.critical('Skipping step "%s"', step.__name__)
return True
return False

Expand Down Expand Up @@ -492,8 +492,8 @@ def _testcase_reports(self, testsuite, testcases, status):
Generate a list of reports for testcases, including parametrization
groups.
"""
parametrization_reports = {}
testcase_reports = []
parametrization_reports = {}

for testcase in testcases:
testcase_report = self._new_testcase_report(testcase)
Expand Down Expand Up @@ -564,6 +564,7 @@ def _new_parametrized_group_report(self, param_template, param_method):
# avoid duplication with the generated testcases.
return testplan.report.TestGroupReport(
name=param_template,
description=strings.get_docstring(param_method),
category=testplan.report.ReportCategories.PARAMETRIZATION,
uid=param_template,
tags=param_method.__tags__,
Expand Down Expand Up @@ -635,10 +636,8 @@ def _run_suite(self, testsuite, testcases):

def _run_serial_testcases(self, testsuite, testcases):
"""Run testcases serially and return a list of test reports."""
parametrization_reports = self._parametrization_reports(
testsuite, testcases
)
testcase_reports = []
parametrization_reports = {}
pre_testcase = getattr(testsuite, "pre_testcase", None)
post_testcase = getattr(testsuite, "post_testcase", None)

Expand All @@ -654,24 +653,25 @@ def _run_serial_testcases(self, testsuite, testcases):
testcase, "_parametrization_template", None
)
if param_template:
if param_template not in parametrization_reports:
param_method = getattr(testsuite, param_template)
param_report = self._new_parametrized_group_report(
param_template, param_method
)
parametrization_reports[param_template] = param_report
testcase_reports.append(param_report)
parametrization_reports[param_template].append(testcase_report)
else:
testcase_reports.append(testcase_report)

if testcase_report.status == testplan.report.Status.ERROR:
if self.cfg.stop_on_error:
self.logger.debug(
"Stopping exeucution of testsuite %s due to error.",
'Stopping exeucution of testsuite "%s" due to error',
mtest_suite.get_testsuite_name(testsuite),
)
break

# Add all non-empty parametrization reports into the list of returned
# testcase reports, to be added to the suite report.
for param_report in parametrization_reports.values():
if param_report.entries:
testcase_reports.append(param_report)

return testcase_reports

def _run_parallel_testcases(self, testsuite, execution_groups):
Expand All @@ -690,7 +690,7 @@ def _run_parallel_testcases(self, testsuite, execution_groups):
post_testcase = getattr(testsuite, "post_testcase", None)

for exec_group in execution_groups:
self.logger.debug("Running execution group %s", exec_group)
self.logger.debug('Running execution group "%s"', exec_group)
results = [
self._thread_pool.submit(
self._run_testcase, testcase, pre_testcase, post_testcase
Expand Down Expand Up @@ -725,7 +725,7 @@ def _run_parallel_testcases(self, testsuite, execution_groups):

if should_stop:
self.logger.debug(
"Stopping execution of testsuite %s due to error.",
'Stopping execution of testsuite "%s" due to error',
self.cfg.name,
)
break
Expand Down Expand Up @@ -808,9 +808,11 @@ def _run_case_related(self, method, testcase, case_result):
interface.check_signature(method, ["self", "name", "env", "result"])
method(testcase.__name__, self.resources, case_result)

def _run_testcase(self, testcase, pre_testcase, post_testcase):
def _run_testcase(
self, testcase, pre_testcase, post_testcase, testcase_report=None
):
"""Runs a testcase method and returns its report."""
testcase_report = self._new_testcase_report(testcase)
testcase_report = testcase_report or self._new_testcase_report(testcase)
testcase_report.runtime_status = testplan.report.RuntimeStatus.RUNNING
case_result = self.cfg.result(
stdout_style=self.stdout_style, _scratch=self.scratch
Expand Down
150 changes: 99 additions & 51 deletions tests/functional/testplan/testing/multitest/test_parametrization.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,11 @@
LOGGER = logging.getLogger()


def check_parametrization(mockplan, suite_kls, parametrization_group):

def check_parametrization(
mockplan, suite_kls, report_list, tag_dict=None
):
tag_dict = tag_dict or {}
multitest = MultiTest(name="MyMultitest", suites=[suite_kls()])

mockplan.add(multitest)

with log_propagation_disabled(TESTPLAN_LOGGER):
Expand All @@ -44,8 +45,9 @@ def check_parametrization(mockplan, suite_kls, parametrization_group):
entries=[
TestGroupReport(
name="MySuite",
tags=tag_dict,
category=ReportCategories.TESTSUITE,
entries=[parametrization_group],
entries=report_list,
)
],
)
Expand All @@ -65,6 +67,7 @@ def test_add(self, env, result, a, b=1, expected=0):

parametrization_group = TestGroupReport(
name="test_add",
description="Simple docstring",
category=ReportCategories.PARAMETRIZATION,
entries=[
TestCaseReport(
Expand All @@ -90,7 +93,7 @@ def test_add(self, env, result, a, b=1, expected=0):
],
)

check_parametrization(mockplan, MySuite, parametrization_group)
check_parametrization(mockplan, MySuite, [parametrization_group])


def test_combinatorial_parametrization(mockplan):
Expand Down Expand Up @@ -123,7 +126,7 @@ def test_sample(self, env, result, a, b):
],
)

check_parametrization(mockplan, MySuite, parametrization_group)
check_parametrization(mockplan, MySuite, [parametrization_group])


@pytest.mark.parametrize(
Expand Down Expand Up @@ -202,7 +205,7 @@ def sample_test(self, env, result, a, b):
],
)

check_parametrization(mockplan, MySuite, parametrization_group)
check_parametrization(mockplan, MySuite, [parametrization_group])


@pytest.mark.parametrize(
Expand Down Expand Up @@ -251,7 +254,7 @@ def sample_test(self, env, result, foo):
],
)

check_parametrization(mockplan, MySuite, parametrization_group)
check_parametrization(mockplan, MySuite, [parametrization_group])


@pytest.mark.parametrize(
Expand Down Expand Up @@ -282,6 +285,29 @@ def sample_test(self, env, result, a):
pytest.fail(msg)


def test_same_name_func():
"""`name_func` should be used for generating method names."""

@testsuite
class MySuite(object):
@testcase(
parameters=(("foo", "bar"), ("alpha", "beta")),
name_func=lambda func_name, kwargs: "same_name",
)
def sample_test(self, env, result, a, b):
pass

@testcase(
parameters=(("foo", "bar"), ("alpha", "beta")),
name_func=lambda func_name, kwargs: "same_name",
)
def other_test(self, env, result, a, b):
pass

with pytest.raises(SchemaError):
MultiTest(name="abc", suites=[MySuite()])


def test_custom_wrapper():
"""Custom wrappers should be applied to each generated testcase."""

Expand Down Expand Up @@ -351,7 +377,7 @@ def adder_test(self, env, result, product, category):
)
def test_docstring_func(docstring_func, expected_docstring):
@testsuite
class DummySuite(object):
class MySuite(object):
@testcase(
parameters=(("foo", "bar"),),
name_func=lambda func_name, kwargs: "dummy_name",
Expand All @@ -361,17 +387,17 @@ def adder_test(self, env, result, first, second):
"""Original docstring"""
pass

assert DummySuite.dummy_name.__doc__ == expected_docstring
assert MySuite.dummy_name.__doc__ == expected_docstring


def test_parametrization_tagging(mockplan):
"""
Parametrization report group should include tags
generated by `tag_func` and native suite tags in `tag_index` attribute.
Parametrization report group should include tags generated by
`tag_func` and native suite tags in `tag_index` attribute.
"""

@testsuite(tags="foo")
class DummySuite(object):
class MySuite(object):
@testcase(
parameters=("red", "blue", "green"),
tags="alpha",
Expand All @@ -397,52 +423,74 @@ def dummy_test(self, env, result, color):
],
)

multitest = MultiTest(name="MyMultitest", suites=[DummySuite()])
check_parametrization(
mockplan, MySuite, [parametrization_group], tag_dict={"simple": {"foo"}}
)

mockplan.add(multitest)

with log_propagation_disabled(TESTPLAN_LOGGER):
mockplan.run()
def test_order_of_parametrization_report(mockplan):
"""
In test suite report, parametrization report group should be
placed at the correct position according to the order
it is defined in source code.
"""

expected_report = TestReport(
name="plan",
entries=[
TestGroupReport(
name="MyMultitest",
category=ReportCategories.MULTITEST,
entries=[
TestGroupReport(
name="DummySuite",
tags={"simple": {"foo"}},
category=ReportCategories.TESTSUITE,
entries=[parametrization_group],
)
],
)
],
)
@testsuite
class MySuite(object):
@testcase(parameters=("red", "blue", "green"))
def dummy_test_1(self, env, result, color):
pass

check_report(expected_report, mockplan.report)
@testcase
def dummy_test_2(self, env, result):
pass

@testcase(parameters=("circle", "squre", "triangle"))
def dummy_test_3(self, env, result, shape):
pass

def test_same_name_func():
"""`name_func` should be used for generating method names."""
@testcase
def dummy_test_4(self, env, result):
pass

@testsuite
class MySuite(object):
@testcase(
parameters=(("foo", "bar"), ("alpha", "beta")),
name_func=lambda func_name, kwargs: "same_name",
)
def sample_test(self, env, result, a, b):
@testcase(parameters=("fragrant", "stinky", "musty"))
def dummy_test_5(self, env, result, smell):
pass

@testcase(
parameters=(("foo", "bar"), ("alpha", "beta")),
name_func=lambda func_name, kwargs: "same_name",
suite_report_entries = [
TestGroupReport(
name="dummy_test_1",
category=ReportCategories.PARAMETRIZATION,
entries=[
TestCaseReport(name="dummy_test_1__color_red"),
TestCaseReport(name="dummy_test_1__color_blue"),
TestCaseReport(name="dummy_test_1__color_green"),
],
),
TestCaseReport(
name="dummy_test_2"
),
TestGroupReport(
name="dummy_test_3",
category=ReportCategories.PARAMETRIZATION,
entries=[
TestCaseReport(name="dummy_test_3__shape_circle"),
TestCaseReport(name="dummy_test_3__shape_squre"),
TestCaseReport(name="dummy_test_3__shape_triangle"),
],
),
TestCaseReport(
name="dummy_test_4"
),
TestGroupReport(
name="dummy_test_5",
category=ReportCategories.PARAMETRIZATION,
entries=[
TestCaseReport(name="dummy_test_5__smell_fragrant"),
TestCaseReport(name="dummy_test_5__smell_stinky"),
TestCaseReport(name="dummy_test_5__smell_musty"),
],
)
def other_test(self, env, result, a, b):
pass
]

with pytest.raises(SchemaError):
MultiTest(name="abc", suites=[MySuite()])
check_parametrization(mockplan, MySuite, suite_report_entries)

0 comments on commit 73f4b79

Please sign in to comment.