From 7c9dae114fdfd56acbc89dda535c89a321e63819 Mon Sep 17 00:00:00 2001 From: Darren Teo Date: Tue, 16 Jul 2024 07:24:20 +0000 Subject: [PATCH] include teardown time in autopart calculation --- doc/en/introduction.rst | 7 +- ...ardown_timer_for_auto_part_calculation.rst | 1 + testplan/runnable/base.py | 9 +- .../testplan/runners/pools/test_auto_part.py | 155 ++++++++++++++++++ 4 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 doc/newsfragments/3023_changed.add_teardown_timer_for_auto_part_calculation.rst diff --git a/doc/en/introduction.rst b/doc/en/introduction.rst index f55a12a30..fb18dff44 100644 --- a/doc/en/introduction.rst +++ b/doc/en/introduction.rst @@ -344,11 +344,12 @@ Command line { "": { - "execution_time": 199.99, - "setup_time": 39.99, + | "execution_time": 199.99, + | "setup_time": 39.99, + | "teardown_time": 39.99, // optional }, - ...... } + --skip-remaining {cases-on-failed,cases-on-error,suites-on-failed,suites-on-error,tests-on-failed,tests-on-error} Make Testplan break from the current execution flow and skip remaining iterations at certain level (choose one from all the options). "on-error" make this skip upon exception raised, and "on-failed" make this skip upon both exception raised and test failure. In other words, "on-failed" has higher precedence. diff --git a/doc/newsfragments/3023_changed.add_teardown_timer_for_auto_part_calculation.rst b/doc/newsfragments/3023_changed.add_teardown_timer_for_auto_part_calculation.rst new file mode 100644 index 000000000..6ff25b88e --- /dev/null +++ b/doc/newsfragments/3023_changed.add_teardown_timer_for_auto_part_calculation.rst @@ -0,0 +1 @@ +Add optional ``teardown_time`` for Multitest auto-part calculation. \ No newline at end of file diff --git a/testplan/runnable/base.py b/testplan/runnable/base.py index efee499a8..8bda7c97e 100644 --- a/testplan/runnable/base.py +++ b/testplan/runnable/base.py @@ -577,6 +577,8 @@ def _get_tasks( tasks: List[TaskInformation] = [] time_info = runtime_data.get(uid, None) + if time_info and "teardown_time" not in time_info: + time_info["teardown_time"] = 0 if num_of_parts: if not isinstance(task_info.materialized_test, MultiTest): @@ -605,6 +607,7 @@ def _get_tasks( / ( self.cfg.auto_part_runtime_limit {self.cfg.auto_part_runtime_limit} - time_info["setup_time"] {time_info["setup_time"]} + - time_info["teardown_time"] {time_info["teardown_time"]} ) ) """ @@ -614,6 +617,7 @@ def _get_tasks( / ( self.cfg.auto_part_runtime_limit - time_info["setup_time"] + - time_info["teardown_time"] ) ) except ZeroDivisionError: @@ -638,6 +642,7 @@ def _get_tasks( math.ceil( (time_info["execution_time"] / num_of_parts) + time_info["setup_time"] + + time_info["teardown_time"] ) if time_info else self.cfg.auto_part_runtime_limit @@ -664,7 +669,9 @@ def _get_tasks( else: if time_info and not task.weight: task_info.target.weight = math.ceil( - time_info["execution_time"] + time_info["setup_time"] + time_info["execution_time"] + + time_info["setup_time"] + + time_info["teardown_time"] ) self.logger.user_info( "%s: weight=%d", uid, task_info.target.weight diff --git a/tests/functional/testplan/runners/pools/test_auto_part.py b/tests/functional/testplan/runners/pools/test_auto_part.py index f1f614033..99a65f5b9 100755 --- a/tests/functional/testplan/runners/pools/test_auto_part.py +++ b/tests/functional/testplan/runners/pools/test_auto_part.py @@ -157,3 +157,158 @@ def test_auto_parts_cap_parts(): assert task.weight == 64 mockplan.run() assert pool.size == 1 + + +def test_auto_parts_discover_with_teardown_time(): + with tempfile.TemporaryDirectory() as runpath: + mockplan = TestplanMock( + "plan", + runpath=runpath, + merge_scheduled_parts=True, + auto_part_runtime_limit=45, + plan_runtime_target=200, + runtime_data={ + "Proj1-suite": { + "execution_time": 199.99, + "setup_time": 2.5, + "teardown_time": 2.5, + } + }, + ) + pool = ProcessPool(name="MyPool", size="auto") + mockplan.add_resource(pool) + current_folder = os.path.dirname(os.path.realpath(__file__)) + mockplan.schedule_all( + path=f"{current_folder}/discover_tasks", + name_pattern=r".*auto_parts_tasks\.py$", + resource="MyPool", + ) + assert len(pool.added_items) == 5 + for task in pool.added_items.values(): + assert task.weight == 45 + mockplan.run() + assert pool.size == 2 + + +def test_auto_parts_discover_interactive_with_teardown_time(runpath): + mockplan = TestplanMock( + "plan", + runpath=runpath, + merge_scheduled_parts=True, + auto_part_runtime_limit=45, + plan_runtime_target=200, + interactive_port=0, + runtime_data={ + "Proj1-suite": { + "execution_time": 199.99, + "setup_time": 5, + "teardown_time": 5, + } + }, + ) + pool = ProcessPool(name="MyPool", size="auto") + mockplan.add_resource(pool) + current_folder = Path(__file__).resolve().parent + mockplan.schedule_all( + path=current_folder / "discover_tasks", + name_pattern=r".*auto_parts_tasks\.py$", + resource="MyPool", + ) + + local_pool = mockplan.resources.get(mockplan.resources.first()) + # validate that only one task added to the local pool without split + + assert len(pool.added_items) == 0 + assert len(local_pool.added_items) == 1 + + +def test_auto_weight_discover_with_teardown_time(): + with tempfile.TemporaryDirectory() as runpath: + mockplan = TestplanMock( + "plan", + runpath=runpath, + merge_scheduled_parts=True, + plan_runtime_target=300, + runtime_data={ + "Proj1-suite": { + "execution_time": 199.99, + "setup_time": 19.99, + "teardown_time": 19.99, + } + }, + ) + pool = ProcessPool(name="MyPool", size="auto") + mockplan.add_resource(pool) + current_folder = os.path.dirname(os.path.realpath(__file__)) + mockplan.schedule_all( + path=f"{current_folder}/discover_tasks", + name_pattern=r".*auto_weight_tasks\.py$", + resource="MyPool", + ) + assert len(pool.added_items) == 2 + for task in pool.added_items.values(): + assert task.weight == 140 + mockplan.run() + assert pool.size == 1 + + +def test_auto_parts_zero_neg_parts_with_teardown_time(): + with tempfile.TemporaryDirectory() as runpath: + mockplan = TestplanMock( + "plan", + runpath=runpath, + merge_scheduled_parts=True, + auto_part_runtime_limit=45, + plan_runtime_target=200, + runtime_data={ + "Proj1-suite": { + "execution_time": 50, + "setup_time": 25, # setup_time + teardown_time > runtime_limit -> negtive num_of_parts + "teardown_time": 25, + } + }, + ) + pool = ProcessPool(name="MyPool", size="auto") + mockplan.add_resource(pool) + current_folder = os.path.dirname(os.path.realpath(__file__)) + mockplan.schedule_all( + path=f"{current_folder}/discover_tasks", + name_pattern=r".*auto_parts_tasks\.py$", + resource="MyPool", + ) + assert len(pool.added_items) == 1 + for task in pool.added_items.values(): + assert task.weight == 100 + mockplan.run() + assert pool.size == 1 + + +def test_auto_parts_cap_parts_with_teardown_time(): + with tempfile.TemporaryDirectory() as runpath: + mockplan = TestplanMock( + "plan", + runpath=runpath, + merge_scheduled_parts=True, + auto_part_runtime_limit=45, + plan_runtime_target=200, + runtime_data={ + "Proj1-suite": { + "execution_time": 60, + "setup_time": 22, + "teardown_time": 22, + } + }, + ) + pool = ProcessPool(name="MyPool", size="auto") + mockplan.add_resource(pool) + current_folder = os.path.dirname(os.path.realpath(__file__)) + mockplan.schedule_all( + path=f"{current_folder}/discover_tasks", + name_pattern=r".*auto_parts_tasks\.py$", + resource="MyPool", + ) + assert len(pool.added_items) == 3 + for task in pool.added_items.values(): + assert task.weight == 64 + mockplan.run() + assert pool.size == 1