From 3c90b27dedd1afa5ba6b822d4bb3b45e8e215777 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 24 Oct 2023 15:45:31 +0100 Subject: [PATCH 01/19] Add output_entity_hint and support for serialising it Signed-off-by: Thomas Newton --- flytekit/core/python_function_task.py | 7 +++++++ flytekit/tools/translator.py | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index e1e80a4227..b6f9e2f3bb 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -102,6 +102,7 @@ def __init__( ignore_input_vars: Optional[List[str]] = None, execution_mode: ExecutionBehavior = ExecutionBehavior.DEFAULT, task_resolver: Optional[TaskResolverMixin] = None, + output_entity_hint = None, **kwargs, ): """ @@ -145,11 +146,17 @@ def __init__( ) self._task_function = task_function self._execution_mode = execution_mode + self._output_entity_hint = output_entity_hint + assert (self._output_entity_hint is not None) and (self._execution_mode != self.ExecutionBehavior.DYNAMIC), "output_entity_hint should only be specified on dynamic tasks." self._wf = None # For dynamic tasks @property def execution_mode(self) -> ExecutionBehavior: return self._execution_mode + + @property + def output_entity_hint(self) -> ExecutionBehavior: + return self._output_entity_hint @property def task_function(self): diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index 72ebe7510c..110611c6cb 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -159,8 +159,10 @@ def fn(settings: SerializationSettings) -> List[str]: def get_serializable_task( + entity_mapping: OrderedDict, settings: SerializationSettings, entity: FlyteLocalEntity, + options: Optional[Options] = None, ) -> TaskSpec: task_id = _identifier_model.Identifier( _identifier_model.ResourceType.TASK, @@ -176,6 +178,9 @@ def get_serializable_task( # during dynamic serialization settings = settings.with_serialized_context() + if entity.output_entity_hint is not None: + get_serializable(entity_mapping, settings, entity.output_entity_hint, options) + container = entity.get_container(settings) # This pod will be incorrect when doing fast serialize pod = entity.get_k8s_pod(settings) @@ -713,7 +718,7 @@ def get_serializable( cp_entity = get_reference_spec(entity_mapping, settings, entity) elif isinstance(entity, PythonTask): - cp_entity = get_serializable_task(settings, entity) + cp_entity = get_serializable_task(entity_mapping, settings, entity) elif isinstance(entity, WorkflowBase): cp_entity = get_serializable_workflow(entity_mapping, settings, entity, options) From c18f69e8c408a939ce4628bd70a81bdd282ec9a8 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 24 Oct 2023 16:13:08 +0100 Subject: [PATCH 02/19] Correct assertion Signed-off-by: Thomas Newton --- flytekit/core/python_function_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index b6f9e2f3bb..084a31f377 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -147,7 +147,7 @@ def __init__( self._task_function = task_function self._execution_mode = execution_mode self._output_entity_hint = output_entity_hint - assert (self._output_entity_hint is not None) and (self._execution_mode != self.ExecutionBehavior.DYNAMIC), "output_entity_hint should only be specified on dynamic tasks." + assert (self._output_entity_hint is None) or (self._execution_mode == self.ExecutionBehavior.DYNAMIC), "output_entity_hint should only be specified on dynamic tasks." self._wf = None # For dynamic tasks @property From 26fc9b272c407dd29f14745b976906ab56c88c97 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 24 Oct 2023 16:20:24 +0100 Subject: [PATCH 03/19] Add output_entity_hint to task decorator Signed-off-by: Thomas Newton --- flytekit/core/task.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flytekit/core/task.py b/flytekit/core/task.py index 547abd41fa..c8ab7c41cf 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -152,6 +152,7 @@ def task( limits: Optional[Resources] = None, secret_requests: Optional[List[Secret]] = None, execution_mode: PythonFunctionTask.ExecutionBehavior = PythonFunctionTask.ExecutionBehavior.DEFAULT, + output_entity_hint = None, task_resolver: Optional[TaskResolverMixin] = None, docs: Optional[Documentation] = None, disable_deck: Optional[bool] = None, @@ -276,6 +277,7 @@ def wrapper(fn: Callable[..., Any]) -> PythonFunctionTask[T]: limits=limits, secret_requests=secret_requests, execution_mode=execution_mode, + output_entity_hint=output_entity_hint, task_resolver=task_resolver, disable_deck=disable_deck, enable_deck=enable_deck, From 3c5b696299ba1f4d9de5631012a254a16f1dd38a Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 24 Oct 2023 19:28:05 +0100 Subject: [PATCH 04/19] Type hints Signed-off-by: Thomas Newton --- flytekit/core/python_function_task.py | 6 ++++-- flytekit/core/task.py | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index 084a31f377..d93e357a2e 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -22,11 +22,13 @@ from flytekit.core.context_manager import ExecutionState, FlyteContext, FlyteContextManager from flytekit.core.docstring import Docstring from flytekit.core.interface import transform_function_to_interface +from flytekit.core.launch_plan import LaunchPlan from flytekit.core.promise import VoidPromise, translate_inputs_to_literals from flytekit.core.python_auto_container import PythonAutoContainerTask, default_task_resolver from flytekit.core.tracker import extract_task_module, is_functools_wrapped_module_level, isnested, istestfunction from flytekit.core.workflow import ( PythonFunctionWorkflow, + WorkflowBase, WorkflowFailurePolicy, WorkflowMetadata, WorkflowMetadataDefaults, @@ -102,7 +104,7 @@ def __init__( ignore_input_vars: Optional[List[str]] = None, execution_mode: ExecutionBehavior = ExecutionBehavior.DEFAULT, task_resolver: Optional[TaskResolverMixin] = None, - output_entity_hint = None, + output_entity_hint: Union["PythonFunctionTask", LaunchPlan, WorkflowBase] = None, **kwargs, ): """ @@ -155,7 +157,7 @@ def execution_mode(self) -> ExecutionBehavior: return self._execution_mode @property - def output_entity_hint(self) -> ExecutionBehavior: + def output_entity_hint(self) -> Union["PythonFunctionTask", LaunchPlan, WorkflowBase]: return self._output_entity_hint @property diff --git a/flytekit/core/task.py b/flytekit/core/task.py index c8ab7c41cf..c7be14351b 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -4,11 +4,13 @@ from flytekit.core.base_task import TaskMetadata, TaskResolverMixin from flytekit.core.interface import transform_function_to_interface +from flytekit.core.launch_plan import LaunchPlan from flytekit.core.pod_template import PodTemplate from flytekit.core.python_function_task import PythonFunctionTask from flytekit.core.reference_entity import ReferenceEntity, TaskReference from flytekit.core.resources import Resources from flytekit.extras.accelerators import BaseAccelerator +from flytekit.core.workflow import WorkflowBase from flytekit.image_spec.image_spec import ImageSpec from flytekit.models.documentation import Documentation from flytekit.models.security import Secret @@ -97,6 +99,7 @@ def task( limits: Optional[Resources] = ..., secret_requests: Optional[List[Secret]] = ..., execution_mode: PythonFunctionTask.ExecutionBehavior = ..., + output_entity_hint: Union[PythonFunctionTask, LaunchPlan, WorkflowBase] = ..., task_resolver: Optional[TaskResolverMixin] = ..., docs: Optional[Documentation] = ..., disable_deck: Optional[bool] = ..., @@ -125,6 +128,7 @@ def task( limits: Optional[Resources] = ..., secret_requests: Optional[List[Secret]] = ..., execution_mode: PythonFunctionTask.ExecutionBehavior = ..., + output_entity_hint: Union[PythonFunctionTask, LaunchPlan, WorkflowBase] = ..., task_resolver: Optional[TaskResolverMixin] = ..., docs: Optional[Documentation] = ..., disable_deck: Optional[bool] = ..., @@ -152,7 +156,7 @@ def task( limits: Optional[Resources] = None, secret_requests: Optional[List[Secret]] = None, execution_mode: PythonFunctionTask.ExecutionBehavior = PythonFunctionTask.ExecutionBehavior.DEFAULT, - output_entity_hint = None, + output_entity_hint: Union[PythonFunctionTask, LaunchPlan, WorkflowBase] = None, task_resolver: Optional[TaskResolverMixin] = None, docs: Optional[Documentation] = None, disable_deck: Optional[bool] = None, From cc00802cec547db0f7877fe536ec9711306d2039 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 24 Oct 2023 20:58:58 +0100 Subject: [PATCH 05/19] Fix circular import of type hints Signed-off-by: Thomas Newton --- flytekit/core/python_function_task.py | 18 +++++++++++++----- flytekit/core/task.py | 18 +++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index d93e357a2e..72ee62a544 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -13,16 +13,18 @@ """ +from __future__ import annotations + from abc import ABC from collections import OrderedDict from enum import Enum from typing import Any, Callable, List, Optional, TypeVar, Union, cast +from flytekit.core import launch_plan as _annotated_launch_plan from flytekit.core.base_task import Task, TaskResolverMixin from flytekit.core.context_manager import ExecutionState, FlyteContext, FlyteContextManager from flytekit.core.docstring import Docstring from flytekit.core.interface import transform_function_to_interface -from flytekit.core.launch_plan import LaunchPlan from flytekit.core.promise import VoidPromise, translate_inputs_to_literals from flytekit.core.python_auto_container import PythonAutoContainerTask, default_task_resolver from flytekit.core.tracker import extract_task_module, is_functools_wrapped_module_level, isnested, istestfunction @@ -104,7 +106,9 @@ def __init__( ignore_input_vars: Optional[List[str]] = None, execution_mode: ExecutionBehavior = ExecutionBehavior.DEFAULT, task_resolver: Optional[TaskResolverMixin] = None, - output_entity_hint: Union["PythonFunctionTask", LaunchPlan, WorkflowBase] = None, + output_entity_hint: Optional[ + Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase] + ] = None, **kwargs, ): """ @@ -149,15 +153,19 @@ def __init__( self._task_function = task_function self._execution_mode = execution_mode self._output_entity_hint = output_entity_hint - assert (self._output_entity_hint is None) or (self._execution_mode == self.ExecutionBehavior.DYNAMIC), "output_entity_hint should only be specified on dynamic tasks." + assert (self._output_entity_hint is None) or ( + self._execution_mode == self.ExecutionBehavior.DYNAMIC + ), "output_entity_hint should only be specified on dynamic tasks." self._wf = None # For dynamic tasks @property def execution_mode(self) -> ExecutionBehavior: return self._execution_mode - + @property - def output_entity_hint(self) -> Union["PythonFunctionTask", LaunchPlan, WorkflowBase]: + def output_entity_hint( + self, + ) -> Optional[Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase]]: return self._output_entity_hint @property diff --git a/flytekit/core/task.py b/flytekit/core/task.py index c7be14351b..5187f76347 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -1,16 +1,18 @@ +from __future__ import annotations + import datetime as _datetime from functools import update_wrapper from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union, overload +from flytekit.core import launch_plan as _annotated_launchplan +from flytekit.core import workflow as _annotated_workflow from flytekit.core.base_task import TaskMetadata, TaskResolverMixin from flytekit.core.interface import transform_function_to_interface -from flytekit.core.launch_plan import LaunchPlan from flytekit.core.pod_template import PodTemplate from flytekit.core.python_function_task import PythonFunctionTask from flytekit.core.reference_entity import ReferenceEntity, TaskReference from flytekit.core.resources import Resources from flytekit.extras.accelerators import BaseAccelerator -from flytekit.core.workflow import WorkflowBase from flytekit.image_spec.image_spec import ImageSpec from flytekit.models.documentation import Documentation from flytekit.models.security import Secret @@ -99,7 +101,9 @@ def task( limits: Optional[Resources] = ..., secret_requests: Optional[List[Secret]] = ..., execution_mode: PythonFunctionTask.ExecutionBehavior = ..., - output_entity_hint: Union[PythonFunctionTask, LaunchPlan, WorkflowBase] = ..., + output_entity_hint: Optional[ + Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase] + ] = ..., task_resolver: Optional[TaskResolverMixin] = ..., docs: Optional[Documentation] = ..., disable_deck: Optional[bool] = ..., @@ -128,7 +132,9 @@ def task( limits: Optional[Resources] = ..., secret_requests: Optional[List[Secret]] = ..., execution_mode: PythonFunctionTask.ExecutionBehavior = ..., - output_entity_hint: Union[PythonFunctionTask, LaunchPlan, WorkflowBase] = ..., + output_entity_hint: Optional[ + Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase] + ] = ..., task_resolver: Optional[TaskResolverMixin] = ..., docs: Optional[Documentation] = ..., disable_deck: Optional[bool] = ..., @@ -156,7 +162,9 @@ def task( limits: Optional[Resources] = None, secret_requests: Optional[List[Secret]] = None, execution_mode: PythonFunctionTask.ExecutionBehavior = PythonFunctionTask.ExecutionBehavior.DEFAULT, - output_entity_hint: Union[PythonFunctionTask, LaunchPlan, WorkflowBase] = None, + output_entity_hint: Optional[ + Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase] + ] = None, task_resolver: Optional[TaskResolverMixin] = None, docs: Optional[Documentation] = None, disable_deck: Optional[bool] = None, From e37e96df52f5c5a7014f85ab9723df550d07a3d6 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Fri, 17 Nov 2023 19:48:49 +0000 Subject: [PATCH 06/19] Support iterable of output_entity_hints (#5) Signed-off-by: Thomas Newton --- flytekit/core/python_function_task.py | 18 +++++++++--------- flytekit/core/task.py | 16 ++++++++-------- flytekit/tools/translator.py | 5 +++-- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index 72ee62a544..e060a9f416 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -18,7 +18,7 @@ from abc import ABC from collections import OrderedDict from enum import Enum -from typing import Any, Callable, List, Optional, TypeVar, Union, cast +from typing import Any, Callable, Iterable, List, Optional, TypeVar, Union, cast from flytekit.core import launch_plan as _annotated_launch_plan from flytekit.core.base_task import Task, TaskResolverMixin @@ -106,8 +106,8 @@ def __init__( ignore_input_vars: Optional[List[str]] = None, execution_mode: ExecutionBehavior = ExecutionBehavior.DEFAULT, task_resolver: Optional[TaskResolverMixin] = None, - output_entity_hint: Optional[ - Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase] + output_entity_hints: Optional[ + Iterable[Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase]] ] = None, **kwargs, ): @@ -152,10 +152,10 @@ def __init__( ) self._task_function = task_function self._execution_mode = execution_mode - self._output_entity_hint = output_entity_hint - assert (self._output_entity_hint is None) or ( + self._output_entity_hints = output_entity_hints + assert (self._output_entity_hints is None) or ( self._execution_mode == self.ExecutionBehavior.DYNAMIC - ), "output_entity_hint should only be specified on dynamic tasks." + ), "output_entity_hints should only be specified on dynamic tasks." self._wf = None # For dynamic tasks @property @@ -163,10 +163,10 @@ def execution_mode(self) -> ExecutionBehavior: return self._execution_mode @property - def output_entity_hint( + def output_entity_hints( self, - ) -> Optional[Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase]]: - return self._output_entity_hint + ) -> Optional[Iterable[Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase]]]: + return self._output_entity_hints @property def task_function(self): diff --git a/flytekit/core/task.py b/flytekit/core/task.py index 5187f76347..74e1ae45d8 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -2,7 +2,7 @@ import datetime as _datetime from functools import update_wrapper -from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union, overload +from typing import Any, Callable, Dict, Iterable, List, Optional, Type, TypeVar, Union, overload from flytekit.core import launch_plan as _annotated_launchplan from flytekit.core import workflow as _annotated_workflow @@ -101,8 +101,8 @@ def task( limits: Optional[Resources] = ..., secret_requests: Optional[List[Secret]] = ..., execution_mode: PythonFunctionTask.ExecutionBehavior = ..., - output_entity_hint: Optional[ - Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase] + output_entity_hints: Optional[ + Iterable[Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase]] ] = ..., task_resolver: Optional[TaskResolverMixin] = ..., docs: Optional[Documentation] = ..., @@ -132,8 +132,8 @@ def task( limits: Optional[Resources] = ..., secret_requests: Optional[List[Secret]] = ..., execution_mode: PythonFunctionTask.ExecutionBehavior = ..., - output_entity_hint: Optional[ - Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase] + output_entity_hints: Optional[ + Iterable[Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase]] ] = ..., task_resolver: Optional[TaskResolverMixin] = ..., docs: Optional[Documentation] = ..., @@ -162,8 +162,8 @@ def task( limits: Optional[Resources] = None, secret_requests: Optional[List[Secret]] = None, execution_mode: PythonFunctionTask.ExecutionBehavior = PythonFunctionTask.ExecutionBehavior.DEFAULT, - output_entity_hint: Optional[ - Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase] + output_entity_hints: Optional[ + Iterable[Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase]] ] = None, task_resolver: Optional[TaskResolverMixin] = None, docs: Optional[Documentation] = None, @@ -289,7 +289,7 @@ def wrapper(fn: Callable[..., Any]) -> PythonFunctionTask[T]: limits=limits, secret_requests=secret_requests, execution_mode=execution_mode, - output_entity_hint=output_entity_hint, + output_entity_hints=output_entity_hints, task_resolver=task_resolver, disable_deck=disable_deck, enable_deck=enable_deck, diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index 110611c6cb..c18dfb11b2 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -178,8 +178,9 @@ def get_serializable_task( # during dynamic serialization settings = settings.with_serialized_context() - if entity.output_entity_hint is not None: - get_serializable(entity_mapping, settings, entity.output_entity_hint, options) + if entity.output_entity_hints is not None: + for output_entity_hint in entity.output_entity_hints: + get_serializable(entity_mapping, settings, output_entity_hint, options) container = entity.get_container(settings) # This pod will be incorrect when doing fast serialize From bcc90f0cd4edf397b312b146eeb99098bb3126a2 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Wed, 13 Dec 2023 18:51:18 +0000 Subject: [PATCH 07/19] Fix circular import for type hints Signed-off-by: Thomas Newton --- flytekit/core/workflow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flytekit/core/workflow.py b/flytekit/core/workflow.py index 41e421fe69..f6cbdf34d8 100644 --- a/flytekit/core/workflow.py +++ b/flytekit/core/workflow.py @@ -9,6 +9,7 @@ from typing import Any, Callable, Coroutine, Dict, List, Optional, Tuple, Type, Union, cast, overload from flytekit.core import constants as _common_constants +from flytekit.core import launch_plan as _annotated_launch_plan from flytekit.core.base_task import PythonTask, Task from flytekit.core.class_based_resolver import ClassStorageTaskResolver from flytekit.core.condition import ConditionalSection, conditional @@ -26,7 +27,6 @@ transform_inputs_to_parameters, transform_interface_to_typed_interface, ) -from flytekit.core.launch_plan import LaunchPlan from flytekit.core.node import Node from flytekit.core.promise import ( NodeOutput, @@ -529,7 +529,7 @@ def create_conditional(self, name: str) -> ConditionalSection: FlyteContextManager.with_context(ctx.with_compilation_state(self.compilation_state)) return conditional(name=name) - def add_entity(self, entity: Union[PythonTask, LaunchPlan, WorkflowBase], **kwargs) -> Node: + def add_entity(self, entity: Union[PythonTask, _annotated_launch_plan.LaunchPlan, WorkflowBase], **kwargs) -> Node: """ Anytime you add an entity, all the inputs to the entity must be bound. """ @@ -612,7 +612,7 @@ def add_workflow_output( def add_task(self, task: PythonTask, **kwargs) -> Node: return self.add_entity(task, **kwargs) - def add_launch_plan(self, launch_plan: LaunchPlan, **kwargs) -> Node: + def add_launch_plan(self, launch_plan: _annotated_launch_plan.LaunchPlan, **kwargs) -> Node: return self.add_entity(launch_plan, **kwargs) def add_subwf(self, sub_wf: WorkflowBase, **kwargs) -> Node: From e18510c2fa5079195e6ea420b9390c57bf804e59 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Wed, 13 Dec 2023 19:11:46 +0000 Subject: [PATCH 08/19] Fix tests Signed-off-by: Thomas Newton --- tests/flytekit/unit/core/test_container_task.py | 3 ++- .../flytekit/unit/core/test_python_auto_container.py | 11 ++++++----- tests/flytekit/unit/core/test_python_function_task.py | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/flytekit/unit/core/test_container_task.py b/tests/flytekit/unit/core/test_container_task.py index 3ac4a47cc4..94a8d9c9c6 100644 --- a/tests/flytekit/unit/core/test_container_task.py +++ b/tests/flytekit/unit/core/test_container_task.py @@ -1,4 +1,5 @@ import pytest +from collections import OrderedDict from kubernetes.client.models import ( V1Affinity, V1Container, @@ -72,7 +73,7 @@ def test_pod_template(): ################# # Test Serialization ################# - ts = get_serializable_task(default_serialization_settings, ct) + ts = get_serializable_task(OrderedDict(), default_serialization_settings, ct) assert ts.template.metadata.pod_template_name == "my-base-template" assert ts.template.container is None assert ts.template.k8s_pod is not None diff --git a/tests/flytekit/unit/core/test_python_auto_container.py b/tests/flytekit/unit/core/test_python_auto_container.py index fed612c98c..92d8689ae2 100644 --- a/tests/flytekit/unit/core/test_python_auto_container.py +++ b/tests/flytekit/unit/core/test_python_auto_container.py @@ -1,6 +1,7 @@ from typing import Any import pytest +from collections import OrderedDict from kubernetes.client.models import V1Container, V1EnvVar, V1PodSpec, V1ResourceRequirements, V1Volume from flytekit.configuration import Image, ImageConfig, SerializationSettings @@ -73,7 +74,7 @@ def test_get_container(default_serialization_settings): assert c.image == "docker.io/xyz:some-git-hash" assert c.env == {"FOO": "bar"} - ts = get_serializable_task(default_serialization_settings, task) + ts = get_serializable_task(OrderedDict(), default_serialization_settings, task) assert ts.template.container.image == "docker.io/xyz:some-git-hash" assert ts.template.container.env == {"FOO": "bar"} @@ -86,7 +87,7 @@ def test_get_container_with_task_envvars(default_serialization_settings): assert c.image == "docker.io/xyz:some-git-hash" assert c.env == {"FOO": "bar", "HAM": "spam"} - ts = get_serializable_task(default_serialization_settings, task_with_env_vars) + ts = get_serializable_task(OrderedDict(), default_serialization_settings, task_with_env_vars) assert ts.template.container.image == "docker.io/xyz:some-git-hash" assert ts.template.container.env == {"FOO": "bar", "HAM": "spam"} @@ -96,7 +97,7 @@ def test_get_container_without_serialization_settings_envvars(minimal_serializat assert c.image == "docker.io/xyz:some-git-hash" assert c.env == {"HAM": "spam"} - ts = get_serializable_task(minimal_serialization_settings, task_with_env_vars) + ts = get_serializable_task(OrderedDict(), minimal_serialization_settings, task_with_env_vars) assert ts.template.container.image == "docker.io/xyz:some-git-hash" assert ts.template.container.env == {"HAM": "spam"} @@ -215,7 +216,7 @@ def test_pod_template(default_serialization_settings): ################# # Test Serialization ################# - ts = get_serializable_task(default_serialization_settings, task_with_pod_template) + ts = get_serializable_task(OrderedDict(), default_serialization_settings, task_with_pod_template) assert ts.template.container is None # k8s_pod content is already verified above, so only check the existence here assert ts.template.k8s_pod is not None @@ -290,7 +291,7 @@ def test_minimum_pod_template(default_serialization_settings): ################# # Test Serialization ################# - ts = get_serializable_task(default_serialization_settings, task_with_minimum_pod_template) + ts = get_serializable_task(OrderedDict(), default_serialization_settings, task_with_minimum_pod_template) assert ts.template.container is None # k8s_pod content is already verified above, so only check the existence here assert ts.template.k8s_pod is not None diff --git a/tests/flytekit/unit/core/test_python_function_task.py b/tests/flytekit/unit/core/test_python_function_task.py index 9ba9d4c780..751bd39b73 100644 --- a/tests/flytekit/unit/core/test_python_function_task.py +++ b/tests/flytekit/unit/core/test_python_function_task.py @@ -1,4 +1,5 @@ import pytest +from collections import OrderedDict from kubernetes.client.models import V1Container, V1PodSpec from flytekit import task @@ -205,7 +206,7 @@ def func_with_pod_template(i: str): ################# # Test Serialization ################# - ts = get_serializable_task(default_serialization_settings, func_with_pod_template) + ts = get_serializable_task(OrderedDict(), default_serialization_settings, func_with_pod_template) assert ts.template.container is None # k8s_pod content is already verified above, so only check the existence here assert ts.template.k8s_pod is not None From 4672cfc9367ce2b954c9f3dcf92631e932d18c06 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Wed, 13 Dec 2023 23:14:41 +0000 Subject: [PATCH 09/19] Auto-format Signed-off-by: Thomas Newton --- tests/flytekit/unit/core/test_container_task.py | 3 ++- tests/flytekit/unit/core/test_python_auto_container.py | 2 +- tests/flytekit/unit/core/test_python_function_task.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/flytekit/unit/core/test_container_task.py b/tests/flytekit/unit/core/test_container_task.py index 94a8d9c9c6..c89ec11345 100644 --- a/tests/flytekit/unit/core/test_container_task.py +++ b/tests/flytekit/unit/core/test_container_task.py @@ -1,5 +1,6 @@ -import pytest from collections import OrderedDict + +import pytest from kubernetes.client.models import ( V1Affinity, V1Container, diff --git a/tests/flytekit/unit/core/test_python_auto_container.py b/tests/flytekit/unit/core/test_python_auto_container.py index 92d8689ae2..f5c7136b0b 100644 --- a/tests/flytekit/unit/core/test_python_auto_container.py +++ b/tests/flytekit/unit/core/test_python_auto_container.py @@ -1,7 +1,7 @@ +from collections import OrderedDict from typing import Any import pytest -from collections import OrderedDict from kubernetes.client.models import V1Container, V1EnvVar, V1PodSpec, V1ResourceRequirements, V1Volume from flytekit.configuration import Image, ImageConfig, SerializationSettings diff --git a/tests/flytekit/unit/core/test_python_function_task.py b/tests/flytekit/unit/core/test_python_function_task.py index 751bd39b73..aabe29bdca 100644 --- a/tests/flytekit/unit/core/test_python_function_task.py +++ b/tests/flytekit/unit/core/test_python_function_task.py @@ -1,5 +1,6 @@ -import pytest from collections import OrderedDict + +import pytest from kubernetes.client.models import V1Container, V1PodSpec from flytekit import task From 2be6e69262b547361df23a6cfeccec5978084715 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Fri, 15 Dec 2023 12:56:36 +0000 Subject: [PATCH 10/19] Add positive test case Signed-off-by: Thomas Newton --- tests/flytekit/unit/core/test_dynamic.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/flytekit/unit/core/test_dynamic.py b/tests/flytekit/unit/core/test_dynamic.py index b9b0ebd3fa..b61c2db635 100644 --- a/tests/flytekit/unit/core/test_dynamic.py +++ b/tests/flytekit/unit/core/test_dynamic.py @@ -1,4 +1,5 @@ import typing +from collections import OrderedDict import pytest @@ -13,6 +14,7 @@ from flytekit.core.type_engine import TypeEngine from flytekit.core.workflow import workflow from flytekit.models.literals import LiteralMap +from flytekit.tools.translator import get_serializable_task settings = flytekit.configuration.SerializationSettings( project="test_proj", @@ -262,3 +264,29 @@ def wf(wf_in: str) -> typing.List[str]: res = dt(ss="hello") assert res == ["In t2 string is hello", "In t3 string is In t2 string is hello"] + + +def test_dynamic_entity_hints_are_serialized(): + @task + def t1() -> int: + return 0 + + @task + def t2() -> int: + return 0 + + @dynamic(output_entity_hints=[t1, t2]) + def dt(mode: int) -> int: + if mode == 1: + return t1() + if mode == 2: + return t2() + + raise ValueError("Invalid mode") + + entity_mapping = OrderedDict() + get_serializable_task(entity_mapping, settings, dt) + + serialised_entities_iterator = iter(entity_mapping.values()) + assert "t1" in next(serialised_entities_iterator).template.id.name + assert "t2" in next(serialised_entities_iterator).template.id.name From 7f7ab2a33496d0d7ee2861ecd8f72196a8a0f225 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Thu, 14 Dec 2023 14:01:47 +0000 Subject: [PATCH 11/19] Add a test for disallowing entity hints on static tasks Signed-off-by: Thomas Newton --- .../flytekit/unit/core/test_python_function_task.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/flytekit/unit/core/test_python_function_task.py b/tests/flytekit/unit/core/test_python_function_task.py index aabe29bdca..1145488912 100644 --- a/tests/flytekit/unit/core/test_python_function_task.py +++ b/tests/flytekit/unit/core/test_python_function_task.py @@ -212,3 +212,16 @@ def func_with_pod_template(i: str): # k8s_pod content is already verified above, so only check the existence here assert ts.template.k8s_pod is not None assert ts.template.metadata.pod_template_name == "A" + + +def test_output_entity_hints_are_not_allowed(): + @task + def t1(i: str): + pass + + with pytest.raises(AssertionError): + @task( + output_entity_hints=[t1] + ) + def t2(i: str): + pass From 89ef3e91c2f1c9d8a4d9129248fb8bd51130af38 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Thu, 14 Dec 2023 14:10:31 +0000 Subject: [PATCH 12/19] raise instead of assert Signed-off-by: Thomas Newton --- flytekit/core/python_function_task.py | 6 +++--- tests/flytekit/unit/core/test_python_function_task.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index e060a9f416..f28976d8da 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -153,9 +153,9 @@ def __init__( self._task_function = task_function self._execution_mode = execution_mode self._output_entity_hints = output_entity_hints - assert (self._output_entity_hints is None) or ( - self._execution_mode == self.ExecutionBehavior.DYNAMIC - ), "output_entity_hints should only be specified on dynamic tasks." + if self._output_entity_hints is not None and self._execution_mode != self.ExecutionBehavior.DYNAMIC: + raise ValueError("output_entity_hints should only be used on dynamic tasks. On static tasks its redundant " + "because flyte can infer the output entities automatically") self._wf = None # For dynamic tasks @property diff --git a/tests/flytekit/unit/core/test_python_function_task.py b/tests/flytekit/unit/core/test_python_function_task.py index 1145488912..997168c0fe 100644 --- a/tests/flytekit/unit/core/test_python_function_task.py +++ b/tests/flytekit/unit/core/test_python_function_task.py @@ -6,7 +6,8 @@ from flytekit import task from flytekit.configuration import Image, ImageConfig, SerializationSettings from flytekit.core.pod_template import PodTemplate -from flytekit.core.python_auto_container import get_registerable_container_image +from flytekit.core.python_auto_container import \ + get_registerable_container_image from flytekit.core.python_function_task import PythonFunctionTask from flytekit.core.tracker import isnested, istestfunction from flytekit.image_spec.image_spec import ImageBuildEngine, ImageSpec @@ -219,7 +220,7 @@ def test_output_entity_hints_are_not_allowed(): def t1(i: str): pass - with pytest.raises(AssertionError): + with pytest.raises(ValueError, match="output_entity_hints should only be used on dynamic tasks"): @task( output_entity_hints=[t1] ) From a1d1016a53953cb06f64d5d1a7634937726a6ca9 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Fri, 15 Dec 2023 12:49:23 +0000 Subject: [PATCH 13/19] Rename output_entity_hints -> node_dependency_hints Signed-off-by: Thomas Newton --- flytekit/core/python_function_task.py | 14 +++++++------- flytekit/core/task.py | 8 ++++---- flytekit/tools/translator.py | 6 +++--- tests/flytekit/unit/core/test_dynamic.py | 4 ++-- .../unit/core/test_python_function_task.py | 13 +++++++------ 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index f28976d8da..480c4e911f 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -106,7 +106,7 @@ def __init__( ignore_input_vars: Optional[List[str]] = None, execution_mode: ExecutionBehavior = ExecutionBehavior.DEFAULT, task_resolver: Optional[TaskResolverMixin] = None, - output_entity_hints: Optional[ + node_dependency_hints: Optional[ Iterable[Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase]] ] = None, **kwargs, @@ -152,10 +152,10 @@ def __init__( ) self._task_function = task_function self._execution_mode = execution_mode - self._output_entity_hints = output_entity_hints - if self._output_entity_hints is not None and self._execution_mode != self.ExecutionBehavior.DYNAMIC: - raise ValueError("output_entity_hints should only be used on dynamic tasks. On static tasks its redundant " - "because flyte can infer the output entities automatically") + self._node_dependency_hints = node_dependency_hints + if self._node_dependency_hints is not None and self._execution_mode != self.ExecutionBehavior.DYNAMIC: + raise ValueError("node_dependency_hints should only be used on dynamic tasks. On static tasks and " + "workflows its redundant because flyte can find the node dependencies automatically") self._wf = None # For dynamic tasks @property @@ -163,10 +163,10 @@ def execution_mode(self) -> ExecutionBehavior: return self._execution_mode @property - def output_entity_hints( + def node_dependency_hints( self, ) -> Optional[Iterable[Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase]]]: - return self._output_entity_hints + return self._node_dependency_hints @property def task_function(self): diff --git a/flytekit/core/task.py b/flytekit/core/task.py index 74e1ae45d8..4451457a4e 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -101,7 +101,7 @@ def task( limits: Optional[Resources] = ..., secret_requests: Optional[List[Secret]] = ..., execution_mode: PythonFunctionTask.ExecutionBehavior = ..., - output_entity_hints: Optional[ + node_dependency_hints: Optional[ Iterable[Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase]] ] = ..., task_resolver: Optional[TaskResolverMixin] = ..., @@ -132,7 +132,7 @@ def task( limits: Optional[Resources] = ..., secret_requests: Optional[List[Secret]] = ..., execution_mode: PythonFunctionTask.ExecutionBehavior = ..., - output_entity_hints: Optional[ + node_dependency_hints: Optional[ Iterable[Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase]] ] = ..., task_resolver: Optional[TaskResolverMixin] = ..., @@ -162,7 +162,7 @@ def task( limits: Optional[Resources] = None, secret_requests: Optional[List[Secret]] = None, execution_mode: PythonFunctionTask.ExecutionBehavior = PythonFunctionTask.ExecutionBehavior.DEFAULT, - output_entity_hints: Optional[ + node_dependency_hints: Optional[ Iterable[Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase]] ] = None, task_resolver: Optional[TaskResolverMixin] = None, @@ -289,7 +289,7 @@ def wrapper(fn: Callable[..., Any]) -> PythonFunctionTask[T]: limits=limits, secret_requests=secret_requests, execution_mode=execution_mode, - output_entity_hints=output_entity_hints, + node_dependency_hints=node_dependency_hints, task_resolver=task_resolver, disable_deck=disable_deck, enable_deck=enable_deck, diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index c18dfb11b2..0049665c6e 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -178,9 +178,9 @@ def get_serializable_task( # during dynamic serialization settings = settings.with_serialized_context() - if entity.output_entity_hints is not None: - for output_entity_hint in entity.output_entity_hints: - get_serializable(entity_mapping, settings, output_entity_hint, options) + if entity.node_dependency_hints is not None: + for entity_hint in entity.node_dependency_hints: + get_serializable(entity_mapping, settings, entity_hint, options) container = entity.get_container(settings) # This pod will be incorrect when doing fast serialize diff --git a/tests/flytekit/unit/core/test_dynamic.py b/tests/flytekit/unit/core/test_dynamic.py index b61c2db635..44cc19b5c3 100644 --- a/tests/flytekit/unit/core/test_dynamic.py +++ b/tests/flytekit/unit/core/test_dynamic.py @@ -266,7 +266,7 @@ def wf(wf_in: str) -> typing.List[str]: assert res == ["In t2 string is hello", "In t3 string is In t2 string is hello"] -def test_dynamic_entity_hints_are_serialized(): +def test_node_dependency_hints_are_serialized(): @task def t1() -> int: return 0 @@ -275,7 +275,7 @@ def t1() -> int: def t2() -> int: return 0 - @dynamic(output_entity_hints=[t1, t2]) + @dynamic(node_dependency_hints=[t1, t2]) def dt(mode: int) -> int: if mode == 1: return t1() diff --git a/tests/flytekit/unit/core/test_python_function_task.py b/tests/flytekit/unit/core/test_python_function_task.py index 997168c0fe..45efaabf5d 100644 --- a/tests/flytekit/unit/core/test_python_function_task.py +++ b/tests/flytekit/unit/core/test_python_function_task.py @@ -215,14 +215,15 @@ def func_with_pod_template(i: str): assert ts.template.metadata.pod_template_name == "A" -def test_output_entity_hints_are_not_allowed(): +def test_node_dependency_hints_are_not_allowed(): @task def t1(i: str): pass - - with pytest.raises(ValueError, match="output_entity_hints should only be used on dynamic tasks"): - @task( - output_entity_hints=[t1] - ) + + with pytest.raises( + ValueError, match="node_dependency_hints should only be used on dynamic tasks" + ): + + @task(node_dependency_hints=[t1]) def t2(i: str): pass From fb97a71002fe0276c48dd4c857eb44af853c79c2 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Fri, 15 Dec 2023 14:14:01 +0000 Subject: [PATCH 14/19] Update docstrings Signed-off-by: Thomas Newton --- flytekit/core/python_function_task.py | 3 +++ flytekit/core/task.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index 480c4e911f..a198dabf36 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -119,6 +119,9 @@ def __init__( :param Optional[ExecutionBehavior] execution_mode: Defines how the execution should behave, for example executing normally or specially handling a dynamic case. :param str task_type: String task type to be associated with this Task + :param Optional[Iterable[Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase]]] + node_dependency_hints: A list of tasks, launchplans, or workflows that this task depends on. This is only + for dynamic tasks/workflows, where flyte cannot automatically determine the dependencies prior to runtime. """ if task_function is None: raise ValueError("TaskFunction is a required parameter for PythonFunctionTask") diff --git a/flytekit/core/task.py b/flytekit/core/task.py index 4451457a4e..c50a6af4e3 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -259,6 +259,27 @@ def foo2(): Refer to :py:class:`Secret` to understand how to specify the request for a secret. It may change based on the backend provider. :param execution_mode: This is mainly for internal use. Please ignore. It is filled in automatically. + :param node_dependency_hints: A list of tasks, launchplans, or workflows that this task depends on. This is only + for dynamic tasks/workflows, where flyte cannot automatically determine the dependencies prior to runtime. + Even on dynamic tasks this is optional, but in some scenarios it will make registering the workflow easier, + because it allows registration to be done the same as for static tasks/workflows. + + For example this is useful to run launchplans dynamically, because launchplans must be registered on flyteadmin + before they can be run. Tasks and workflows do not have this requirement. + + .. code-block:: python + @workflow + def workflow0(): + ... + + launchplan0 = LaunchPlan.get_or_create(workflow0) + + # Specify node_dependency_hints so that launchplan0 will be registered on flyteadmin, despite this being a + # dynamic task. + @dynamic(node_dependency_hints=[launchplan0]) + def launch_dynamically(): + # To run a sub-launchplan it must have previously been registered on flyteadmin. + return [launchplan0]*10 :param task_resolver: Provide a custom task resolver. :param disable_deck: (deprecated) If true, this task will not output deck html file :param enable_deck: If true, this task will output deck html file From 6cf0205f745cbcd3710167e3b7c847c32d2ef035 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Fri, 15 Dec 2023 14:15:14 +0000 Subject: [PATCH 15/19] Auto-format Signed-off-by: Thomas Newton --- flytekit/core/python_function_task.py | 6 ++++-- tests/flytekit/unit/core/test_python_function_task.py | 7 ++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index a198dabf36..ec52b61b87 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -157,8 +157,10 @@ def __init__( self._execution_mode = execution_mode self._node_dependency_hints = node_dependency_hints if self._node_dependency_hints is not None and self._execution_mode != self.ExecutionBehavior.DYNAMIC: - raise ValueError("node_dependency_hints should only be used on dynamic tasks. On static tasks and " - "workflows its redundant because flyte can find the node dependencies automatically") + raise ValueError( + "node_dependency_hints should only be used on dynamic tasks. On static tasks and " + "workflows its redundant because flyte can find the node dependencies automatically" + ) self._wf = None # For dynamic tasks @property diff --git a/tests/flytekit/unit/core/test_python_function_task.py b/tests/flytekit/unit/core/test_python_function_task.py index 45efaabf5d..9203b66bb9 100644 --- a/tests/flytekit/unit/core/test_python_function_task.py +++ b/tests/flytekit/unit/core/test_python_function_task.py @@ -6,8 +6,7 @@ from flytekit import task from flytekit.configuration import Image, ImageConfig, SerializationSettings from flytekit.core.pod_template import PodTemplate -from flytekit.core.python_auto_container import \ - get_registerable_container_image +from flytekit.core.python_auto_container import get_registerable_container_image from flytekit.core.python_function_task import PythonFunctionTask from flytekit.core.tracker import isnested, istestfunction from flytekit.image_spec.image_spec import ImageBuildEngine, ImageSpec @@ -220,9 +219,7 @@ def test_node_dependency_hints_are_not_allowed(): def t1(i: str): pass - with pytest.raises( - ValueError, match="node_dependency_hints should only be used on dynamic tasks" - ): + with pytest.raises(ValueError, match="node_dependency_hints should only be used on dynamic tasks"): @task(node_dependency_hints=[t1]) def t2(i: str): From cf1cba53425f4fcbacffa9fccf16b48658b4cbe9 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 19 Dec 2023 16:13:23 +0000 Subject: [PATCH 16/19] Fix test_serialize_vscode Signed-off-by: Thomas Newton --- plugins/flytekit-flyin/tests/test_flyin_plugin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/flytekit-flyin/tests/test_flyin_plugin.py b/plugins/flytekit-flyin/tests/test_flyin_plugin.py index 7eb937394d..9f0dbb9624 100644 --- a/plugins/flytekit-flyin/tests/test_flyin_plugin.py +++ b/plugins/flytekit-flyin/tests/test_flyin_plugin.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + import mock import pytest from flytekitplugins.flyin import ( @@ -318,7 +320,7 @@ def t(): project="p", domain="d", version="v", image_config=default_image_config ) - serialized_task = get_serializable_task(default_serialization_settings, t) + serialized_task = get_serializable_task(OrderedDict(), default_serialization_settings, t) assert serialized_task.template.config == {"link_type": "vscode", "port": "8081"} From 8d1e73446a9aa7fbc2115f76258c2ea11c516d30 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 19 Dec 2023 20:51:47 +0000 Subject: [PATCH 17/19] Maybe fix the docs error Signed-off-by: Thomas Newton --- flytekit/core/python_function_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index ec52b61b87..c26bdd6c6e 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -119,8 +119,8 @@ def __init__( :param Optional[ExecutionBehavior] execution_mode: Defines how the execution should behave, for example executing normally or specially handling a dynamic case. :param str task_type: String task type to be associated with this Task - :param Optional[Iterable[Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase]]] - node_dependency_hints: A list of tasks, launchplans, or workflows that this task depends on. This is only + :param Optional[Iterable[Union["PythonFunctionTask", "_annotated_launch_plan.LaunchPlan", WorkflowBase]]] node_dependency_hints: + A list of tasks, launchplans, or workflows that this task depends on. This is only for dynamic tasks/workflows, where flyte cannot automatically determine the dependencies prior to runtime. """ if task_function is None: From 62c52091d9072c340581aff3851e769f8e4d1a04 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Wed, 20 Dec 2023 00:15:00 +0000 Subject: [PATCH 18/19] Fix a newly added test from master Signed-off-by: Thomas Newton --- tests/flytekit/unit/core/test_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/flytekit/unit/core/test_utils.py b/tests/flytekit/unit/core/test_utils.py index 910156f50a..ca0d07565d 100644 --- a/tests/flytekit/unit/core/test_utils.py +++ b/tests/flytekit/unit/core/test_utils.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + import pytest import flytekit @@ -90,7 +92,7 @@ def t() -> str: assert t() == "hello world" assert t.get_config(settings=ss) == {} - ts = get_serializable_task(ss, t) + ts = get_serializable_task(OrderedDict(), ss, t) assert ts.template.config == {"foo": "bar"} @task @@ -98,5 +100,5 @@ def t() -> str: def t() -> str: return "hello world" - ts = get_serializable_task(ss, t) + ts = get_serializable_task(OrderedDict(), ss, t) assert ts.template.config == {"foo": "baz"} From 6ae0e89d8996512ba80126d1e49387f607241b37 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Wed, 20 Dec 2023 15:41:04 +0000 Subject: [PATCH 19/19] Fix monodocs build Signed-off-by: Thomas Newton --- flytekit/core/task.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flytekit/core/task.py b/flytekit/core/task.py index c50a6af4e3..a99fbf599e 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -268,6 +268,7 @@ def foo2(): before they can be run. Tasks and workflows do not have this requirement. .. code-block:: python + @workflow def workflow0(): ...