diff --git a/flytekit/__init__.py b/flytekit/__init__.py index 376987f6a6..f1ff472f73 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -1,5 +1,20 @@ -from __future__ import absolute_import +import logging as _logging import flytekit.plugins # noqa: F401 -__version__ = "0.12.6" +__version__ = "0.13.0b4" + +logger = _logging.getLogger("flytekit") + +# create console handler and set level to debug +ch = _logging.StreamHandler() +ch.setLevel(_logging.DEBUG) + +# create formatter +formatter = _logging.Formatter("%(asctime)s-%(name)s-%(levelname)s$ %(message)s") + +# add formatter to ch +ch.setFormatter(formatter) + +# add ch to logger +logger.addHandler(ch) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 98e1a464ac..8adafd51bc 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime import importlib as _importlib import os as _os diff --git a/flytekit/clients/friendly.py b/flytekit/clients/friendly.py index 05f16d37e0..21a4b280f4 100644 --- a/flytekit/clients/friendly.py +++ b/flytekit/clients/friendly.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import six as _six from flyteidl.admin import common_pb2 as _common_pb2 from flyteidl.admin import execution_pb2 as _execution_pb2 diff --git a/flytekit/clients/raw.py b/flytekit/clients/raw.py index 758ba4b74a..c3076dcd43 100644 --- a/flytekit/clients/raw.py +++ b/flytekit/clients/raw.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import logging as _logging import six as _six diff --git a/flytekit/clis/auth/credentials.py b/flytekit/clis/auth/credentials.py index 05d41e9f42..4e2a700f7c 100644 --- a/flytekit/clis/auth/credentials.py +++ b/flytekit/clis/auth/credentials.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import logging as _logging import urllib.parse as _urlparse diff --git a/flytekit/clis/flyte_cli/main.py b/flytekit/clis/flyte_cli/main.py index c111fa6787..882639e925 100644 --- a/flytekit/clis/flyte_cli/main.py +++ b/flytekit/clis/flyte_cli/main.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import importlib as _importlib import os as _os import stat as _stat diff --git a/flytekit/clis/helpers.py b/flytekit/clis/helpers.py index 741cf5dbf2..c38baa638a 100644 --- a/flytekit/clis/helpers.py +++ b/flytekit/clis/helpers.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import six as _six from flytekit.common.types.helpers import get_sdk_type_from_literal_type as _get_sdk_type_from_literal_type diff --git a/flytekit/clis/sdk_in_container/basic_auth.py b/flytekit/clis/sdk_in_container/basic_auth.py index e74d0bd08f..8806c5c406 100644 --- a/flytekit/clis/sdk_in_container/basic_auth.py +++ b/flytekit/clis/sdk_in_container/basic_auth.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import base64 as _base64 import logging as _logging diff --git a/flytekit/clis/sdk_in_container/launch_plan.py b/flytekit/clis/sdk_in_container/launch_plan.py index bb8e5e0239..b8d1f18cef 100644 --- a/flytekit/clis/sdk_in_container/launch_plan.py +++ b/flytekit/clis/sdk_in_container/launch_plan.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import logging as _logging import click diff --git a/flytekit/clis/sdk_in_container/pyflyte.py b/flytekit/clis/sdk_in_container/pyflyte.py index 1be298b9c2..93da6fd54e 100644 --- a/flytekit/clis/sdk_in_container/pyflyte.py +++ b/flytekit/clis/sdk_in_container/pyflyte.py @@ -1,15 +1,9 @@ -from __future__ import absolute_import, print_function - import logging as _logging import os as _os +from pathlib import Path import click -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path # python 2 backport - from flytekit.clis.sdk_in_container.constants import CTX_DOMAIN, CTX_PACKAGES, CTX_PROJECT, CTX_VERSION from flytekit.clis.sdk_in_container.launch_plan import launch_plans from flytekit.clis.sdk_in_container.register import register diff --git a/flytekit/clis/sdk_in_container/register.py b/flytekit/clis/sdk_in_container/register.py index 249a45ab95..76c5489325 100644 --- a/flytekit/clis/sdk_in_container/register.py +++ b/flytekit/clis/sdk_in_container/register.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import logging as _logging import click diff --git a/flytekit/clis/sdk_in_container/serialize.py b/flytekit/clis/sdk_in_container/serialize.py index a977c42b03..785b1d2374 100644 --- a/flytekit/clis/sdk_in_container/serialize.py +++ b/flytekit/clis/sdk_in_container/serialize.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - import logging as _logging import math as _math import os as _os diff --git a/flytekit/common/component_nodes.py b/flytekit/common/component_nodes.py index 3a8f1d3d65..ddd59424db 100644 --- a/flytekit/common/component_nodes.py +++ b/flytekit/common/component_nodes.py @@ -1,15 +1,11 @@ -from __future__ import absolute_import - import logging as _logging -import six as _six - from flytekit.common import sdk_bases as _sdk_bases from flytekit.common.exceptions import system as _system_exceptions from flytekit.models.core import workflow as _workflow_model -class SdkTaskNode(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _workflow_model.TaskNode)): +class SdkTaskNode(_workflow_model.TaskNode, metaclass=_sdk_bases.ExtendedSdkType): def __init__(self, sdk_task): """ :param flytekit.common.tasks.task.SdkTask sdk_task: @@ -60,7 +56,7 @@ def promote_from_model(cls, base_model, tasks): return cls(sdk_task) -class SdkWorkflowNode(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _workflow_model.WorkflowNode)): +class SdkWorkflowNode(_workflow_model.WorkflowNode, metaclass=_sdk_bases.ExtendedSdkType): def __init__(self, sdk_workflow=None, sdk_launch_plan=None): """ :param flytekit.common.workflow.SdkWorkflow sdk_workflow: diff --git a/flytekit/common/constants.py b/flytekit/common/constants.py index 10f7fe8ec1..c97853ccde 100644 --- a/flytekit/common/constants.py +++ b/flytekit/common/constants.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - INPUT_FILE_NAME = "inputs.pb" OUTPUT_FILE_NAME = "outputs.pb" FUTURES_FILE_NAME = "futures.pb" diff --git a/flytekit/common/core/identifier.py b/flytekit/common/core/identifier.py index 1ef112215a..1510bdf293 100644 --- a/flytekit/common/core/identifier.py +++ b/flytekit/common/core/identifier.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import six as _six from flytekit.common import sdk_bases as _sdk_bases @@ -7,7 +5,7 @@ from flytekit.models.core import identifier as _core_identifier -class Identifier(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _core_identifier.Identifier)): +class Identifier(_core_identifier.Identifier, metaclass=_sdk_bases.ExtendedSdkType): _STRING_TO_TYPE_MAP = { "lp": _core_identifier.ResourceType.LAUNCH_PLAN, @@ -61,9 +59,7 @@ def __str__(self): ) -class WorkflowExecutionIdentifier( - _six.with_metaclass(_sdk_bases.ExtendedSdkType, _core_identifier.WorkflowExecutionIdentifier) -): +class WorkflowExecutionIdentifier(_core_identifier.WorkflowExecutionIdentifier, metaclass=_sdk_bases.ExtendedSdkType): @classmethod def promote_from_model(cls, base_model): """ @@ -101,9 +97,7 @@ def __str__(self): return "ex:{}:{}:{}".format(self.project, self.domain, self.name) -class TaskExecutionIdentifier( - _six.with_metaclass(_sdk_bases.ExtendedSdkType, _core_identifier.TaskExecutionIdentifier) -): +class TaskExecutionIdentifier(_core_identifier.TaskExecutionIdentifier, metaclass=_sdk_bases.ExtendedSdkType): @classmethod def promote_from_model(cls, base_model): """ diff --git a/flytekit/common/exceptions/base.py b/flytekit/common/exceptions/base.py index a8c1087556..2fca878abf 100644 --- a/flytekit/common/exceptions/base.py +++ b/flytekit/common/exceptions/base.py @@ -1,15 +1,10 @@ -from __future__ import absolute_import - -import six as _six - - class _FlyteCodedExceptionMetaclass(type): @property def error_code(cls): return cls._ERROR_CODE -class FlyteException(_six.with_metaclass(_FlyteCodedExceptionMetaclass, Exception)): +class FlyteException(Exception, metaclass=_FlyteCodedExceptionMetaclass): _ERROR_CODE = "UnknownFlyteException" diff --git a/flytekit/common/exceptions/scopes.py b/flytekit/common/exceptions/scopes.py index 671e045db5..fdd4e1a802 100644 --- a/flytekit/common/exceptions/scopes.py +++ b/flytekit/common/exceptions/scopes.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from sys import exc_info as _exc_info from traceback import format_tb as _format_tb diff --git a/flytekit/common/exceptions/system.py b/flytekit/common/exceptions/system.py index a3787f8234..5802279590 100644 --- a/flytekit/common/exceptions/system.py +++ b/flytekit/common/exceptions/system.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.exceptions import base as _base_exceptions diff --git a/flytekit/common/exceptions/user.py b/flytekit/common/exceptions/user.py index 102e94413c..671ebf66ec 100644 --- a/flytekit/common/exceptions/user.py +++ b/flytekit/common/exceptions/user.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.exceptions.base import FlyteException as _FlyteException from flytekit.common.exceptions.base import FlyteRecoverableException as _Recoverable diff --git a/flytekit/common/interface.py b/flytekit/common/interface.py index 37fe4b64fa..0f8e5569fa 100644 --- a/flytekit/common/interface.py +++ b/flytekit/common/interface.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import six as _six from flytekit.common import promise as _promise @@ -12,7 +10,7 @@ from flytekit.models import literals as _literal_models -class BindingData(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _literal_models.BindingData)): +class BindingData(_literal_models.BindingData, metaclass=_sdk_bases.ExtendedSdkType): @staticmethod def _has_sub_bindings(m): """ @@ -96,7 +94,7 @@ def from_python_std(cls, literal_type, t_value, upstream_nodes=None): return cls(scalar=scalar, collection=collection, map=map, promise=promise) -class TypedInterface(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _interface_models.TypedInterface)): +class TypedInterface(_interface_models.TypedInterface, metaclass=_sdk_bases.ExtendedSdkType): @classmethod def promote_from_model(cls, model): """ diff --git a/flytekit/common/launch_plan.py b/flytekit/common/launch_plan.py index a6d217e710..f720e5c38c 100644 --- a/flytekit/common/launch_plan.py +++ b/flytekit/common/launch_plan.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime import logging as _logging import uuid as _uuid @@ -21,8 +19,9 @@ from flytekit.common.types import helpers as _type_helpers from flytekit.configuration import auth as _auth_config from flytekit.configuration import sdk as _sdk_config -from flytekit.engines import loader as _engine_loader +from flytekit.engines.flyte import engine as _flyte_engine from flytekit.models import common as _common_models +from flytekit.models import execution as _execution_models from flytekit.models import interface as _interface_models from flytekit.models import launch_plan as _launch_plan_models from flytekit.models import literals as _literal_models @@ -32,9 +31,11 @@ class SdkLaunchPlan( - _six.with_metaclass( - _sdk_bases.ExtendedSdkType, _launch_plan_models.LaunchPlanSpec, _launchable_mixin.LaunchableEntity, - ) + _launchable_mixin.LaunchableEntity, + _registerable.HasDependencies, + _registerable.RegisterableEntity, + _launch_plan_models.LaunchPlanSpec, + metaclass=_sdk_bases.ExtendedSdkType, ): def __init__(self, *args, **kwargs): super(SdkLaunchPlan, self).__init__(*args, **kwargs) @@ -45,7 +46,7 @@ def __init__(self, *args, **kwargs): self._interface = None @classmethod - def promote_from_model(cls, model): + def promote_from_model(cls, model) -> "SdkLaunchPlan": """ :param flytekit.models.launch_plan.LaunchPlanSpec model: :rtype: SdkLaunchPlan @@ -66,6 +67,27 @@ def promote_from_model(cls, model): raw_output_data_config=model.raw_output_data_config, ) + @_exception_scopes.system_entry_point + def register(self, project, domain, name, version): + """ + :param Text project: + :param Text domain: + :param Text name: + :param Text version: + """ + self.validate() + id_to_register = _identifier.Identifier( + _identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name, version + ) + client = _flyte_engine.get_client() + try: + client.create_launch_plan(id_to_register, self) + except _user_exceptions.FlyteEntityAlreadyExistsException: + pass + + self._id = id_to_register + return str(self.id) + @classmethod @_exception_scopes.system_entry_point def fetch(cls, project, domain, name, version=None): @@ -83,7 +105,15 @@ def fetch(cls, project, domain, name, version=None): launch_plan_id = _identifier.Identifier( _identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name, version ) - lp = _engine_loader.get_engine().fetch_launch_plan(launch_plan_id) + + if launch_plan_id.version: + lp = _flyte_engine.get_client().get_launch_plan(launch_plan_id) + else: + named_entity_id = _common_models.NamedEntityIdentifier( + launch_plan_id.project, launch_plan_id.domain, launch_plan_id.name + ) + lp = _flyte_engine.get_client().get_active_launch_plan(named_entity_id) + sdk_lp = cls.promote_from_model(lp.spec) sdk_lp._id = lp.id @@ -91,8 +121,17 @@ def fetch(cls, project, domain, name, version=None): wf_id = sdk_lp.workflow_id lp_wf = _workflow.SdkWorkflow.fetch(wf_id.project, wf_id.domain, wf_id.name, wf_id.version) sdk_lp._interface = lp_wf.interface + sdk_lp._has_registered = True return sdk_lp + @_exception_scopes.system_entry_point + def serialize(self): + """ + Unlike the SdkWorkflow serialize call, nothing special needs to be done here. + :rtype: flyteidl.admin.launch_plan_pb2.LaunchPlanSpec + """ + return self.to_flyte_idl() + @property def id(self): """ @@ -128,16 +167,26 @@ def auth_role(self): if not (assumable_iam_role or kubernetes_service_account): _logging.warning( - "Using deprecated `role` from config. " "Please update your config to use `assumable_iam_role` instead" + "Using deprecated `role` from config. Please update your config to use `assumable_iam_role` instead" ) assumable_iam_role = _sdk_config.ROLE.get() return _common_models.AuthRole( assumable_iam_role=assumable_iam_role, kubernetes_service_account=kubernetes_service_account, ) + @property + def workflow_id(self): + """ + :rtype: flytekit.common.core.identifier.Identifier + """ + return self._workflow_id + @property def interface(self): """ + The interface is not technically part of the admin.LaunchPlanSpec in the IDL, however the workflow ID is, and + from the workflow ID, fetch will fill in the interface. This is nice because then you can __call__ the= + object and get a node. :rtype: flytekit.common.interface.TypedInterface """ return self._interface @@ -184,7 +233,7 @@ def update(self, state): "Failed to update launch plan because the launch plan's ID is not set. Please call register to fetch " "or register the identifier first" ) - return _engine_loader.get_engine().get_launch_plan(self).update(self.id, state) + return _flyte_engine.get_client().update_launch_plan(self.id, state) def _python_std_input_map_to_literal_map(self, inputs): """ @@ -244,19 +293,36 @@ def launch_with_literals( """ # Kubernetes requires names starting with an alphabet for some resources. name = name or "f" + _uuid.uuid4().hex[:19] - execution = ( - _engine_loader.get_engine() - .get_launch_plan(self) - .launch( + disable_all = notification_overrides == [] + if disable_all: + notification_overrides = None + else: + notification_overrides = _execution_models.NotificationList(notification_overrides or []) + disable_all = None + + client = _flyte_engine.get_client() + try: + exec_id = client.create_execution( project, domain, name, + _execution_models.ExecutionSpec( + self.id, + _execution_models.ExecutionMetadata( + _execution_models.ExecutionMetadata.ExecutionMode.MANUAL, + "sdk", # TODO: get principle + 0, # TODO: Detect nesting + ), + notifications=notification_overrides, + disable_all=disable_all, + labels=label_overrides, + annotations=annotation_overrides, + ), literal_inputs, - notification_overrides=notification_overrides, - label_overrides=label_overrides, - annotation_overrides=annotation_overrides, ) - ) + except _user_exceptions.FlyteEntityAlreadyExistsException: + exec_id = _identifier.WorkflowExecutionIdentifier(project, domain, name) + execution = client.get_execution(exec_id) return _workflow_execution.SdkWorkflowExecution.promote_from_model(execution) @_exception_scopes.system_entry_point @@ -295,9 +361,7 @@ def __repr__(self): # The difference between this and the SdkLaunchPlan class is that this runnable class is supposed to only be used for # launch plans loaded alongside the current Python interpreter. -class SdkRunnableLaunchPlan( - _hash_mixin.HashOnReferenceMixin, SdkLaunchPlan, _registerable.RegisterableEntity, -): +class SdkRunnableLaunchPlan(_hash_mixin.HashOnReferenceMixin, SdkLaunchPlan): def __init__( self, sdk_workflow, @@ -312,7 +376,7 @@ def __init__( raw_output_data_config=None, ): """ - :param flytekit.common.workflow.SdkWorkflow sdk_workflow: + :param flytekit.common.local_workflow.SdkRunnableWorkflow sdk_workflow: :param dict[Text,flytekit.common.promise.Input] default_inputs: :param dict[Text,Any] fixed_inputs: These inputs will be fixed and not need to be set when executing this launch plan. @@ -367,30 +431,6 @@ def __init__( self._upstream_entities = {sdk_workflow} self._sdk_workflow = sdk_workflow - @_exception_scopes.system_entry_point - def register(self, project, domain, name, version): - """ - :param Text project: - :param Text domain: - :param Text name: - :param Text version: - """ - self.validate() - id_to_register = _identifier.Identifier( - _identifier_model.ResourceType.LAUNCH_PLAN, project, domain, name, version - ) - _engine_loader.get_engine().get_launch_plan(self).register(id_to_register) - self._id = id_to_register - return _six.text_type(self.id) - - @_exception_scopes.system_entry_point - def serialize(self): - """ - Unlike the SdkWorkflow serialize call, nothing special needs to be done here. - :rtype: flyteidl.admin.launch_plan_pb2.LaunchPlanSpec - """ - return self.to_flyte_idl() - @classmethod def from_flyte_idl(cls, _): raise _user_exceptions.FlyteAssertion( @@ -399,10 +439,6 @@ def from_flyte_idl(cls, _): @classmethod def promote_from_model(cls, model): - """ - :param flytekit.models.launch_plan.LaunchPlanSpec model: - :rtype: SdkRunnableLaunchPlan - """ raise _user_exceptions.FlyteAssertion( "An SdkRunnableLaunchPlan must be created from a reference to local Python code only." ) @@ -422,21 +458,6 @@ def fetch(cls, project, domain, name, version=None): "An SdkRunnableLaunchPlan must be created from a reference to local Python code only." ) - @property - def interface(self): - """ - :rtype: flytekit.common.interface.TypedInterface - """ - return self._interface - - @property - def upstream_entities(self): - """ - Task, workflow, and launch plan that need to be registered in advance of this workflow. - :rtype: set[_registerable.RegisterableEntity] - """ - return self._upstream_entities - @property def workflow_id(self): """ diff --git a/flytekit/common/local_workflow.py b/flytekit/common/local_workflow.py new file mode 100644 index 0000000000..9aaecf5385 --- /dev/null +++ b/flytekit/common/local_workflow.py @@ -0,0 +1,385 @@ +import uuid as _uuid +from typing import Any, Dict, List + +import six as _six +from six.moves import queue as _queue + +from flytekit.common import interface as _interface +from flytekit.common import launch_plan as _launch_plan +from flytekit.common import nodes as _nodes +from flytekit.common import promise as _promise +from flytekit.common.core import identifier as _identifier +from flytekit.common.exceptions import user as _user_exceptions +from flytekit.common.types import helpers as _type_helpers +from flytekit.common.workflow import SdkWorkflow +from flytekit.configuration import internal as _internal_config +from flytekit.models import common as _common_models +from flytekit.models import interface as _interface_models +from flytekit.models import literals as _literal_models +from flytekit.models import schedule as _schedule_models +from flytekit.models.core import identifier as _identifier_model +from flytekit.models.core import workflow as _workflow_models + + +# Local-only wrapper around binding data and variables. Note that the Output object used by the end user is a yet +# another layer on top of this. +class Output(object): + def __init__(self, name, value, sdk_type=None, help=None): + """ + :param Text name: + :param T value: + :param U sdk_type: If specified, the value provided must cast to this type. Normally should be an instance of + flytekit.common.types.base_sdk_types.FlyteSdkType. But could also be something like: + + list[flytekit.common.types.base_sdk_types.FlyteSdkType], + dict[flytekit.common.types.base_sdk_types.FlyteSdkType,flytekit.common.types.base_sdk_types.FlyteSdkType], + (flytekit.common.types.base_sdk_types.FlyteSdkType, flytekit.common.types.base_sdk_types.FlyteSdkType, ...) + """ + if sdk_type is None: + # This syntax didn't work for some reason: sdk_type = sdk_type or Output._infer_type(value) + sdk_type = Output._infer_type(value) + sdk_type = _type_helpers.python_std_to_sdk_type(sdk_type) + + self._binding_data = _interface.BindingData.from_python_std(sdk_type.to_flyte_literal_type(), value) + self._var = _interface_models.Variable(sdk_type.to_flyte_literal_type(), help or "") + self._name = name + + def rename_and_return_reference(self, new_name): + self._name = new_name + return self + + @staticmethod + def _infer_type(value): + # TODO: Infer types + raise NotImplementedError( + "Currently the SDK cannot infer a workflow output type, so please use the type kwarg " + "when instantiating an output." + ) + + @property + def name(self): + """ + :rtype: Text + """ + return self._name + + @property + def binding_data(self): + """ + :rtype: flytekit.models.literals.BindingData + """ + return self._binding_data + + @property + def var(self): + """ + :rtype: flytekit.models.interface.Variable + """ + return self._var + + +class SdkRunnableWorkflow(SdkWorkflow): + """ + Wrapper class for workflows defined using Python, written in a Flyte workflow repo. This class is misnamed. It is + more appropriately called PythonWorkflow. The reason we are calling it SdkRunnableWorkflow instead is merely in + keeping with the established convention in other parts of this codebase. We will likely change this naming scheme + entirely before a 1.0 release. + + Being "runnable" or not "runnable" is not a distinction we care to make at this point. Do not read into it, + pretend it's not there. The purpose of this class is merely to differentiate between + i) A workflow object, as created by a user's workflow code, using the @workflow_class decorator for instance. If + you have one of these classes, it means you have the actual Python code available in the Python process you + are running. + ii) The SdkWorkflow object, which represents a workflow as retrieved from Flyte Admin. Anyone with access + to Admin, can instantiate an SdkWorkflow object by 'fetch'ing it. You don't need to have any of + the actual code checked out. SdkWorkflow's are effectively then the control plane model of a workflow, + represented as a Python object. + """ + + def __init__( + self, + inputs: List[_promise.Input], + nodes: List[_nodes.SdkNode], + interface, + output_bindings, + id=None, + metadata=None, + metadata_defaults=None, + disable_default_launch_plan=False, + ): + """ + :param list[flytekit.common.nodes.SdkNode] nodes: + :param flytekit.models.interface.TypedInterface interface: Defines a strongly typed interface for the + Workflow (inputs, outputs). This can include some optional parameters. + :param list[flytekit.models.literals.Binding] output_bindings: A list of output bindings that specify how to construct + workflow outputs. Bindings can pull node outputs or specify literals. All workflow outputs specified in + the interface field must be bound + in order for the workflow to be validated. A workflow has an implicit dependency on all of its nodes + to execute successfully in order to bind final outputs. + :param flytekit.models.core.identifier.Identifier id: This is an autogenerated id by the system. The id is + globally unique across Flyte. + :param WorkflowMetadata metadata: This contains information on how to run the workflow. + :param flytekit.models.core.workflow.WorkflowMetadataDefaults metadata_defaults: Defaults to be passed + to nodes contained within workflow. + :param bool disable_default_launch_plan: Determines whether to create a default launch plan for the workflow. + """ + # Save the promise.Input objects for future use. + self._user_inputs = inputs + + # Set optional settings + id = ( + id + if id is not None + else _identifier.Identifier( + _identifier_model.ResourceType.WORKFLOW, + _internal_config.PROJECT.get(), + _internal_config.DOMAIN.get(), + _uuid.uuid4().hex, + _internal_config.VERSION.get(), + ) + ) + metadata = metadata if metadata is not None else _workflow_models.WorkflowMetadata() + metadata_defaults = ( + metadata_defaults if metadata_defaults is not None else _workflow_models.WorkflowMetadataDefaults() + ) + + super(SdkRunnableWorkflow, self).__init__( + nodes=nodes, + interface=interface, + output_bindings=output_bindings, + id=id, + metadata=metadata, + metadata_defaults=metadata_defaults, + ) + + # Set this last as it's set in constructor + self._upstream_entities = set(n.executable_sdk_object for n in nodes) + self._should_create_default_launch_plan = not disable_default_launch_plan + + @property + def should_create_default_launch_plan(self): + """ + Determines whether registration flow should create a default launch plan for this workflow or not. + :rtype: bool + """ + return self._should_create_default_launch_plan + + def __call__(self, *args, **input_map): + # Take the default values from the Inputs + compiled_inputs = {v.name: v.sdk_default for v in self.user_inputs if not v.sdk_required} + compiled_inputs.update(input_map) + + return super().__call__(*args, **compiled_inputs) + + @classmethod + def construct_from_class_definition( + cls, + inputs: List[_promise.Input], + outputs: List[Output], + nodes: List[_nodes.SdkNode], + metadata: _workflow_models.WorkflowMetadata = None, + metadata_defaults: _workflow_models.WorkflowMetadataDefaults = None, + disable_default_launch_plan: bool = False, + ) -> "SdkRunnableWorkflow": + """ + This constructor is here to provide backwards-compatibility for class-defined Workflows + + :param list[flytekit.common.promise.Input] inputs: + :param list[Output] outputs: + :param list[flytekit.common.nodes.SdkNode] nodes: + :param WorkflowMetadata metadata: This contains information on how to run the workflow. + :param flytekit.models.core.workflow.WorkflowMetadataDefaults metadata_defaults: Defaults to be passed + to nodes contained within workflow. + :param bool disable_default_launch_plan: Determines whether to create a default launch plan for the workflow or not. + + :rtype: SdkRunnableWorkflow + """ + for n in nodes: + for upstream in n.upstream_nodes: + if upstream.id is None: + raise _user_exceptions.FlyteAssertion( + "Some nodes contained in the workflow were not found in the workflow description. Please " + "ensure all nodes are either assigned to attributes within the class or an element in a " + "list, dict, or tuple which is stored as an attribute in the class." + ) + + id = _identifier.Identifier( + _identifier_model.ResourceType.WORKFLOW, + _internal_config.PROJECT.get(), + _internal_config.DOMAIN.get(), + _uuid.uuid4().hex, + _internal_config.VERSION.get(), + ) + interface = _interface.TypedInterface({v.name: v.var for v in inputs}, {v.name: v.var for v in outputs}) + + output_bindings = [_literal_models.Binding(v.name, v.binding_data) for v in outputs] + + return cls( + inputs=inputs, + nodes=nodes, + interface=interface, + output_bindings=output_bindings, + id=id, + metadata=metadata, + metadata_defaults=metadata_defaults, + disable_default_launch_plan=disable_default_launch_plan, + ) + + @property + def id(self): + return self._id + + @id.setter + def id(self, new_id): + self._id = new_id + + @property + def user_inputs(self) -> List[_promise.Input]: + """ + :rtype: list[flytekit.common.promise.Input] + """ + return self._user_inputs + + def create_launch_plan( + self, + default_inputs: Dict[str, _promise.Input] = None, + fixed_inputs: Dict[str, Any] = None, + schedule: _schedule_models.Schedule = None, + role: str = None, + notifications: List[_common_models.Notification] = None, + labels: _common_models.Labels = None, + annotations: _common_models.Annotations = None, + assumable_iam_role: str = None, + kubernetes_service_account: str = None, + raw_output_data_prefix: str = None, + ): + """ + This method will create a launch plan object that can execute this workflow. + :param dict[Text,flytekit.common.promise.Input] default_inputs: + :param dict[Text,T] fixed_inputs: + :param flytekit.models.schedule.Schedule schedule: A schedule on which to execute this launch plan. + :param Text role: Deprecated. Use assumable_iam_role instead. + :param list[flytekit.models.common.Notification] notifications: A list of notifications to enact by default for + this launch plan. + :param flytekit.models.common.Labels labels: + :param flytekit.models.common.Annotations annotations: + :param cls: This parameter can be used by users to define an extension of a launch plan to instantiate. The + class provided should be a subclass of flytekit.common.launch_plan.SdkLaunchPlan. + :param Text assumable_iam_role: The IAM role to execute the workflow with. + :param Text kubernetes_service_account: The kubernetes service account to execute the workflow with. + :param Text raw_output_data_prefix: Bucket for offloaded data + :rtype: flytekit.common.launch_plan.SdkRunnableLaunchPlan + """ + # TODO: Actually ensure the parameters conform. + if role and (assumable_iam_role or kubernetes_service_account): + raise ValueError("Cannot set both role and auth. Role is deprecated, use auth instead.") + fixed_inputs = fixed_inputs or {} + merged_default_inputs = {v.name: v for v in self.user_inputs if v.name not in fixed_inputs} + merged_default_inputs.update(default_inputs or {}) + + if role: + assumable_iam_role = role # For backwards compatibility + auth_role = _common_models.AuthRole( + assumable_iam_role=assumable_iam_role, kubernetes_service_account=kubernetes_service_account, + ) + + raw_output_config = _common_models.RawOutputDataConfig(raw_output_data_prefix or "") + + return _launch_plan.SdkRunnableLaunchPlan( + sdk_workflow=self, + default_inputs={ + k: user_input.rename_and_return_reference(k) for k, user_input in _six.iteritems(merged_default_inputs) + }, + fixed_inputs=fixed_inputs, + schedule=schedule, + notifications=notifications, + labels=labels, + annotations=annotations, + auth_role=auth_role, + raw_output_data_config=raw_output_config, + ) + + +def build_sdk_workflow_from_metaclass(metaclass, on_failure=None, disable_default_launch_plan=False, cls=None): + """ + :param T metaclass: This is the user-defined workflow class, prior to decoration. + :param on_failure flytekit.models.core.workflow.WorkflowMetadata.OnFailurePolicy: [Optional] The execution policy + when the workflow detects a failure. + :param bool disable_default_launch_plan: Determines whether to create a default launch plan for the workflow or not. + :param cls: This is the class that will be instantiated from the inputs, outputs, and nodes. This will be used + by users extending the base Flyte programming model. If set, it must be a subclass of PythonWorkflow. + + :rtype: SdkRunnableWorkflow + """ + inputs, outputs, nodes = _discover_workflow_components(metaclass) + metadata = _workflow_models.WorkflowMetadata(on_failure=on_failure if on_failure else None) + + return (cls or SdkRunnableWorkflow).construct_from_class_definition( + inputs=[i for i in sorted(inputs, key=lambda x: x.name)], + outputs=[o for o in sorted(outputs, key=lambda x: x.name)], + nodes=[n for n in sorted(nodes, key=lambda x: x.id)], + metadata=metadata, + disable_default_launch_plan=disable_default_launch_plan, + ) + + +def _discover_workflow_components(workflow_class): + """ + This task iterates over the attributes of a user-defined class in order to return a list of inputs, outputs and + nodes. + :param class workflow_class: User-defined class with task instances as attributes. + :rtype: (list[flytekit.common.promise.Input], list[Output], list[flytekit.common.nodes.SdkNode]) + """ + + inputs = [] + outputs = [] + nodes = [] + + to_visit_objs = _queue.Queue() + top_level_attributes = set() + for attribute_name in dir(workflow_class): + to_visit_objs.put((attribute_name, getattr(workflow_class, attribute_name))) + top_level_attributes.add(attribute_name) + + # For all task instances defined within the workflow, bind them to this specific workflow and hook-up to the + # engine (when available) + visited_obj_ids = set() + while not to_visit_objs.empty(): + attribute_name, current_obj = to_visit_objs.get() + + current_obj_id = id(current_obj) + if current_obj_id in visited_obj_ids: + continue + visited_obj_ids.add(current_obj_id) + + if isinstance(current_obj, _nodes.SdkNode): + # TODO: If an attribute name is on the form node_name[index], the resulting + # node name might not be correct. + nodes.append(current_obj.assign_id_and_return(attribute_name)) + elif isinstance(current_obj, _promise.Input): + if attribute_name is None or attribute_name not in top_level_attributes: + raise _user_exceptions.FlyteValueException( + attribute_name, "Detected workflow input specified outside of top level.", + ) + inputs.append(current_obj.rename_and_return_reference(attribute_name)) + elif isinstance(current_obj, Output): + if attribute_name is None or attribute_name not in top_level_attributes: + raise _user_exceptions.FlyteValueException( + attribute_name, "Detected workflow output specified outside of top level.", + ) + outputs.append(current_obj.rename_and_return_reference(attribute_name)) + elif isinstance(current_obj, list) or isinstance(current_obj, set) or isinstance(current_obj, tuple): + for idx, value in enumerate(current_obj): + to_visit_objs.put((_assign_indexed_attribute_name(attribute_name, idx), value)) + elif isinstance(current_obj, dict): + # Visit dictionary keys. + for key in current_obj.keys(): + to_visit_objs.put((_assign_indexed_attribute_name(attribute_name, key), key)) + # Visit dictionary values. + for key, value in _six.iteritems(current_obj): + to_visit_objs.put((_assign_indexed_attribute_name(attribute_name, key), value)) + return inputs, outputs, nodes + + +def _assign_indexed_attribute_name(attribute_name, index): + return "{}[{}]".format(attribute_name, index) diff --git a/flytekit/common/mixins/artifact.py b/flytekit/common/mixins/artifact.py index 89bde8f34a..41e5ebb0b7 100644 --- a/flytekit/common/mixins/artifact.py +++ b/flytekit/common/mixins/artifact.py @@ -1,16 +1,12 @@ -from __future__ import absolute_import - import abc as _abc import datetime as _datetime import time as _time -import six as _six - from flytekit.common.exceptions import user as _user_exceptions from flytekit.models import common as _common_models -class ExecutionArtifact(_six.with_metaclass(_common_models.FlyteABCMeta, object)): +class ExecutionArtifact(object, metaclass=_common_models.FlyteABCMeta): @_abc.abstractproperty def inputs(self): """ diff --git a/flytekit/common/mixins/hash.py b/flytekit/common/mixins/hash.py index f7a062a574..8b2559f696 100644 --- a/flytekit/common/mixins/hash.py +++ b/flytekit/common/mixins/hash.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import - - class HashOnReferenceMixin(object): def __hash__(self): return hash(id(self)) diff --git a/flytekit/common/mixins/launchable.py b/flytekit/common/mixins/launchable.py index 97f8d48455..4a3522f9d6 100644 --- a/flytekit/common/mixins/launchable.py +++ b/flytekit/common/mixins/launchable.py @@ -1,12 +1,9 @@ -from __future__ import absolute_import - import abc as _abc -import six as _six from deprecated import deprecated as _deprecated -class LaunchableEntity(_six.with_metaclass(_abc.ABCMeta, object)): +class LaunchableEntity(object, metaclass=_abc.ABCMeta): def launch( self, project, diff --git a/flytekit/common/mixins/registerable.py b/flytekit/common/mixins/registerable.py index 9eed525ba1..8f8c4005c4 100644 --- a/flytekit/common/mixins/registerable.py +++ b/flytekit/common/mixins/registerable.py @@ -1,11 +1,8 @@ -from __future__ import absolute_import - import abc as _abc import importlib as _importlib import inspect as _inspect import logging as _logging - -import six as _six +from typing import Set from flytekit.common import sdk_bases as _sdk_bases from flytekit.common import utils as _utils @@ -39,36 +36,9 @@ def __call__(cls, *args, **kwargs): return o -class RegisterableEntity(_six.with_metaclass(_InstanceTracker, object)): - def __init__(self, *args, **kwargs): - self._platform_valid_name = None - super(RegisterableEntity, self).__init__(*args, **kwargs) - - @_abc.abstractmethod - def register(self, project, domain, name, version): - """ - :param Text project: The project in which to register this task. - :param Text domain: The domain in which to register this task. - :param Text name: The name to give this task. - :param Text version: The version in which to register this task. - """ - pass - +class FlyteEntity(object, metaclass=_sdk_bases.ExtendedSdkType): + @property @_abc.abstractmethod - def serialize(self, project, domain, name, version): - """ - Registerable entities also are required to be serialized. This allows flytekit to separate serialization from - the network call to Admin (mostly at least, if a Launch Plan is fetched for instance as part of another - workflow, it will still hit Admin. - - :param Text project: The project in which to serialize this task. - :param Text domain: The domain in which to serialize this task. - :param Text name: The name to give this task. - :param Text version: The version in which to serialize this task. - """ - pass - - @_abc.abstractproperty def resource_type(self): """ Integer from _identifier.ResourceType enum @@ -76,20 +46,20 @@ def resource_type(self): """ pass - @_abc.abstractproperty + @property + @_abc.abstractmethod def entity_type_text(self): """ + TODO: Rename to resource type text :rtype: Text """ pass - @property - def upstream_entities(self): - """ - Task, workflow, and launch plan that need to be registered in advance of this workflow. - :rtype: set[RegisterableEntity] - """ - return self._upstream_entities + +class TrackableEntity(FlyteEntity, metaclass=_InstanceTracker): + def __init__(self, *args, **kwargs): + self._platform_valid_name = None + super(TrackableEntity, self).__init__(*args, **kwargs) @property def instantiated_in(self): @@ -97,7 +67,7 @@ def instantiated_in(self): If found, we try to specify the module where the task was first instantiated. :rtype: Optional[Text] """ - return self._instantiated_in + return self._instantiated_in # Set in metaclass @property def has_valid_name(self): @@ -147,7 +117,7 @@ def some_task() When Flytekit calls the module loader and loads the task, the name of the task is the name of the function itself. It's known at time of creation. In contrast, when - xyz = SomeWorflow.create_launch_plan() + xyz = SomeWorkflow.create_launch_plan() is called, the name of the launch plan isn't known until after creation, it's not "SomeWorkflow", it's "xyz" """ @@ -156,7 +126,7 @@ def some_task() for k in dir(m): try: - if getattr(m, k) == self: + if getattr(m, k) is self: self._platform_valid_name = _utils.fqdn(m.__name__, k, entity_type=self.resource_type) _logging.debug("Auto-assigning name to {}".format(self._platform_valid_name)) return @@ -171,3 +141,52 @@ def some_task() _logging.error("Could not auto-assign name") raise _system_exceptions.FlyteSystemException("Error looking for object while auto-assigning name.") + + +class RegisterableEntity(TrackableEntity): + def __init__(self, *args, **kwargs): + self._has_registered = False + super(RegisterableEntity, self).__init__(*args, **kwargs) + + @_abc.abstractmethod + def register(self, project, domain, name, version): + """ + :param Text project: The project in which to register this task. + :param Text domain: The domain in which to register this task. + :param Text name: The name to give this task. + :param Text version: The version in which to register this task. + """ + pass + + @_abc.abstractmethod + def serialize(self): + """ + Registerable entities also are required to be serialized. This allows flytekit to separate serialization from + the network call to Admin (mostly at least, if a Launch Plan is fetched for instance as part of another + workflow, it will still hit Admin). + """ + pass + + @property + def has_registered(self) -> bool: + return self._has_registered + + +class HasDependencies(object): + """ + This interface is meant to describe Flyte entities that can have upstream dependencies. For instance, currently a + launch plan depends on the underlying workflow, and a workflow is dependent on its tasks, and other launch plans, + and subworkflows. + """ + + def __init__(self, *args, **kwargs): + self._upstream_entities = set() + super(HasDependencies, self).__init__(*args, **kwargs) + + @property + def upstream_entities(self) -> Set[RegisterableEntity]: + """ + Task, workflow, and launch plan that need to be registered in advance of this workflow. + :rtype: set[RegisterableEntity] + """ + return self._upstream_entities diff --git a/flytekit/common/nodes.py b/flytekit/common/nodes.py index 419455bc61..7a79cee8e6 100644 --- a/flytekit/common/nodes.py +++ b/flytekit/common/nodes.py @@ -1,15 +1,17 @@ -from __future__ import absolute_import - import abc as _abc import logging as _logging +import os as _os import six as _six +from flyteidl.core import literals_pb2 as _literals_pb2 from sortedcontainers import SortedDict as _SortedDict +from flytekit.clients.helpers import iterate_task_executions as _iterate_task_executions from flytekit.common import component_nodes as _component_nodes from flytekit.common import constants as _constants from flytekit.common import promise as _promise from flytekit.common import sdk_bases as _sdk_bases +from flytekit.common import utils as _common_utils from flytekit.common.exceptions import scopes as _exception_scopes from flytekit.common.exceptions import system as _system_exceptions from flytekit.common.exceptions import user as _user_exceptions @@ -18,14 +20,16 @@ from flytekit.common.tasks import executions as _task_executions from flytekit.common.types import helpers as _type_helpers from flytekit.common.utils import _dnsify -from flytekit.engines import loader as _engine_loader +from flytekit.engines.flyte import engine as _flyte_engine +from flytekit.interfaces.data import data_proxy as _data_proxy from flytekit.models import common as _common_models +from flytekit.models import literals as _literal_models from flytekit.models import node_execution as _node_execution_models from flytekit.models.core import execution as _execution_models from flytekit.models.core import workflow as _workflow_model -class ParameterMapper(_six.with_metaclass(_common_models.FlyteABCMeta, _SortedDict)): +class ParameterMapper(_SortedDict, metaclass=_common_models.FlyteABCMeta): """ This abstract class provides functionality to reference specific inputs and outputs for a task instance. This allows for syntax such as: @@ -106,7 +110,7 @@ def _return_mapping_object(self, sdk_node, sdk_type, name): return _promise.NodeOutput(sdk_node, sdk_type, name) -class SdkNode(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _hash_mixin.HashOnReferenceMixin, _workflow_model.Node,)): +class SdkNode(_hash_mixin.HashOnReferenceMixin, _workflow_model.Node, metaclass=_sdk_bases.ExtendedSdkType): def __init__( self, id, @@ -149,6 +153,8 @@ def __init__( elif sdk_launch_plan is not None: workflow_node = _component_nodes.SdkWorkflowNode(sdk_launch_plan=sdk_launch_plan) + # TODO: this calls the constructor which means it will set all the upstream node ids to None if at the time of + # this instantiation, the upstream nodes have not had their nodes assigned yet. super(SdkNode, self).__init__( id=_dnsify(id) if id else None, metadata=metadata, @@ -300,9 +306,7 @@ def __repr__(self): class SdkNodeExecution( - _six.with_metaclass( - _sdk_bases.ExtendedSdkType, _node_execution_models.NodeExecution, _artifact_mixin.ExecutionArtifact, - ) + _node_execution_models.NodeExecution, _artifact_mixin.ExecutionArtifact, metaclass=_sdk_bases.ExtendedSdkType ): def __init__(self, *args, **kwargs): super(SdkNodeExecution, self).__init__(*args, **kwargs) @@ -342,9 +346,23 @@ def inputs(self): :rtype: dict[Text, T] """ if self._inputs is None: - self._inputs = _type_helpers.unpack_literal_map_to_sdk_python_std( - _engine_loader.get_engine().get_node_execution(self).get_inputs() - ) + client = _flyte_engine.get_client() + execution_data = client.get_node_execution_data(self.id) + + # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. + if bool(execution_data.full_inputs.literals): + input_map = execution_data.full_inputs + elif execution_data.inputs.bytes > 0: + with _common_utils.AutoDeletingTempDir() as t: + tmp_name = _os.path.join(t.name, "inputs.pb") + _data_proxy.Data.get_data(execution_data.inputs.url, tmp_name) + input_map = _literal_models.LiteralMap.from_flyte_idl( + _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name) + ) + else: + input_map = _literal_models.LiteralMap({}) + + self._inputs = _type_helpers.unpack_literal_map_to_sdk_python_std(input_map) return self._inputs @property @@ -356,15 +374,30 @@ def outputs(self): """ if not self.is_complete: raise _user_exceptions.FlyteAssertion( - "Please what until the node execution has completed before " "requesting the outputs." + "Please what until the node execution has completed before requesting the outputs." ) if self.error: raise _user_exceptions.FlyteAssertion("Outputs could not be found because the execution ended in failure.") if self._outputs is None: - self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std( - _engine_loader.get_engine().get_node_execution(self).get_outputs() - ) + client = _flyte_engine.get_client() + execution_data = client.get_node_execution_data(self.id) + + # Outputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. + if bool(execution_data.full_outputs.literals): + output_map = execution_data.full_outputs + + elif execution_data.outputs.bytes > 0: + with _common_utils.AutoDeletingTempDir() as t: + tmp_name = _os.path.join(t.name, "outputs.pb") + _data_proxy.Data.get_data(execution_data.outputs.url, tmp_name) + output_map = _literal_models.LiteralMap.from_flyte_idl( + _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name) + ) + else: + output_map = _literal_models.LiteralMap({}) + + self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std(output_map) return self._outputs @property @@ -376,7 +409,7 @@ def error(self): """ if not self.is_complete: raise _user_exceptions.FlyteAssertion( - "Please what until the node execution has completed before " "requesting error information." + "Please what until the node execution has completed before requesting error information." ) return self.closure.error @@ -408,11 +441,10 @@ def sync(self): :rtype: None """ if not self.is_complete or self.task_executions is not None: - ne = _engine_loader.get_engine().get_node_execution(self) - ne.sync() - self._task_executions = [ - _task_executions.SdkTaskExecution.promote_from_model(te) for te in ne.get_task_executions() - ] + client = _flyte_engine.get_client() + self._closure = client.get_node_execution(self.id).closure + task_executions = list(_iterate_task_executions(client, self.id)) + self._task_executions = [_task_executions.SdkTaskExecution.promote_from_model(te) for te in task_executions] # TODO: Sub-workflows too once implemented def _sync_closure(self): @@ -420,5 +452,5 @@ def _sync_closure(self): Syncs the closure of the underlying execution artifact with the state observed by the platform. :rtype: None """ - ne = _engine_loader.get_engine().get_node_execution(self) - ne.sync() + client = _flyte_engine.get_client() + self._closure = client.get_node_execution(self.id).closure diff --git a/flytekit/common/notifications.py b/flytekit/common/notifications.py index a9b5fbfc44..09b1d11358 100644 --- a/flytekit/common/notifications.py +++ b/flytekit/common/notifications.py @@ -1,14 +1,10 @@ -from __future__ import absolute_import - -import six as _six - from flytekit.common import sdk_bases as _sdk_bases from flytekit.common.exceptions import user as _user_exceptions from flytekit.models import common as _common_model from flytekit.models.core import execution as _execution_model -class Notification(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _common_model.Notification)): +class Notification(_common_model.Notification, metaclass=_sdk_bases.ExtendedSdkType): VALID_PHASES = { _execution_model.WorkflowExecutionPhase.ABORTED, diff --git a/flytekit/common/promise.py b/flytekit/common/promise.py index 9c31b0534c..2606bd87a4 100644 --- a/flytekit/common/promise.py +++ b/flytekit/common/promise.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import - -import six as _six - from flytekit.common import constants as _constants from flytekit.common import sdk_bases as _sdk_bases from flytekit.common.exceptions import user as _user_exceptions @@ -10,7 +6,7 @@ from flytekit.models import types as _type_models -class Input(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _interface_models.Parameter)): +class Input(_interface_models.Parameter, metaclass=_sdk_bases.ExtendedSdkType): def __init__(self, name, sdk_type, help=None, **kwargs): """ :param Text name: @@ -107,7 +103,7 @@ def __repr__(self): def promote_from_model(cls, model): """ :param flytekit.models.interface.Parameter model: - :rtype: Parameter + :rtype: Input """ sdk_type = _type_helpers.get_sdk_type_from_literal_type(model.var.type) @@ -118,7 +114,7 @@ def promote_from_model(cls, model): return cls("", sdk_type, help=model.var.description, required=True) -class NodeOutput(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _type_models.OutputReference)): +class NodeOutput(_type_models.OutputReference, metaclass=_sdk_bases.ExtendedSdkType): def __init__(self, sdk_node, sdk_type, var): """ :param flytekit.common.nodes.SdkNode sdk_node: @@ -163,4 +159,5 @@ def sdk_type(self): return self._type def __repr__(self): + # TODO: fix this so that if upstream node ids have any None's in it, this still prints instead of erroring. return "NodeOutput({}:{})".format(self.sdk_node, self.var) diff --git a/flytekit/common/schedules.py b/flytekit/common/schedules.py index 76afcab18b..82af7deacc 100644 --- a/flytekit/common/schedules.py +++ b/flytekit/common/schedules.py @@ -1,9 +1,6 @@ -from __future__ import absolute_import, division - import datetime as _datetime import croniter as _croniter -import six as _six from flytekit.common import sdk_bases as _sdk_bases from flytekit.common.exceptions import user as _user_exceptions @@ -20,7 +17,7 @@ def from_flyte_idl(cls, proto): return cls.promote_from_model(_schedule_models.Schedule.from_flyte_idl(proto)) -class CronSchedule(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _ExtendedSchedule)): +class CronSchedule(_ExtendedSchedule, metaclass=_sdk_bases.ExtendedSdkType): def __init__(self, cron_expression, kickoff_time_input_arg=None): """ :param Text cron_expression: @@ -74,7 +71,7 @@ def promote_from_model(cls, base_model): return cls(base_model.cron_expression, kickoff_time_input_arg=base_model.kickoff_time_input_arg,) -class FixedRate(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _ExtendedSchedule)): +class FixedRate(_ExtendedSchedule, metaclass=_sdk_bases.ExtendedSdkType): def __init__(self, duration, kickoff_time_input_arg=None): """ :param datetime.timedelta duration: diff --git a/flytekit/common/sdk_bases.py b/flytekit/common/sdk_bases.py index 743ee072ff..d082302343 100644 --- a/flytekit/common/sdk_bases.py +++ b/flytekit/common/sdk_bases.py @@ -1,13 +1,9 @@ -from __future__ import absolute_import - import abc as _abc -import six as _six - from flytekit.models import common as _common -class ExtendedSdkType(_six.with_metaclass(_common.FlyteABCMeta, _common.FlyteType)): +class ExtendedSdkType(_common.FlyteType, metaclass=_common.FlyteABCMeta): """ Abstract class that all SDK objects must inherit from. This provides the ability to promote a data model object into an actionable object. diff --git a/flytekit/common/tasks/executions.py b/flytekit/common/tasks/executions.py index 9b380ada89..d87c558c09 100644 --- a/flytekit/common/tasks/executions.py +++ b/flytekit/common/tasks/executions.py @@ -1,20 +1,23 @@ -from __future__ import absolute_import +import os as _os import six as _six +from flyteidl.core import literals_pb2 as _literals_pb2 +from flytekit.clients.helpers import iterate_node_executions as _iterate_node_executions from flytekit.common import sdk_bases as _sdk_bases +from flytekit.common import utils as _common_utils from flytekit.common.exceptions import user as _user_exceptions from flytekit.common.mixins import artifact as _artifact_mixin from flytekit.common.types import helpers as _type_helpers -from flytekit.engines import loader as _engine_loader +from flytekit.engines.flyte import engine as _flyte_engine +from flytekit.interfaces.data import data_proxy as _data_proxy +from flytekit.models import literals as _literal_models from flytekit.models.admin import task_execution as _task_execution_model from flytekit.models.core import execution as _execution_models class SdkTaskExecution( - _six.with_metaclass( - _sdk_bases.ExtendedSdkType, _task_execution_model.TaskExecution, _artifact_mixin.ExecutionArtifact, - ) + _task_execution_model.TaskExecution, _artifact_mixin.ExecutionArtifact, metaclass=_sdk_bases.ExtendedSdkType ): def __init__(self, *args, **kwargs): super(SdkTaskExecution, self).__init__(*args, **kwargs) @@ -41,9 +44,23 @@ def inputs(self): :rtype: dict[Text, T] """ if self._inputs is None: - self._inputs = _type_helpers.unpack_literal_map_to_sdk_python_std( - _engine_loader.get_engine().get_task_execution(self).get_inputs() - ) + client = _flyte_engine.get_client() + execution_data = client.get_task_execution_data(self.id) + + # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. + if bool(execution_data.full_inputs.literals): + input_map = execution_data.full_inputs + elif execution_data.inputs.bytes > 0: + with _common_utils.AutoDeletingTempDir() as t: + tmp_name = _os.path.join(t.name, "inputs.pb") + _data_proxy.Data.get_data(execution_data.inputs.url, tmp_name) + input_map = _literal_models.LiteralMap.from_flyte_idl( + _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name) + ) + else: + input_map = _literal_models.LiteralMap({}) + + self._inputs = _type_helpers.unpack_literal_map_to_sdk_python_std(input_map) return self._inputs @property @@ -56,15 +73,29 @@ def outputs(self): """ if not self.is_complete: raise _user_exceptions.FlyteAssertion( - "Please what until the task execution has completed before " "requesting the outputs." + "Please what until the task execution has completed before requesting the outputs." ) if self.error: raise _user_exceptions.FlyteAssertion("Outputs could not be found because the execution ended in failure.") if self._outputs is None: - self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std( - _engine_loader.get_engine().get_task_execution(self).get_outputs() - ) + client = _flyte_engine.get_client() + execution_data = client.get_task_execution_data(self.id) + + # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. + if bool(execution_data.full_outputs.literals): + output_map = execution_data.full_outputs + + elif execution_data.outputs.bytes > 0: + with _common_utils.AutoDeletingTempDir() as t: + tmp_name = _os.path.join(t.name, "outputs.pb") + _data_proxy.Data.get_data(execution_data.outputs.url, tmp_name) + output_map = _literal_models.LiteralMap.from_flyte_idl( + _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name) + ) + else: + output_map = _literal_models.LiteralMap({}) + self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std(output_map) return self._outputs @property @@ -76,7 +107,7 @@ def error(self): """ if not self.is_complete: raise _user_exceptions.FlyteAssertion( - "Please what until the task execution has completed before " "requesting error information." + "Please what until the task execution has completed before requesting error information." ) return self.closure.error @@ -89,7 +120,12 @@ def get_child_executions(self, filters=None): if not self.is_parent: raise _user_exceptions.FlyteAssertion("Only task executions marked with 'is_parent' have child executions.") - models = _engine_loader.get_engine().get_task_execution(self).get_child_executions(filters=filters) + client = _flyte_engine.get_client() + models = { + v.id.node_id: v + for v in _iterate_node_executions(client, task_execution_identifier=self.id, filters=filters) + } + return {k: _nodes.SdkNodeExecution.promote_from_model(v) for k, v in _six.iteritems(models)} @classmethod @@ -113,4 +149,5 @@ def _sync_closure(self): Syncs the closure of the underlying execution artifact with the state observed by the platform. :rtype: None """ - _engine_loader.get_engine().get_task_execution(self).sync() + client = _flyte_engine.get_client() + self._closure = client.get_task_execution(self.id).closure diff --git a/flytekit/common/tasks/generic_spark_task.py b/flytekit/common/tasks/generic_spark_task.py index 03d00b00e6..3a7c0ef1bc 100644 --- a/flytekit/common/tasks/generic_spark_task.py +++ b/flytekit/common/tasks/generic_spark_task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import sys as _sys import six as _six diff --git a/flytekit/common/tasks/hive_task.py b/flytekit/common/tasks/hive_task.py index 42436dc61b..6023bf0ec3 100644 --- a/flytekit/common/tasks/hive_task.py +++ b/flytekit/common/tasks/hive_task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import uuid as _uuid import six as _six diff --git a/flytekit/common/tasks/output.py b/flytekit/common/tasks/output.py index caf64deff5..8ca1b307cb 100644 --- a/flytekit/common/tasks/output.py +++ b/flytekit/common/tasks/output.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.exceptions import scopes as _exception_scopes from flytekit.common.types import base_sdk_types as _base_sdk_types diff --git a/flytekit/common/tasks/presto_task.py b/flytekit/common/tasks/presto_task.py index 9612f551a9..1bf8d0e51c 100644 --- a/flytekit/common/tasks/presto_task.py +++ b/flytekit/common/tasks/presto_task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime import six as _six diff --git a/flytekit/common/tasks/pytorch_task.py b/flytekit/common/tasks/pytorch_task.py index d728db2895..17c2cd7ea1 100644 --- a/flytekit/common/tasks/pytorch_task.py +++ b/flytekit/common/tasks/pytorch_task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from google.protobuf.json_format import MessageToDict as _MessageToDict from flytekit.common.tasks import sdk_runnable as _sdk_runnable diff --git a/flytekit/common/tasks/raw_container.py b/flytekit/common/tasks/raw_container.py index fd185e6094..113ec3000f 100644 --- a/flytekit/common/tasks/raw_container.py +++ b/flytekit/common/tasks/raw_container.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime from typing import Dict, List diff --git a/flytekit/common/tasks/sdk_dynamic.py b/flytekit/common/tasks/sdk_dynamic.py index 5d3fe92e57..e1ffc6f861 100644 --- a/flytekit/common/tasks/sdk_dynamic.py +++ b/flytekit/common/tasks/sdk_dynamic.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import itertools as _itertools import math import os as _os @@ -97,10 +95,14 @@ def _can_run_as_array(task_type): @staticmethod def _add_upstream_entities(executable_sdk_object, sub_workflows, tasks): - for upstream_entity in executable_sdk_object.upstream_entities: + upstream_entities = [] + if isinstance(executable_sdk_object, _workflow.SdkWorkflow): + upstream_entities = [n.executable_sdk_object for n in executable_sdk_object.nodes] + + for upstream_entity in upstream_entities: # If the upstream entity is either a Workflow or a Task, yield them in the # dynamic job spec. Otherwise (e.g. a LaunchPlan), we will assume it already - # is registered (can't be dynamically created). Thi will cause a runtime error + # is registered (can't be dynamically created). This will cause a runtime error # if it's not already registered with the control plane. if isinstance(upstream_entity, _workflow.SdkWorkflow): sub_workflows.add(upstream_entity) @@ -165,7 +167,7 @@ def _produce_dynamic_job_spec(self, context, inputs): # If the executable object that we're dealing with is registerable (ie, SdkRunnableLaunchPlan, SdkWorkflow # SdkTask, or SdkRunnableTask), then it should have the ability to give itself a name. After assigning # itself the name, also make sure the id is properly set according to current config values. - if isinstance(executable, _registerable.RegisterableEntity): + if isinstance(executable, _registerable.TrackableEntity) and not executable.has_valid_name: executable.auto_assign_name() executable._id = _identifier.Identifier( executable.resource_type, diff --git a/flytekit/common/tasks/sdk_runnable.py b/flytekit/common/tasks/sdk_runnable.py index 08dd815fa5..85e0b95494 100644 --- a/flytekit/common/tasks/sdk_runnable.py +++ b/flytekit/common/tasks/sdk_runnable.py @@ -1,9 +1,4 @@ -from __future__ import absolute_import - -try: - from inspect import getfullargspec as _getargspec -except ImportError: - from inspect import getargspec as _getargspec +from inspect import getfullargspec as _getargspec import six as _six @@ -26,7 +21,6 @@ class ExecutionParameters(object): - """ This is the parameter object that will be provided as the first parameter for every execution of any @*_task decorated function. @@ -103,7 +97,11 @@ def execution_id(self): return self._execution_id -class SdkRunnableContainer(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _task_models.Container)): +class SdkRunnableContainer(_task_models.Container, metaclass=_sdk_bases.ExtendedSdkType): + """ + This is not necessarily a local-only Container object. So long as configuration is present, you can use this object + """ + def __init__( self, command, args, resources, env, config, ): @@ -142,8 +140,60 @@ def env(self): ) return env + @classmethod + def get_resources( + cls, + storage_request=None, + cpu_request=None, + gpu_request=None, + memory_request=None, + storage_limit=None, + cpu_limit=None, + gpu_limit=None, + memory_limit=None, + ): + """ + :param Text storage_request: + :param Text cpu_request: + :param Text gpu_request: + :param Text memory_request: + :param Text storage_limit: + :param Text cpu_limit: + :param Text gpu_limit: + :param Text memory_limit: + """ + requests = [] + if storage_request: + requests.append( + _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.STORAGE, storage_request) + ) + if cpu_request: + requests.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.CPU, cpu_request)) + if gpu_request: + requests.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.GPU, gpu_request)) + if memory_request: + requests.append( + _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.MEMORY, memory_request) + ) -class SdkRunnableTask(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _base_task.SdkTask)): + limits = [] + if storage_limit: + limits.append( + _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.STORAGE, storage_limit) + ) + if cpu_limit: + limits.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.CPU, cpu_limit)) + if gpu_limit: + limits.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.GPU, gpu_limit)) + if memory_limit: + limits.append( + _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.MEMORY, memory_limit) + ) + + return _task_models.Resources(limits=limits, requests=requests) + + +class SdkRunnableTask(_base_task.SdkTask, metaclass=_sdk_bases.ExtendedSdkType): """ This class includes the additional logic for building a task that executes in Python code. It has even more validation checks to ensure proper behavior than it's superclasses. @@ -194,7 +244,6 @@ def __init__( :param dict[Text, T] custom: """ self._task_function = task_function - super(SdkRunnableTask, self).__init__( task_type, _task_models.TaskMetadata( @@ -407,33 +456,9 @@ def _get_container_definition( memory_limit = memory_limit or _resource_config.DEFAULT_MEMORY_LIMIT.get() memory_request = memory_request or _resource_config.DEFAULT_MEMORY_REQUEST.get() - requests = [] - if storage_request: - requests.append( - _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.STORAGE, storage_request) - ) - if cpu_request: - requests.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.CPU, cpu_request)) - if gpu_request: - requests.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.GPU, gpu_request)) - if memory_request: - requests.append( - _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.MEMORY, memory_request) - ) - - limits = [] - if storage_limit: - limits.append( - _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.STORAGE, storage_limit) - ) - if cpu_limit: - limits.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.CPU, cpu_limit)) - if gpu_limit: - limits.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.GPU, gpu_limit)) - if memory_limit: - limits.append( - _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.MEMORY, memory_limit) - ) + resources = SdkRunnableContainer.get_resources( + storage_request, cpu_request, gpu_request, memory_request, storage_limit, cpu_limit, gpu_limit, memory_limit + ) return (cls or SdkRunnableContainer)( command=[], @@ -450,7 +475,7 @@ def _get_container_definition( "--raw-output-data-prefix", "{{.rawOutputDataPrefix}}", ], - resources=_task_models.Resources(limits=limits, requests=requests), + resources=resources, env=environment, config={}, ) diff --git a/flytekit/common/tasks/sidecar_task.py b/flytekit/common/tasks/sidecar_task.py index 1f3b9effe5..bf1f6019ec 100644 --- a/flytekit/common/tasks/sidecar_task.py +++ b/flytekit/common/tasks/sidecar_task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import six as _six from flyteidl.core import tasks_pb2 as _core_task from google.protobuf.json_format import MessageToDict as _MessageToDict diff --git a/flytekit/common/tasks/spark_task.py b/flytekit/common/tasks/spark_task.py index 950fe6cc25..afe79f43f5 100644 --- a/flytekit/common/tasks/spark_task.py +++ b/flytekit/common/tasks/spark_task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - try: from inspect import getfullargspec as _getargspec except ImportError: diff --git a/flytekit/common/tasks/task.py b/flytekit/common/tasks/task.py index db60803ee3..8ffac81958 100644 --- a/flytekit/common/tasks/task.py +++ b/flytekit/common/tasks/task.py @@ -1,7 +1,6 @@ -from __future__ import absolute_import - import hashlib as _hashlib import json as _json +import logging as _logging import uuid as _uuid import six as _six @@ -14,28 +13,29 @@ from flytekit.common import workflow_execution as _workflow_execution from flytekit.common.core import identifier as _identifier from flytekit.common.exceptions import scopes as _exception_scopes -from flytekit.common.exceptions import system as _system_exceptions from flytekit.common.exceptions import user as _user_exceptions from flytekit.common.mixins import hash as _hash_mixin from flytekit.common.mixins import launchable as _launchable_mixin from flytekit.common.mixins import registerable as _registerable from flytekit.common.types import helpers as _type_helpers +from flytekit.configuration import auth as _auth_config from flytekit.configuration import internal as _internal_config -from flytekit.engines import loader as _engine_loader +from flytekit.configuration import sdk as _sdk_config +from flytekit.engines.flyte import engine as _flyte_engine from flytekit.models import common as _common_model +from flytekit.models import execution as _admin_execution_models from flytekit.models import task as _task_model +from flytekit.models.admin import common as _admin_common from flytekit.models.core import identifier as _identifier_model from flytekit.models.core import workflow as _workflow_model class SdkTask( - _six.with_metaclass( - _sdk_bases.ExtendedSdkType, - _hash_mixin.HashOnReferenceMixin, - _task_model.TaskTemplate, - _registerable.RegisterableEntity, - _launchable_mixin.LaunchableEntity, - ) + _hash_mixin.HashOnReferenceMixin, + _registerable.RegisterableEntity, + _launchable_mixin.LaunchableEntity, + _task_model.TaskTemplate, + metaclass=_sdk_bases.ExtendedSdkType, ): def __init__(self, type, metadata, interface, custom, container=None): """ @@ -47,6 +47,7 @@ def __init__(self, type, metadata, interface, custom, container=None): :param Container container: Provides the necessary entrypoint information for execution. For instance, a Container might be specified with the necessary command line arguments. """ + # TODO: Remove the identifier portion and fill in with local values. super(SdkTask, self).__init__( _identifier.Identifier( _identifier_model.ResourceType.TASK, @@ -69,14 +70,6 @@ def interface(self): """ return super(SdkTask, self).interface - @property - def upstream_entities(self): - """ - Task, workflow, and launch plan that need to be registered in advance of this workflow. - :rtype: set[T] - """ - return set() - @property def resource_type(self): """ @@ -135,6 +128,9 @@ def __call__(self, *args, **input_map): bindings, upstream_nodes = self.interface.create_bindings_for_inputs(input_map) # TODO: Remove DEADBEEF + # One thing to note - this function is not overloaded at the SdkRunnableTask layer, which means 'self' here + # will sometimes refer to an object that can be executed locally, and other times will refer to something + # that cannot (ie a pure SdkTask object, fetched from Admin for instance). return _nodes.SdkNode( id=None, metadata=_workflow_model.NodeMetadata( @@ -153,13 +149,21 @@ def register(self, project, domain, name, version): :param Text name: The name to give this task. :param Text version: The version in which to register this task. """ + # TODO: Revisit the notion of supplying the project, domain, name, version, as opposed to relying on the + # current ID. self.validate() id_to_register = _identifier.Identifier(_identifier_model.ResourceType.TASK, project, domain, name, version) old_id = self.id + + client = _flyte_engine.get_client() try: self._id = id_to_register - _engine_loader.get_engine().get_task(self).register(id_to_register) - return _six.text_type(self.id) + client.create_task(id_to_register, _task_model.TaskSpec(self)) + self._id = old_id + self._has_registered = True + return str(id_to_register) + except _user_exceptions.FlyteEntityAlreadyExistsException: + pass except Exception: self._id = old_id raise @@ -183,9 +187,11 @@ def fetch(cls, project, domain, name, version): :rtype: SdkTask """ task_id = _identifier.Identifier(_identifier_model.ResourceType.TASK, project, domain, name, version) - admin_task = _engine_loader.get_engine().fetch_task(task_id=task_id) + admin_task = _flyte_engine.get_client().get_task(task_id) + sdk_task = cls.promote_from_model(admin_task.closure.compiled_task.template) sdk_task._id = task_id + sdk_task._has_registered = True return sdk_task @classmethod @@ -199,7 +205,12 @@ def fetch_latest(cls, project, domain, name): :rtype: SdkTask """ named_task = _common_model.NamedEntityIdentifier(project, domain, name) - admin_task = _engine_loader.get_engine().fetch_latest_task(named_task) + client = _flyte_engine.get_client() + task_list, _ = client.list_tasks_paginated( + named_task, limit=1, sort_by=_admin_common.Sort("created_at", _admin_common.Sort.Direction.DESCENDING), + ) + admin_task = task_list[0] if task_list else None + if not admin_task: raise _user_exceptions.FlyteEntityNotExistException("Named task {} not found".format(named_task)) sdk_task = cls.promote_from_model(admin_task.closure.compiled_task.template) @@ -303,7 +314,7 @@ def _produce_deterministic_version(self, version=None): return _hashlib.md5(str(task_body).encode("utf-8")).hexdigest() @_exception_scopes.system_entry_point - def register_and_launch(self, project, domain, name=None, version=None, inputs=None): + def register_and_launch(self, project, domain, name, version=None, inputs=None): """ :param Text project: The project in which to register and launch this task. :param Text domain: The domain in which to register and launch this task. @@ -316,23 +327,7 @@ def register_and_launch(self, project, domain, name=None, version=None, inputs=N """ self.validate() version = self._produce_deterministic_version(version) - - if name is None: - try: - self.auto_assign_name() - generated_name = self._platform_valid_name - except _system_exceptions.FlyteSystemException: - # If we're not able to assign a platform valid name, use the deterministically-produced version instead. - generated_name = version - name = name if name else generated_name - id_to_register = _identifier.Identifier(_identifier_model.ResourceType.TASK, project, domain, name, version) - old_id = self.id - try: - self._id = id_to_register - _engine_loader.get_engine().get_task(self).register(id_to_register) - except Exception: - self._id = old_id - raise + self.register(project, domain, name, version) return self.launch(project, domain, inputs=inputs) @_exception_scopes.system_entry_point @@ -360,17 +355,48 @@ def launch_with_literals( :param flytekit.models.common.Annotations annotation_overrides: :rtype: flytekit.common.workflow_execution.SdkWorkflowExecution """ - execution = ( - _engine_loader.get_engine() - .get_task(self) - .launch( + disable_all = notification_overrides == [] + if disable_all: + notification_overrides = None + else: + notification_overrides = _admin_execution_models.NotificationList(notification_overrides or []) + disable_all = None + + assumable_iam_role = _auth_config.ASSUMABLE_IAM_ROLE.get() + kubernetes_service_account = _auth_config.KUBERNETES_SERVICE_ACCOUNT.get() + + if not (assumable_iam_role or kubernetes_service_account): + _logging.warning( + "Using deprecated `role` from config. " "Please update your config to use `assumable_iam_role` instead" + ) + assumable_iam_role = _sdk_config.ROLE.get() + auth_role = _common_model.AuthRole( + assumable_iam_role=assumable_iam_role, kubernetes_service_account=kubernetes_service_account, + ) + + client = _flyte_engine.get_client() + try: + # TODO(katrogan): Add handling to register the underlying task if it's not already. + exec_id = client.create_execution( project, domain, - name=name, - inputs=literal_inputs, - notification_overrides=notification_overrides, - label_overrides=label_overrides, - annotation_overrides=annotation_overrides, + name, + _admin_execution_models.ExecutionSpec( + self.id, + _admin_execution_models.ExecutionMetadata( + _admin_execution_models.ExecutionMetadata.ExecutionMode.MANUAL, + "sdk", # TODO: get principle + 0, # TODO: Detect nesting + ), + notifications=notification_overrides, + disable_all=disable_all, + labels=label_overrides, + annotations=annotation_overrides, + auth_role=auth_role, + ), + literal_inputs, ) - ) + except _user_exceptions.FlyteEntityAlreadyExistsException: + exec_id = _identifier.WorkflowExecutionIdentifier(project, domain, name) + execution = client.get_execution(exec_id) return _workflow_execution.SdkWorkflowExecution.promote_from_model(execution) diff --git a/flytekit/common/types/base_sdk_types.py b/flytekit/common/types/base_sdk_types.py index ec77a28424..7ac659c3a6 100644 --- a/flytekit/common/types/base_sdk_types.py +++ b/flytekit/common/types/base_sdk_types.py @@ -1,16 +1,12 @@ -from __future__ import absolute_import - import abc as _abc -import six as _six - from flytekit.common import sdk_bases as _sdk_bases from flytekit.common.exceptions import user as _user_exceptions from flytekit.models import common as _common_models from flytekit.models import literals as _literal_models -class FlyteSdkType(_six.with_metaclass(_common_models.FlyteABCMeta, _sdk_bases.ExtendedSdkType)): +class FlyteSdkType(_sdk_bases.ExtendedSdkType, metaclass=_common_models.FlyteABCMeta): @_abc.abstractmethod def is_castable_from(cls, other): """ @@ -56,7 +52,7 @@ def __hash__(cls): return hash(cls.to_flyte_literal_type()) -class FlyteSdkValue(_six.with_metaclass(FlyteSdkType, _literal_models.Literal)): +class FlyteSdkValue(_literal_models.Literal, metaclass=FlyteSdkType): @classmethod def from_flyte_idl(cls, pb2_object): """ @@ -73,7 +69,7 @@ def to_python_std(self): pass -class InstantiableType(_six.with_metaclass(_common_models.FlyteABCMeta, FlyteSdkType)): +class InstantiableType(FlyteSdkType, metaclass=_common_models.FlyteABCMeta): @_abc.abstractmethod def __call__(cls, *args, **kwargs): """ diff --git a/flytekit/common/types/blobs.py b/flytekit/common/types/blobs.py index ce7a7498bb..206fbf198f 100644 --- a/flytekit/common/types/blobs.py +++ b/flytekit/common/types/blobs.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import - -import six as _six - from flytekit.common.exceptions import user as _user_exceptions from flytekit.common.types import base_sdk_types as _base_sdk_types from flytekit.common.types.impl import blobs as _blob_impl @@ -45,7 +41,7 @@ def __call__(cls, *args, **kwargs): # TODO: Make blobs and schemas pluggable -class Blob(_six.with_metaclass(BlobInstantiator, _base_sdk_types.FlyteSdkValue)): +class Blob(_base_sdk_types.FlyteSdkValue, metaclass=BlobInstantiator): @classmethod def from_string(cls, string_value): """ @@ -163,7 +159,7 @@ def __call__(cls, *args, **kwargs): return super(MultiPartBlobInstantiator, cls).__call__(*args, **kwargs) -class MultiPartBlob(_six.with_metaclass(MultiPartBlobInstantiator, _base_sdk_types.FlyteSdkValue)): +class MultiPartBlob(_base_sdk_types.FlyteSdkValue, metaclass=MultiPartBlobInstantiator): @classmethod def from_string(cls, string_value): """ @@ -283,7 +279,7 @@ def __call__(cls, *args, **kwargs): return super(CsvInstantiator, cls).__call__(*args, **kwargs) -class CSV(_six.with_metaclass(CsvInstantiator, Blob)): +class CSV(Blob, metaclass=CsvInstantiator): @classmethod def from_string(cls, string_value): """ @@ -386,7 +382,7 @@ def __call__(cls, *args, **kwargs): return super(MultiPartCsvInstantiator, cls).__call__(*args, **kwargs) -class MultiPartCSV(_six.with_metaclass(MultiPartCsvInstantiator, MultiPartBlob)): +class MultiPartCSV(MultiPartBlob, metaclass=MultiPartCsvInstantiator): @classmethod def from_string(cls, string_value): """ diff --git a/flytekit/common/types/containers.py b/flytekit/common/types/containers.py index 46fe545aac..f61a882bd0 100644 --- a/flytekit/common/types/containers.py +++ b/flytekit/common/types/containers.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import json as _json import six as _six @@ -43,12 +41,12 @@ class TList(TypedListImpl): return TList -class ListImpl(_six.with_metaclass(CollectionType, _base_sdk_types.FlyteSdkValue)): +class ListImpl(_base_sdk_types.FlyteSdkValue, metaclass=CollectionType): def __len__(self): return len(self.collection.literals) -class TypedListImpl(_six.with_metaclass(TypedCollectionType, ListImpl)): +class TypedListImpl(ListImpl, metaclass=TypedCollectionType): @classmethod def from_string(cls, string_value): """ diff --git a/flytekit/common/types/helpers.py b/flytekit/common/types/helpers.py index a5a52f56b6..2f6d4089df 100644 --- a/flytekit/common/types/helpers.py +++ b/flytekit/common/types/helpers.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import importlib as _importlib import six as _six diff --git a/flytekit/common/types/impl/blobs.py b/flytekit/common/types/impl/blobs.py index 709a92e4f7..f69d4215b8 100644 --- a/flytekit/common/types/impl/blobs.py +++ b/flytekit/common/types/impl/blobs.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os as _os import shutil as _shutil import sys as _sys @@ -16,7 +14,7 @@ from flytekit.models.core import types as _core_types -class Blob(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _literal_models.Blob)): +class Blob(_literal_models.Blob, metaclass=_sdk_bases.ExtendedSdkType): def __init__(self, remote_path, mode="rb", format=None): """ :param Text remote_path: Path to location where the Blob should be synced to. @@ -228,7 +226,7 @@ def upload(self): _data_proxy.Data.put_data(self.local_path, self.remote_location, is_multipart=False) -class MultiPartBlob(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _literal_models.Blob)): +class MultiPartBlob(_literal_models.Blob, metaclass=_sdk_bases.ExtendedSdkType): def __init__(self, remote_path, mode="rb", format=None): """ :param Text remote_path: Path to location where the Blob should be synced to. diff --git a/flytekit/common/types/impl/schema.py b/flytekit/common/types/impl/schema.py index 1e544d3bd6..10a28eaa29 100644 --- a/flytekit/common/types/impl/schema.py +++ b/flytekit/common/types/impl/schema.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import collections as _collections import os as _os import uuid as _uuid @@ -395,7 +393,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): return super(_SchemaBackingMpBlob, self).__exit__(exc_type, exc_val, exc_tb) -class SchemaType(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _type_models.SchemaType)): +class SchemaType(_type_models.SchemaType, metaclass=_sdk_bases.ExtendedSdkType): _LITERAL_TYPE_TO_PROTO_ENUM = { _primitives.Integer.to_flyte_literal_type(): _type_models.SchemaType.SchemaColumn.SchemaColumnType.INTEGER, _primitives.Float.to_flyte_literal_type(): _type_models.SchemaType.SchemaColumn.SchemaColumnType.FLOAT, @@ -510,7 +508,7 @@ def _set_columns(self, columns): self._sdk_columns = _collections.OrderedDict(columns) -class Schema(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _literal_models.Schema)): +class Schema(_literal_models.Schema, metaclass=_sdk_bases.ExtendedSdkType): def __init__(self, remote_path, mode="rb", schema_type=None): """ :param Text remote_path: diff --git a/flytekit/common/types/primitives.py b/flytekit/common/types/primitives.py index d94350aff5..77fbf7b313 100644 --- a/flytekit/common/types/primitives.py +++ b/flytekit/common/types/primitives.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime import json as _json diff --git a/flytekit/common/types/proto.py b/flytekit/common/types/proto.py index e7a0562c0b..d6202cb39b 100644 --- a/flytekit/common/types/proto.py +++ b/flytekit/common/types/proto.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import base64 as _base64 import six as _six @@ -52,7 +50,7 @@ def tag(cls): return "{}{}".format(Protobuf.TAG_PREFIX, cls.descriptor) -class Protobuf(_six.with_metaclass(ProtobufType, _base_sdk_types.FlyteSdkValue)): +class Protobuf(_base_sdk_types.FlyteSdkValue, metaclass=ProtobufType): PB_FIELD_KEY = "pb_type" TAG_PREFIX = "{}=".format(PB_FIELD_KEY) diff --git a/flytekit/common/types/schema.py b/flytekit/common/types/schema.py index 0c16f71869..8f0a9555d1 100644 --- a/flytekit/common/types/schema.py +++ b/flytekit/common/types/schema.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import - -import six as _six - from flytekit.common.exceptions import user as _user_exceptions from flytekit.common.types import base_sdk_types as _base_sdk_types from flytekit.common.types.impl import schema as _schema_impl @@ -81,7 +77,7 @@ def columns(cls): return cls.schema_type.sdk_columns -class Schema(_six.with_metaclass(SchemaInstantiator, _base_sdk_types.FlyteSdkValue)): +class Schema(_base_sdk_types.FlyteSdkValue, metaclass=SchemaInstantiator): @classmethod def from_string(cls, string_value): """ @@ -169,7 +165,7 @@ def schema_instantiator(columns=None): "When specifying a Schema type with a known set of columns, a non-empty list must be provided as " "inputs", ) - class _Schema(_six.with_metaclass(SchemaInstantiator, Schema)): + class _Schema(Schema, metaclass=SchemaInstantiator): _schema_type = _schema_impl.SchemaType(columns=columns) return _Schema @@ -181,7 +177,7 @@ def schema_instantiator_from_proto(schema_type): :rtype: SchemaInstantiator """ - class _Schema(_six.with_metaclass(SchemaInstantiator, Schema)): + class _Schema(Schema, metaclass=SchemaInstantiator): _schema_type = _schema_impl.SchemaType.promote_from_model(schema_type) return _Schema diff --git a/flytekit/common/utils.py b/flytekit/common/utils.py index 92737b4a1d..ac8216d075 100644 --- a/flytekit/common/utils.py +++ b/flytekit/common/utils.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import logging as _logging import os as _os import shutil as _shutil diff --git a/flytekit/common/workflow.py b/flytekit/common/workflow.py index 4035c97060..c9f8edbc18 100644 --- a/flytekit/common/workflow.py +++ b/flytekit/common/workflow.py @@ -1,16 +1,8 @@ -from __future__ import absolute_import - import datetime as _datetime -import uuid as _uuid - -import six as _six -from six.moves import queue as _queue from flytekit.common import constants as _constants from flytekit.common import interface as _interface -from flytekit.common import launch_plan as _launch_plan from flytekit.common import nodes as _nodes -from flytekit.common import promise as _promise from flytekit.common import sdk_bases as _sdk_bases from flytekit.common.core import identifier as _identifier from flytekit.common.exceptions import scopes as _exception_scopes @@ -18,99 +10,32 @@ from flytekit.common.exceptions import user as _user_exceptions from flytekit.common.mixins import hash as _hash_mixin from flytekit.common.mixins import registerable as _registerable -from flytekit.common.types import helpers as _type_helpers from flytekit.configuration import internal as _internal_config -from flytekit.engines import loader as _engine_loader -from flytekit.models import common as _common_models -from flytekit.models import interface as _interface_models +from flytekit.engines.flyte import engine as _flyte_engine from flytekit.models import literals as _literal_models from flytekit.models.admin import workflow as _admin_workflow_model from flytekit.models.core import identifier as _identifier_model from flytekit.models.core import workflow as _workflow_models -class Output(object): - def __init__(self, name, value, sdk_type=None, help=None): - """ - :param Text name: - :param T value: - :param U sdk_type: If specified, the value provided must cast to this type. Normally should be an instance of - flytekit.common.types.base_sdk_types.FlyteSdkType. But could also be something like: - - list[flytekit.common.types.base_sdk_types.FlyteSdkType], - dict[flytekit.common.types.base_sdk_types.FlyteSdkType,flytekit.common.types.base_sdk_types.FlyteSdkType], - (flytekit.common.types.base_sdk_types.FlyteSdkType, flytekit.common.types.base_sdk_types.FlyteSdkType, ...) - """ - if sdk_type is None: - # This syntax didn't work for some reason: sdk_type = sdk_type or Output._infer_type(value) - sdk_type = Output._infer_type(value) - sdk_type = _type_helpers.python_std_to_sdk_type(sdk_type) - - self._binding_data = _interface.BindingData.from_python_std(sdk_type.to_flyte_literal_type(), value) - self._var = _interface_models.Variable(sdk_type.to_flyte_literal_type(), help or "") - self._name = name - - def rename_and_return_reference(self, new_name): - self._name = new_name - return self - - @staticmethod - def _infer_type(value): - # TODO: Infer types - raise NotImplementedError( - "Currently the SDK cannot infer a workflow output type, so please use the type kwarg " - "when instantiating an output." - ) - - @property - def name(self): - """ - :rtype: Text - """ - return self._name - - @property - def binding_data(self): - """ - :rtype: flytekit.models.literals.BindingData - """ - return self._binding_data - - @property - def var(self): - """ - :rtype: flytekit.models.interface.Variable - """ - return self._var - - class SdkWorkflow( - _six.with_metaclass( - _sdk_bases.ExtendedSdkType, - _hash_mixin.HashOnReferenceMixin, - _workflow_models.WorkflowTemplate, - _registerable.RegisterableEntity, - ) + _hash_mixin.HashOnReferenceMixin, + _registerable.HasDependencies, + _registerable.RegisterableEntity, + _workflow_models.WorkflowTemplate, + metaclass=_sdk_bases.ExtendedSdkType, ): + """ + Previously this class represented both local and control plane constructs. As of this writing, we are making this + class only a control plane class. Workflow constructs that rely on local code being present have been moved to + the new PythonWorkflow class. + """ + def __init__( - self, - inputs, - outputs, - nodes, - id=None, - metadata=None, - metadata_defaults=None, - interface=None, - output_bindings=None, - disable_default_launch_plan=False, + self, nodes, interface, output_bindings, id, metadata, metadata_defaults, ): """ - :param list[flytekit.common.promise.Input] inputs: - :param list[Output] outputs: :param list[flytekit.common.nodes.SdkNode] nodes: - :param flytekit.models.core.identifier.Identifier id: This is an autogenerated id by the system. The id is - globally unique across Flyte. - :param WorkflowMetadata metadata: This contains information on how to run the workflow. :param flytekit.models.interface.TypedInterface interface: Defines a strongly typed interface for the Workflow (inputs, outputs). This can include some optional parameters. :param list[flytekit.models.literals.Binding] output_bindings: A list of output bindings that specify how to construct @@ -118,8 +43,11 @@ def __init__( the interface field must be bound in order for the workflow to be validated. A workflow has an implicit dependency on all of its nodes to execute successfully in order to bind final outputs. - :param bool disable_default_launch_plan: Determines whether to create a default launch plan for the workflow. - + :param flytekit.models.core.identifier.Identifier id: This is an autogenerated id by the system. The id is + globally unique across Flyte. + :param WorkflowMetadata metadata: This contains information on how to run the workflow. + :param flytekit.models.core.workflow.WorkflowMetadataDefaults metadata_defaults: Defaults to be passed + to nodes contained within workflow. """ for n in nodes: for upstream in n.upstream_nodes: @@ -130,59 +58,19 @@ def __init__( "list, dict, or tuple which is stored as an attribute in the class." ) - # Allow overrides if specified for all the arguments to the parent class constructor - id = ( - id - if id is not None - else _identifier.Identifier( - _identifier_model.ResourceType.WORKFLOW, - _internal_config.PROJECT.get(), - _internal_config.DOMAIN.get(), - _uuid.uuid4().hex, - _internal_config.VERSION.get(), - ) - ) - metadata = metadata if metadata is not None else _workflow_models.WorkflowMetadata() - - interface = ( - interface - if interface is not None - else _interface.TypedInterface({v.name: v.var for v in inputs}, {v.name: v.var for v in outputs}) - ) - - output_bindings = ( - output_bindings - if output_bindings is not None - else [_literal_models.Binding(v.name, v.binding_data) for v in outputs] - ) - super(SdkWorkflow, self).__init__( id=id, metadata=metadata, - metadata_defaults=_workflow_models.WorkflowMetadataDefaults(), + metadata_defaults=metadata_defaults, interface=interface, nodes=nodes, outputs=output_bindings, ) - self._user_inputs = inputs - self._upstream_entities = set(n.executable_sdk_object for n in nodes) - self._should_create_default_launch_plan = not disable_default_launch_plan - - @property - def should_create_default_launch_plan(self): - """ - Determines whether registration flow should create a default launch plan for this workflow or not. - :rtype: bool - """ - return self._should_create_default_launch_plan + self._sdk_nodes = nodes @property def upstream_entities(self): - """ - Task, workflow, and launch plan that need to be registered in advance of this workflow. - :rtype: set[T] - """ - return self._upstream_entities + return set(n.executable_sdk_object for n in self._sdk_nodes) @property def interface(self): @@ -206,13 +94,6 @@ def resource_type(self): """ return _identifier_model.ResourceType.WORKFLOW - @property - def user_inputs(self): - """ - :rtype: list[flytekit.common.promise.Input] - """ - return self._user_inputs - def get_sub_workflows(self): """ Recursive call that returns all subworkflows in the current workflow @@ -247,13 +128,14 @@ def fetch(cls, project, domain, name, version=None): """ version = version or _internal_config.VERSION.get() workflow_id = _identifier.Identifier(_identifier_model.ResourceType.WORKFLOW, project, domain, name, version) - admin_workflow = _engine_loader.get_engine().fetch_workflow(workflow_id) + admin_workflow = _flyte_engine.get_client().get_workflow(workflow_id) cwc = admin_workflow.closure.compiled_workflow primary_template = cwc.primary.template sub_workflow_map = {sw.template.id: sw.template for sw in cwc.sub_workflows} task_map = {t.template.id: t.template for t in cwc.tasks} sdk_workflow = cls.promote_from_model(primary_template, sub_workflow_map, task_map) sdk_workflow._id = workflow_id + sdk_workflow._has_registered = True return sdk_workflow @classmethod @@ -292,8 +174,6 @@ def promote_from_model(cls, base_model, sub_workflows=None, tasks=None): # No inputs/outputs specified, see the constructor for more information on the overrides. return cls( - inputs=None, - outputs=None, nodes=list(node_map.values()), id=_identifier.Identifier.promote_from_model(base_model.id), metadata=base_model.metadata, @@ -313,10 +193,16 @@ def register(self, project, domain, name, version): self.validate() id_to_register = _identifier.Identifier(_identifier_model.ResourceType.WORKFLOW, project, domain, name, version) old_id = self.id + self._id = id_to_register try: + client = _flyte_engine.get_client() + sub_workflows = self.get_sub_workflows() + client.create_workflow(id_to_register, _admin_workflow_model.WorkflowSpec(self, sub_workflows,)) self._id = id_to_register - _engine_loader.get_engine().get_workflow(self).register(id_to_register) - return _six.text_type(self.id) + self._has_registered = True + return str(id_to_register) + except _user_exceptions.FlyteEntityAlreadyExistsException: + pass except Exception: self._id = old_id raise @@ -337,66 +223,8 @@ def validate(self): pass @_exception_scopes.system_entry_point - def create_launch_plan( - self, - default_inputs=None, - fixed_inputs=None, - schedule=None, - role=None, - notifications=None, - labels=None, - annotations=None, - assumable_iam_role=None, - kubernetes_service_account=None, - raw_output_data_prefix=None, - cls=None, - ): - """ - This method will create a launch plan object that can execute this workflow. - :param dict[Text,flytekit.common.promise.Input] default_inputs: - :param dict[Text,T] fixed_inputs: - :param flytekit.models.schedule.Schedule schedule: A schedule on which to execute this launch plan. - :param Text role: Deprecated. Use assumable_iam_role instead. - :param list[flytekit.models.common.Notification] notifications: A list of notifications to enact by default for - this launch plan. - :param flytekit.models.common.Labels labels: - :param flytekit.models.common.Annotations annotations: - :param cls: This parameter can be used by users to define an extension of a launch plan to instantiate. The - class provided should be a subclass of flytekit.common.launch_plan.SdkLaunchPlan. - :param Text assumable_iam_role: The IAM role to execute the workflow with. - :param Text kubernetes_service_account: The kubernetes service account to execute the workflow with. - :param Text raw_output_data_prefix: Bucket for offloaded data - - :rtype: flytekit.common.launch_plan.SdkRunnableLaunchPlan - """ - # TODO: Actually ensure the parameters conform. - if role and (assumable_iam_role or kubernetes_service_account): - raise ValueError("Cannot set both role and auth. Role is deprecated, use auth instead.") - fixed_inputs = fixed_inputs or {} - merged_default_inputs = {v.name: v for v in self._user_inputs if v.name not in fixed_inputs} - merged_default_inputs.update(default_inputs or {}) - - if role: - assumable_iam_role = role - auth_role = _common_models.AuthRole( - assumable_iam_role=assumable_iam_role, kubernetes_service_account=kubernetes_service_account, - ) - - raw_output_config = _common_models.RawOutputDataConfig(raw_output_data_prefix or "") - - return (cls or _launch_plan.SdkRunnableLaunchPlan)( - sdk_workflow=self, - default_inputs={ - k: user_input.rename_and_return_reference(k) for k, user_input in _six.iteritems(merged_default_inputs) - }, - fixed_inputs=fixed_inputs, - schedule=schedule, - notifications=notifications, - labels=labels, - annotations=annotations, - auth_role=auth_role, - raw_output_data_config=raw_output_config, - ) + def create_launch_plan(self, *args, **kwargs): + raise Exception("This can't be done right now.") @_exception_scopes.system_entry_point def __call__(self, *args, **input_map): @@ -406,11 +234,7 @@ def __call__(self, *args, **input_map): "detected {} positional args.".format(len(args)) ) - # Take the default values from the Inputs - compiled_inputs = {v.name: v.sdk_default for v in self.user_inputs if not v.sdk_required} - compiled_inputs.update(input_map) - - bindings, upstream_nodes = self.interface.create_bindings_for_inputs(compiled_inputs) + bindings, upstream_nodes = self.interface.create_bindings_for_inputs(input_map) node = _nodes.SdkNode( id=None, @@ -422,86 +246,3 @@ def __call__(self, *args, **input_map): sdk_workflow=self, ) return node - - -def _assign_indexed_attribute_name(attribute_name, index): - return "{}[{}]".format(attribute_name, index) - - -def _discover_workflow_components(workflow_class): - """ - This task iterates over the attributes of a user-defined class in order to return a list of inputs, outputs and - nodes. - :param class workflow_class: User-defined class with task instances as attributes. - :rtype: (list[flytekit.common.promise.Input], list[Output], list[flytekit.common.nodes.SdkNode]) - """ - - inputs = [] - outputs = [] - nodes = [] - - to_visit_objs = _queue.Queue() - top_level_attributes = set() - for attribute_name in dir(workflow_class): - to_visit_objs.put((attribute_name, getattr(workflow_class, attribute_name))) - top_level_attributes.add(attribute_name) - - # For all task instances defined within the workflow, bind them to this specific workflow and hook-up to the - # engine (when available) - visited_obj_ids = set() - while not to_visit_objs.empty(): - attribute_name, current_obj = to_visit_objs.get() - - current_obj_id = id(current_obj) - if current_obj_id in visited_obj_ids: - continue - visited_obj_ids.add(current_obj_id) - - if isinstance(current_obj, _nodes.SdkNode): - # TODO: If an attribute name is on the form node_name[index], the resulting - # node name might not be correct. - nodes.append(current_obj.assign_id_and_return(attribute_name)) - elif isinstance(current_obj, _promise.Input): - if attribute_name is None or attribute_name not in top_level_attributes: - raise _user_exceptions.FlyteValueException( - attribute_name, "Detected workflow input specified outside of top level.", - ) - inputs.append(current_obj.rename_and_return_reference(attribute_name)) - elif isinstance(current_obj, Output): - if attribute_name is None or attribute_name not in top_level_attributes: - raise _user_exceptions.FlyteValueException( - attribute_name, "Detected workflow output specified outside of top level.", - ) - outputs.append(current_obj.rename_and_return_reference(attribute_name)) - elif isinstance(current_obj, list) or isinstance(current_obj, set) or isinstance(current_obj, tuple): - for idx, value in enumerate(current_obj): - to_visit_objs.put((_assign_indexed_attribute_name(attribute_name, idx), value)) - elif isinstance(current_obj, dict): - # Visit dictionary keys. - for key in current_obj.keys(): - to_visit_objs.put((_assign_indexed_attribute_name(attribute_name, key), key)) - # Visit dictionary values. - for key, value in _six.iteritems(current_obj): - to_visit_objs.put((_assign_indexed_attribute_name(attribute_name, key), value)) - return inputs, outputs, nodes - - -def build_sdk_workflow_from_metaclass(metaclass, on_failure=None, disable_default_launch_plan=False, cls=None): - """ - :param T metaclass: - :param cls: This is the class that will be instantiated from the inputs, outputs, and nodes. This will be used - by users extending the base Flyte programming model. If set, it must be a subclass of SdkWorkflow. - :param flytekit.models.core.workflow.WorkflowMetadata.OnFailurePolicy on_failure: [Optional] The execution policy - when the workflow detects a failure. - :param bool disable_default_launch_plan: Determines whether to create a default launch plan for the workflow or not. - :rtype: SdkWorkflow - """ - inputs, outputs, nodes = _discover_workflow_components(metaclass) - metadata = _workflow_models.WorkflowMetadata(on_failure=on_failure if on_failure else None) - return (cls or SdkWorkflow)( - inputs=[i for i in sorted(inputs, key=lambda x: x.name)], - outputs=[o for o in sorted(outputs, key=lambda x: x.name)], - nodes=[n for n in sorted(nodes, key=lambda x: x.id)], - metadata=metadata, - disable_default_launch_plan=disable_default_launch_plan, - ) diff --git a/flytekit/common/workflow_execution.py b/flytekit/common/workflow_execution.py index 1af0020d05..f5064cc136 100644 --- a/flytekit/common/workflow_execution.py +++ b/flytekit/common/workflow_execution.py @@ -1,20 +1,25 @@ -from __future__ import absolute_import +import os as _os import six as _six +from flyteidl.core import literals_pb2 as _literals_pb2 +from flytekit.clients.helpers import iterate_node_executions as _iterate_node_executions from flytekit.common import nodes as _nodes from flytekit.common import sdk_bases as _sdk_bases +from flytekit.common import utils as _common_utils from flytekit.common.core import identifier as _core_identifier from flytekit.common.exceptions import user as _user_exceptions from flytekit.common.mixins import artifact as _artifact from flytekit.common.types import helpers as _type_helpers -from flytekit.engines import loader as _engine_loader +from flytekit.engines.flyte import engine as _flyte_engine +from flytekit.interfaces.data import data_proxy as _data_proxy from flytekit.models import execution as _execution_models +from flytekit.models import literals as _literal_models from flytekit.models.core import execution as _core_execution_models class SdkWorkflowExecution( - _six.with_metaclass(_sdk_bases.ExtendedSdkType, _execution_models.Execution, _artifact.ExecutionArtifact,) + _execution_models.Execution, _artifact.ExecutionArtifact, metaclass=_sdk_bases.ExtendedSdkType, ): def __init__(self, *args, **kwargs): super(SdkWorkflowExecution, self).__init__(*args, **kwargs) @@ -36,9 +41,23 @@ def inputs(self): :rtype: dict[Text, T] """ if self._inputs is None: - self._inputs = _type_helpers.unpack_literal_map_to_sdk_python_std( - _engine_loader.get_engine().get_workflow_execution(self).get_inputs() - ) + client = _flyte_engine.get_client() + execution_data = client.get_execution_data(self.id) + + # Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. + if bool(execution_data.full_inputs.literals): + input_map = execution_data.full_inputs + elif execution_data.inputs.bytes > 0: + with _common_utils.AutoDeletingTempDir() as t: + tmp_name = _os.path.join(t.name, "inputs.pb") + _data_proxy.Data.get_data(execution_data.inputs.url, tmp_name) + input_map = _literal_models.LiteralMap.from_flyte_idl( + _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name) + ) + else: + input_map = _literal_models.LiteralMap({}) + + self._inputs = _type_helpers.unpack_literal_map_to_sdk_python_std(input_map) return self._inputs @property @@ -56,9 +75,24 @@ def outputs(self): raise _user_exceptions.FlyteAssertion("Outputs could not be found because the execution ended in failure.") if self._outputs is None: - self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std( - _engine_loader.get_engine().get_workflow_execution(self).get_outputs() - ) + client = _flyte_engine.get_client() + + execution_data = client.get_execution_data(self.id) + # Outputs are returned inline unless they are too big, in which case a url blob pointing to them is returned. + if bool(execution_data.full_outputs.literals): + output_map = execution_data.full_outputs + + elif execution_data.outputs.bytes > 0: + with _common_utils.AutoDeletingTempDir() as t: + tmp_name = _os.path.join(t.name, "outputs.pb") + _data_proxy.Data.get_data(execution_data.outputs.url, tmp_name) + output_map = _literal_models.LiteralMap.from_flyte_idl( + _common_utils.load_proto_from_file(_literals_pb2.LiteralMap, tmp_name) + ) + else: + output_map = _literal_models.LiteralMap({}) + + self._outputs = _type_helpers.unpack_literal_map_to_sdk_python_std(output_map) return self._outputs @property @@ -107,11 +141,10 @@ def fetch(cls, project, domain, name): :param Text name: :rtype: SdkWorkflowExecution """ - return cls.promote_from_model( - _engine_loader.get_engine().fetch_workflow_execution( - _core_identifier.WorkflowExecutionIdentifier(project=project, domain=domain, name=name) - ) - ) + wf_exec_id = _core_identifier.WorkflowExecutionIdentifier(project=project, domain=domain, name=name) + admin_exec = _flyte_engine.get_client().get_execution(wf_exec_id) + + return cls.promote_from_model(admin_exec) def sync(self): """ @@ -128,18 +161,21 @@ def _sync_closure(self): :rtype: None """ if not self.is_complete: - _engine_loader.get_engine().get_workflow_execution(self).sync() + client = _flyte_engine.get_client() + self._closure = client.get_execution(self.id).closure def get_node_executions(self, filters=None): """ :param list[flytekit.models.filters.Filter] filters: :rtype: dict[Text, flytekit.common.nodes.SdkNodeExecution] """ - models = _engine_loader.get_engine().get_workflow_execution(self).get_node_executions(filters=filters) - return {k: _nodes.SdkNodeExecution.promote_from_model(v) for k, v in _six.iteritems(models)} + client = _flyte_engine.get_client() + node_exec_models = {v.id.node_id: v for v in _iterate_node_executions(client, self.id, filters=filters)} + + return {k: _nodes.SdkNodeExecution.promote_from_model(v) for k, v in _six.iteritems(node_exec_models)} def terminate(self, cause): """ :param Text cause: """ - _engine_loader.get_engine().get_workflow_execution(self).terminate(cause) + _flyte_engine.get_client().terminate_execution(self.id, cause) diff --git a/flytekit/configuration/__init__.py b/flytekit/configuration/__init__.py index 2df8c7d4f4..ca24895524 100644 --- a/flytekit/configuration/__init__.py +++ b/flytekit/configuration/__init__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import logging as _logging import os as _os diff --git a/flytekit/configuration/auth.py b/flytekit/configuration/auth.py index a7d1d9917c..5eef062244 100644 --- a/flytekit/configuration/auth.py +++ b/flytekit/configuration/auth.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.configuration import common as _config_common ASSUMABLE_IAM_ROLE = _config_common.FlyteStringConfigurationEntry("auth", "assumable_iam_role", default=None) diff --git a/flytekit/configuration/aws.py b/flytekit/configuration/aws.py index 962f7f9f52..a6af62bf6f 100644 --- a/flytekit/configuration/aws.py +++ b/flytekit/configuration/aws.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.configuration import common as _config_common S3_SHARD_FORMATTER = _config_common.FlyteRequiredStringConfigurationEntry("aws", "s3_shard_formatter") diff --git a/flytekit/configuration/common.py b/flytekit/configuration/common.py index 55cab24010..8429b1d0e9 100644 --- a/flytekit/configuration/common.py +++ b/flytekit/configuration/common.py @@ -1,11 +1,7 @@ -from __future__ import absolute_import - import abc as _abc import configparser as _configparser import os as _os -import six as _six - from flytekit.common.exceptions import user as _user_exceptions @@ -132,7 +128,7 @@ def _get_file_contents(location): return None -class _FlyteConfigurationEntry(_six.with_metaclass(_abc.ABCMeta, object)): +class _FlyteConfigurationEntry(object, metaclass=_abc.ABCMeta): def __init__(self, section, key, default=None, validator=None, fallback=None): self._section = section self._key = key diff --git a/flytekit/configuration/creds.py b/flytekit/configuration/creds.py index 06202f9a7b..7f559b1924 100644 --- a/flytekit/configuration/creds.py +++ b/flytekit/configuration/creds.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.configuration import common as _config_common CLIENT_ID = _config_common.FlyteStringConfigurationEntry("credentials", "client_id", default=None) diff --git a/flytekit/configuration/gcp.py b/flytekit/configuration/gcp.py index 9e5d50cd1a..11e9452083 100644 --- a/flytekit/configuration/gcp.py +++ b/flytekit/configuration/gcp.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.configuration import common as _config_common GCS_PREFIX = _config_common.FlyteRequiredStringConfigurationEntry("gcp", "gcs_prefix") diff --git a/flytekit/configuration/internal.py b/flytekit/configuration/internal.py index bd3a238ca7..b640dc2082 100644 --- a/flytekit/configuration/internal.py +++ b/flytekit/configuration/internal.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import re from flytekit.configuration import common as _common_config diff --git a/flytekit/configuration/platform.py b/flytekit/configuration/platform.py index 42a72175f4..2dd59850b7 100644 --- a/flytekit/configuration/platform.py +++ b/flytekit/configuration/platform.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common import constants as _constants from flytekit.configuration import common as _config_common diff --git a/flytekit/configuration/resources.py b/flytekit/configuration/resources.py index 783a8e672f..391b3316e4 100644 --- a/flytekit/configuration/resources.py +++ b/flytekit/configuration/resources.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.configuration import common as _config_common DEFAULT_CPU_LIMIT = _config_common.FlyteStringConfigurationEntry("resources", "default_cpu_limit") diff --git a/flytekit/configuration/sdk.py b/flytekit/configuration/sdk.py index 14c8d21e65..2177eba699 100644 --- a/flytekit/configuration/sdk.py +++ b/flytekit/configuration/sdk.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.configuration import common as _config_common WORKFLOW_PACKAGES = _config_common.FlyteStringListConfigurationEntry("sdk", "workflow_packages", default=[]) diff --git a/flytekit/configuration/statsd.py b/flytekit/configuration/statsd.py index 682b88ac6e..98f2f2ed96 100644 --- a/flytekit/configuration/statsd.py +++ b/flytekit/configuration/statsd.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.configuration import common as _common_config HOST = _common_config.FlyteStringConfigurationEntry("statsd", "host", default="localhost") diff --git a/flytekit/contrib/notebook/tasks.py b/flytekit/contrib/notebook/tasks.py index ca47eabd62..3428ec402f 100644 --- a/flytekit/contrib/notebook/tasks.py +++ b/flytekit/contrib/notebook/tasks.py @@ -365,42 +365,12 @@ def _get_container_definition( gpu_limit = gpu_limit or gpu_request memory_limit = memory_limit or memory_request - requests = [] - if storage_request: - requests.append( - _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.STORAGE, storage_request) - ) - if cpu_request: - requests.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.CPU, cpu_request)) - if gpu_request: - requests.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.GPU, gpu_request)) - if memory_request: - requests.append( - _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.MEMORY, memory_request) - ) - - limits = [] - if storage_limit: - limits.append( - _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.STORAGE, storage_limit) - ) - if cpu_limit: - limits.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.CPU, cpu_limit)) - if gpu_limit: - limits.append(_task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.GPU, gpu_limit)) - if memory_limit: - limits.append( - _task_models.Resources.ResourceEntry(_task_models.Resources.ResourceName.MEMORY, memory_limit) - ) - - return _sdk_runnable.SdkRunnableContainer( - command=[], - args=[], - resources=_task_models.Resources(limits=limits, requests=requests), - env=environment, - config={}, + resources = _sdk_runnable.SdkRunnableContainer.get_resources( + storage_request, cpu_request, gpu_request, memory_request, storage_limit, cpu_limit, gpu_limit, memory_limit ) + return _sdk_runnable.SdkRunnableContainer(command=[], args=[], resources=resources, env=environment, config={},) + def spark_notebook( notebook_path, diff --git a/flytekit/contrib/sensors/base_sensor.py b/flytekit/contrib/sensors/base_sensor.py index 323976b001..8522918a34 100644 --- a/flytekit/contrib/sensors/base_sensor.py +++ b/flytekit/contrib/sensors/base_sensor.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import abc as _abc import datetime as _datetime import logging as _logging @@ -10,7 +8,7 @@ import six as _six -class Sensor(_six.with_metaclass(_abc.ABCMeta, object)): +class Sensor(object, metaclass=_abc.ABCMeta): def __init__(self, evaluation_interval=None, max_failures=0): """ :param datetime.timedelta evaluation_interval: This is the time to wait between evaluation attempts of this diff --git a/flytekit/contrib/sensors/impl.py b/flytekit/contrib/sensors/impl.py index ce601922c1..670df86b5d 100644 --- a/flytekit/contrib/sensors/impl.py +++ b/flytekit/contrib/sensors/impl.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.contrib.sensors.base_sensor import Sensor as _Sensor from flytekit.plugins import hmsclient as _hmsclient diff --git a/flytekit/contrib/sensors/task.py b/flytekit/contrib/sensors/task.py index 6f427cbb51..2bacae122f 100644 --- a/flytekit/contrib/sensors/task.py +++ b/flytekit/contrib/sensors/task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common import constants as _common_constants from flytekit.common.exceptions import user as _user_exceptions from flytekit.common.tasks import sdk_runnable as _sdk_runnable diff --git a/flytekit/engines/common.py b/flytekit/engines/common.py index 48d7864c77..3250d8d107 100644 --- a/flytekit/engines/common.py +++ b/flytekit/engines/common.py @@ -1,13 +1,9 @@ -from __future__ import absolute_import - import abc as _abc -import six as _six - from flytekit.models import common as _common_models -class BaseWorkflowExecutor(_six.with_metaclass(_common_models.FlyteABCMeta, object)): +class BaseWorkflowExecutor(object, metaclass=_common_models.FlyteABCMeta): """ This class must be implemented for any engine to create, interact with, and execute workflows using the FlyteKit SDK. @@ -35,7 +31,7 @@ def register(self, identifier): pass -class BaseWorkflowExecution(_six.with_metaclass(_common_models.FlyteABCMeta, object)): +class BaseWorkflowExecution(object, metaclass=_common_models.FlyteABCMeta): """ This class must be implemented for any engine to track and interact with the executions of workflows. """ @@ -90,7 +86,7 @@ def terminate(self, cause): pass -class BaseNodeExecution(_six.with_metaclass(_common_models.FlyteABCMeta, object)): +class BaseNodeExecution(object, metaclass=_common_models.FlyteABCMeta): def __init__(self, node_execution): """ :param flytekit.common.nodes.SdkNodeExecution node_execution: @@ -140,7 +136,7 @@ def sync(self): pass -class BaseTaskExecution(_six.with_metaclass(_common_models.FlyteABCMeta, object)): +class BaseTaskExecution(object, metaclass=_common_models.FlyteABCMeta): def __init__(self, task_exec): """ :param flytekit.common.tasks.executions.SdkTaskExecution task_exec: @@ -184,7 +180,7 @@ def get_child_executions(self, filters=None): pass -class BaseLaunchPlanLauncher(_six.with_metaclass(_common_models.FlyteABCMeta, object)): +class BaseLaunchPlanLauncher(object, metaclass=_common_models.FlyteABCMeta): def __init__(self, sdk_launch_plan): """ :param flytekit.common.launch_plan.SdkLaunchPlan sdk_launch_plan: @@ -240,7 +236,7 @@ def update(self, identifier, state): pass -class BaseTaskExecutor(_six.with_metaclass(_common_models.FlyteABCMeta, object)): +class BaseTaskExecutor(object, metaclass=_common_models.FlyteABCMeta): def __init__(self, sdk_task): """ :param flytekit.common.tasks.task.SdkTask sdk_task: @@ -297,19 +293,11 @@ def launch( pass -class BaseExecutionEngineFactory(_six.with_metaclass(_common_models.FlyteABCMeta, object)): +class BaseExecutionEngineFactory(object, metaclass=_common_models.FlyteABCMeta): """ This object should be implemented to satisfy the basic engine interface. """ - @_abc.abstractmethod - def get_workflow(self, sdk_workflow): - """ - :param flytekit.common.workflow.SdkWorkflow sdk_workflow: - :rtype: BaseWorkflowExecutor - """ - pass - @_abc.abstractmethod def get_task(self, sdk_task): """ @@ -376,24 +364,6 @@ def fetch_latest_task(self, named_task): """ pass - @_abc.abstractmethod - def fetch_launch_plan(self, launch_plan_id): - """ - :param flytekit.models.core.identifier.Identifier launch_plan_id: This identifier should have a resource - type of kind LaunchPlan. - :rtype: flytekit.models.launch_plan.LaunchPlan - """ - pass - - @_abc.abstractmethod - def fetch_workflow(self, workflow_id): - """ - :param flytekit.models.core.identifier.Identifier workflow_id: This identifier should have a resource - type of kind workflow. - :rtype: flytekit.models.admin.workflow.Workflow - """ - pass - class EngineContext(object): def __init__(self, execution_date, tmp_dir, stats, execution_id, logging, raw_output_data_prefix=None): diff --git a/flytekit/engines/flyte/engine.py b/flytekit/engines/flyte/engine.py index 32b7c39ea4..d0e0629f7b 100644 --- a/flytekit/engines/flyte/engine.py +++ b/flytekit/engines/flyte/engine.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import logging as _logging import os as _os import traceback as _traceback @@ -52,6 +50,12 @@ def client(self): return type(self)._CLIENT +# This is a simple helper function that ties the client together with the configuration construct. +# This will be refactored away when we move to a heavier context object. +def get_client() -> _SynchronousFlyteClient: + return _FlyteClientManager(_platform_config.URL.get(), insecure=_platform_config.INSECURE.get()).client + + class FlyteEngineFactory(_common_engine.BaseExecutionEngineFactory): def get_workflow(self, sdk_workflow): """ @@ -95,6 +99,9 @@ def get_workflow_execution(self, wf_exec): """ return FlyteWorkflowExecution(wf_exec) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def fetch_workflow_execution(self, wf_exec_id): """ :param flytekit.models.core.identifier.WorkflowExecutionIdentifier wf_exec_id: @@ -104,6 +111,9 @@ def fetch_workflow_execution(self, wf_exec_id): _platform_config.URL.get(), insecure=_platform_config.INSECURE.get() ).client.get_execution(wf_exec_id) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def fetch_task(self, task_id): """ Queries Admin for an existing Admin task @@ -114,6 +124,9 @@ def fetch_task(self, task_id): _platform_config.URL.get(), insecure=_platform_config.INSECURE.get() ).client.get_task(task_id) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def fetch_latest_task(self, named_task): """ Fetches the latest task @@ -127,6 +140,9 @@ def fetch_latest_task(self, named_task): ) return task_list[0] if task_list else None + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def fetch_launch_plan(self, launch_plan_id): """ :param flytekit.models.core.identifier.Identifier launch_plan_id: This identifier should have a resource @@ -145,6 +161,9 @@ def fetch_launch_plan(self, launch_plan_id): _platform_config.URL.get(), insecure=_platform_config.INSECURE.get() ).client.get_active_launch_plan(named_entity_id) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def fetch_workflow(self, workflow_id): """ :param flytekit.models.core.identifier.Identifier workflow_id: This identifier should have a resource @@ -157,6 +176,9 @@ def fetch_workflow(self, workflow_id): class FlyteLaunchPlan(_common_engine.BaseLaunchPlanLauncher): + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def register(self, identifier): client = _FlyteClientManager(_platform_config.URL.get(), insecure=_platform_config.INSECURE.get()).client try: @@ -182,6 +204,9 @@ def execute( project, domain, name, inputs, notification_overrides, label_overrides, annotation_overrides, ) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def launch( self, project, @@ -235,6 +260,9 @@ def launch( exec_id = _identifier.WorkflowExecutionIdentifier(project, domain, name) return client.get_execution(exec_id) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def update(self, identifier, state): """ :param flytekit.models.core.identifier.Identifier identifier: Identifier for launch plan to update @@ -246,6 +274,9 @@ def update(self, identifier, state): class FlyteWorkflow(_common_engine.BaseWorkflowExecutor): + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def register(self, identifier): client = _FlyteClientManager(_platform_config.URL.get(), insecure=_platform_config.INSECURE.get()).client try: @@ -256,6 +287,9 @@ def register(self, identifier): class FlyteTask(_common_engine.BaseTaskExecutor): + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def register(self, identifier): client = _FlyteClientManager(_platform_config.URL.get(), insecure=_platform_config.INSECURE.get()).client try: @@ -341,6 +375,9 @@ def execute(self, inputs, context=None): temp_dir.name, context["output_prefix"], is_multipart=True, ) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def launch( self, project, @@ -414,6 +451,9 @@ def launch( class FlyteWorkflowExecution(_common_engine.BaseWorkflowExecution): + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def get_node_executions(self, filters=None): """ :param list[flytekit.models.filters.Filter] filters: @@ -424,6 +464,9 @@ def get_node_executions(self, filters=None): v.id.node_id: v for v in _iterate_node_executions(client, self.sdk_workflow_execution.id, filters=filters) } + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def sync(self): """ :rtype: None @@ -431,6 +474,9 @@ def sync(self): client = _FlyteClientManager(_platform_config.URL.get(), insecure=_platform_config.INSECURE.get()).client self.sdk_workflow_execution._closure = client.get_execution(self.sdk_workflow_execution.id).closure + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def get_inputs(self): """ :rtype: flytekit.models.literals.LiteralMap @@ -451,6 +497,9 @@ def get_inputs(self): ) return _literals.LiteralMap({}) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def get_outputs(self): """ :rtype: flytekit.models.literals.LiteralMap @@ -471,6 +520,9 @@ def get_outputs(self): ) return _literals.LiteralMap({}) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def terminate(self, cause): """ :param Text cause: @@ -481,6 +533,9 @@ def terminate(self, cause): class FlyteNodeExecution(_common_engine.BaseNodeExecution): + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def get_task_executions(self): """ :rtype: list[flytekit.common.tasks.executions.SdkTaskExecution] @@ -488,12 +543,18 @@ def get_task_executions(self): client = _FlyteClientManager(_platform_config.URL.get(), insecure=_platform_config.INSECURE.get()).client return list(_iterate_task_executions(client, self.sdk_node_execution.id)) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def get_subworkflow_executions(self): """ :rtype: list[flytekit.common.workflow_execution.SdkWorkflowExecution] """ raise NotImplementedError("Cannot retrieve sub-workflow information from a node execution yet.") + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def get_inputs(self): """ :rtype: flytekit.models.literals.LiteralMap @@ -514,6 +575,9 @@ def get_inputs(self): ) return _literals.LiteralMap({}) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def get_outputs(self): """ :rtype: flytekit.models.literals.LiteralMap @@ -534,6 +598,9 @@ def get_outputs(self): ) return _literals.LiteralMap({}) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def sync(self): """ :rtype: None @@ -543,6 +610,9 @@ def sync(self): class FlyteTaskExecution(_common_engine.BaseTaskExecution): + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def get_inputs(self): """ :rtype: flytekit.models.literals.LiteralMap @@ -563,6 +633,9 @@ def get_inputs(self): ) return _literals.LiteralMap({}) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def get_outputs(self): """ :rtype: flytekit.models.literals.LiteralMap @@ -583,6 +656,9 @@ def get_outputs(self): ) return _literals.LiteralMap({}) + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def sync(self): """ :rtype: None @@ -590,6 +666,9 @@ def sync(self): client = _FlyteClientManager(_platform_config.URL.get(), insecure=_platform_config.INSECURE.get()).client self.sdk_task_execution._closure = client.get_task_execution(self.sdk_task_execution.id).closure + @_deprecated( + reason="Objects should access client directly, will be removed by 1.0", version="0.13.0", + ) def get_child_executions(self, filters=None): """ :param list[flytekit.models.filters.Filter] filters: diff --git a/flytekit/engines/loader.py b/flytekit/engines/loader.py index 9471e55ce1..336d5b9de6 100644 --- a/flytekit/engines/loader.py +++ b/flytekit/engines/loader.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import importlib as _importlib from flytekit.common.exceptions import scopes as _exception_scopes diff --git a/flytekit/engines/unit/engine.py b/flytekit/engines/unit/engine.py index 95a2f17b1e..8ec3fdbc8c 100644 --- a/flytekit/engines/unit/engine.py +++ b/flytekit/engines/unit/engine.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import logging as _logging import os as _os from datetime import datetime as _datetime diff --git a/flytekit/engines/unit/mock_stats.py b/flytekit/engines/unit/mock_stats.py index 2239156080..e6fe78fe8f 100644 --- a/flytekit/engines/unit/mock_stats.py +++ b/flytekit/engines/unit/mock_stats.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime import logging diff --git a/flytekit/interfaces/data/common.py b/flytekit/interfaces/data/common.py index 0491c1476f..cae090a8eb 100644 --- a/flytekit/interfaces/data/common.py +++ b/flytekit/interfaces/data/common.py @@ -1,11 +1,7 @@ -from __future__ import absolute_import - import abc as _abc -import six as _six - -class DataProxy(_six.with_metaclass(_abc.ABCMeta, object)): +class DataProxy(object, metaclass=_abc.ABCMeta): def exists(self, path): """ :param path: diff --git a/flytekit/interfaces/data/data_proxy.py b/flytekit/interfaces/data/data_proxy.py index 9a24135204..c9d59e8553 100644 --- a/flytekit/interfaces/data/data_proxy.py +++ b/flytekit/interfaces/data/data_proxy.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import six as _six from flytekit.common import constants as _constants diff --git a/flytekit/interfaces/data/gcs/gcs_proxy.py b/flytekit/interfaces/data/gcs/gcs_proxy.py index 3b40e664d3..4f79c02c65 100644 --- a/flytekit/interfaces/data/gcs/gcs_proxy.py +++ b/flytekit/interfaces/data/gcs/gcs_proxy.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os as _os import sys as _sys import uuid as _uuid diff --git a/flytekit/interfaces/data/http/http_data_proxy.py b/flytekit/interfaces/data/http/http_data_proxy.py index bc648c4511..0b07d36a6b 100644 --- a/flytekit/interfaces/data/http/http_data_proxy.py +++ b/flytekit/interfaces/data/http/http_data_proxy.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import requests as _requests from flytekit.common.exceptions import user as _user_exceptions diff --git a/flytekit/interfaces/data/local/local_file_proxy.py b/flytekit/interfaces/data/local/local_file_proxy.py index 77c70503a0..ff3c572368 100644 --- a/flytekit/interfaces/data/local/local_file_proxy.py +++ b/flytekit/interfaces/data/local/local_file_proxy.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os as _os import uuid as _uuid from distutils import dir_util as _dir_util diff --git a/flytekit/interfaces/data/s3/s3proxy.py b/flytekit/interfaces/data/s3/s3proxy.py index 2ed864b7d4..e259b99e3f 100644 --- a/flytekit/interfaces/data/s3/s3proxy.py +++ b/flytekit/interfaces/data/s3/s3proxy.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import logging import os as _os import re as _re diff --git a/flytekit/interfaces/random.py b/flytekit/interfaces/random.py index 8a112c95f8..aac57b2fb2 100644 --- a/flytekit/interfaces/random.py +++ b/flytekit/interfaces/random.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import random as _random random = _random.Random() diff --git a/flytekit/interfaces/stats/client.py b/flytekit/interfaces/stats/client.py index 3dc3086101..759979670a 100644 --- a/flytekit/interfaces/stats/client.py +++ b/flytekit/interfaces/stats/client.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- # -from __future__ import absolute_import, division, print_function, unicode_literals - import re import sys diff --git a/flytekit/interfaces/stats/taggable.py b/flytekit/interfaces/stats/taggable.py index 12a78488dc..32bc6f3ebf 100644 --- a/flytekit/interfaces/stats/taggable.py +++ b/flytekit/interfaces/stats/taggable.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.interfaces.stats import client as _stats_client diff --git a/flytekit/models/admin/common.py b/flytekit/models/admin/common.py index 0a0cdde0c1..3baba54e24 100644 --- a/flytekit/models/admin/common.py +++ b/flytekit/models/admin/common.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.admin import common_pb2 as _common_pb2 from flytekit.models import common as _common diff --git a/flytekit/models/admin/task_execution.py b/flytekit/models/admin/task_execution.py index 8100e9343f..41d2e85c69 100644 --- a/flytekit/models/admin/task_execution.py +++ b/flytekit/models/admin/task_execution.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.admin import task_execution_pb2 as _task_execution_pb2 from flytekit.models import common as _common diff --git a/flytekit/models/admin/workflow.py b/flytekit/models/admin/workflow.py index 8a5f87e62a..ce05f430ef 100644 --- a/flytekit/models/admin/workflow.py +++ b/flytekit/models/admin/workflow.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.admin import workflow_pb2 as _admin_workflow from flytekit.models import common as _common diff --git a/flytekit/models/array_job.py b/flytekit/models/array_job.py index a420f289f2..99214655e9 100644 --- a/flytekit/models/array_job.py +++ b/flytekit/models/array_job.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import json as _json from flyteidl.plugins import array_job_pb2 as _array_job diff --git a/flytekit/models/common.py b/flytekit/models/common.py index e1a4808e58..7d2fd96b6e 100644 --- a/flytekit/models/common.py +++ b/flytekit/models/common.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import abc as _abc import json as _json @@ -40,7 +38,7 @@ def from_flyte_idl(cls, idl_object): pass -class FlyteIdlEntity(_six.with_metaclass(FlyteType, object)): +class FlyteIdlEntity(object, metaclass=FlyteType): def __eq__(self, other): return isinstance(other, FlyteIdlEntity) and other.to_flyte_idl() == self.to_flyte_idl() diff --git a/flytekit/models/core/compiler.py b/flytekit/models/core/compiler.py index f1e56ff7d8..6b6e03003c 100644 --- a/flytekit/models/core/compiler.py +++ b/flytekit/models/core/compiler.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import six as _six from flyteidl.core import compiler_pb2 as _compiler_pb2 diff --git a/flytekit/models/core/condition.py b/flytekit/models/core/condition.py index d51e258113..54b99e6b21 100644 --- a/flytekit/models/core/condition.py +++ b/flytekit/models/core/condition.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.core import condition_pb2 as _condition from flytekit.models import common as _common diff --git a/flytekit/models/core/errors.py b/flytekit/models/core/errors.py index d109d8382e..0a1196caec 100644 --- a/flytekit/models/core/errors.py +++ b/flytekit/models/core/errors.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.core import errors_pb2 as _errors_pb2 from flytekit.models import common as _common diff --git a/flytekit/models/core/execution.py b/flytekit/models/core/execution.py index d16a649b5d..2c20edc48b 100644 --- a/flytekit/models/core/execution.py +++ b/flytekit/models/core/execution.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.core import execution_pb2 as _execution_pb2 from flytekit.models import common as _common diff --git a/flytekit/models/core/identifier.py b/flytekit/models/core/identifier.py index 1b0b0d4e17..17e27644e6 100644 --- a/flytekit/models/core/identifier.py +++ b/flytekit/models/core/identifier.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.core import identifier_pb2 as _identifier_pb2 from flytekit.models import common as _common_models diff --git a/flytekit/models/core/types.py b/flytekit/models/core/types.py index fb0ec4090e..97bb53cbe7 100644 --- a/flytekit/models/core/types.py +++ b/flytekit/models/core/types.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.core import types_pb2 as _types_pb2 from flytekit.models import common as _common diff --git a/flytekit/models/core/workflow.py b/flytekit/models/core/workflow.py index 2425cb1fbf..97efab24cc 100644 --- a/flytekit/models/core/workflow.py +++ b/flytekit/models/core/workflow.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.core import workflow_pb2 as _core_workflow from flytekit.models import common as _common @@ -40,7 +38,7 @@ def to_flyte_idl(self): """ :rtype: flyteidl.core.workflow_pb2.IfBlock """ - return _core_workflow.IfBlock(condition=self.condition.to_flyte_idl(), then_node=self.then_node.to_flyte_idl(),) + return _core_workflow.IfBlock(condition=self.condition.to_flyte_idl(), then_node=self.then_node.to_flyte_idl()) @classmethod def from_flyte_idl(cls, pb2_object): @@ -248,6 +246,7 @@ def __init__( self._metadata = metadata self._inputs = inputs self._upstream_node_ids = upstream_node_ids + # TODO: For proper graph handling, we need to keep track of the node objects themselves, not just the node IDs self._output_aliases = output_aliases self._task_node = task_node self._workflow_node = workflow_node @@ -282,7 +281,7 @@ def inputs(self): @property def upstream_node_ids(self): """ - [Optional] Specifies execution depdendency for this node ensuring it will + [Optional] Specifies execution dependency for this node ensuring it will only get scheduled to run after all its upstream nodes have completed. This node will have an implicit dependency on any node that appears in inputs field. :rtype: list[Text] @@ -481,7 +480,6 @@ class OnFailurePolicy(object): def __init__(self, on_failure=None): """ Metadata for the workflow. - :param on_failure flytekit.models.core.workflow.WorkflowMetadata.OnFailurePolicy: [Optional] The execution policy when the workflow detects a failure. """ self._on_failure = on_failure diff --git a/flytekit/models/dynamic_job.py b/flytekit/models/dynamic_job.py index 7018e3b664..44a985a5e7 100644 --- a/flytekit/models/dynamic_job.py +++ b/flytekit/models/dynamic_job.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.core import dynamic_job_pb2 as _dynamic_job from flytekit.models import common as _common diff --git a/flytekit/models/execution.py b/flytekit/models/execution.py index 7fc6b8d9ec..977495bae6 100644 --- a/flytekit/models/execution.py +++ b/flytekit/models/execution.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import flyteidl.admin.execution_pb2 as _execution_pb2 import flyteidl.admin.node_execution_pb2 as _node_execution_pb2 import flyteidl.admin.task_execution_pb2 as _task_execution_pb2 diff --git a/flytekit/models/filters.py b/flytekit/models/filters.py index 749671406d..be5ef75e27 100644 --- a/flytekit/models/filters.py +++ b/flytekit/models/filters.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.models.common import FlyteIdlEntity as _FlyteIdlEntity diff --git a/flytekit/models/interface.py b/flytekit/models/interface.py index 3e2cab71c1..d7078a7206 100644 --- a/flytekit/models/interface.py +++ b/flytekit/models/interface.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import six as _six from flyteidl.core import interface_pb2 as _interface_pb2 diff --git a/flytekit/models/launch_plan.py b/flytekit/models/launch_plan.py index 9651f753f7..f523bddcbb 100644 --- a/flytekit/models/launch_plan.py +++ b/flytekit/models/launch_plan.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.admin import launch_plan_pb2 as _launch_plan from flytekit.models import common as _common diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py index 2cf8f4008a..0961f80108 100644 --- a/flytekit/models/literals.py +++ b/flytekit/models/literals.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from datetime import datetime as _datetime import pytz as _pytz diff --git a/flytekit/models/node_execution.py b/flytekit/models/node_execution.py index f47d71b45d..b0c103891b 100644 --- a/flytekit/models/node_execution.py +++ b/flytekit/models/node_execution.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import flyteidl.admin.node_execution_pb2 as _node_execution_pb2 import pytz as _pytz diff --git a/flytekit/models/presto.py b/flytekit/models/presto.py index 13edc42b9f..04c4b22b41 100644 --- a/flytekit/models/presto.py +++ b/flytekit/models/presto.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - ## Todo - change this to qubole_presto once Luis's PR get's merged # from flyteidl.plugins import qubole_presto as _qubole from flyteidl.plugins import presto_pb2 as _presto diff --git a/flytekit/models/project.py b/flytekit/models/project.py index 80d80e35b8..0410cb9f6f 100644 --- a/flytekit/models/project.py +++ b/flytekit/models/project.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.admin import project_pb2 as _project_pb2 from flytekit.models import common as _common diff --git a/flytekit/models/qubole.py b/flytekit/models/qubole.py index 1e421ec89e..3464158ad3 100644 --- a/flytekit/models/qubole.py +++ b/flytekit/models/qubole.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.plugins import qubole_pb2 as _qubole from flytekit.models import common as _common diff --git a/flytekit/models/schedule.py b/flytekit/models/schedule.py index 91cb811824..f49fbda180 100644 --- a/flytekit/models/schedule.py +++ b/flytekit/models/schedule.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.admin import schedule_pb2 as _schedule_pb2 from flytekit.models import common as _common diff --git a/flytekit/models/task.py b/flytekit/models/task.py index 662139c50b..be9fcdc2f4 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import json as _json import six as _six @@ -702,8 +700,7 @@ def to_flyte_idl(self) -> _core_task.DataLoadingConfig: ) @classmethod - def from_flyte_idl(cls, pb2: _core_task.DataLoadingConfig): - # TODO use python 3.7+ only and then https://stackoverflow.com/questions/33533148/how-do-i-specify-that-the-return-type-of-a-method-is-the-same-as-the-class-itsel -> DataLoadingConfig: + def from_flyte_idl(cls, pb2: _core_task.DataLoadingConfig) -> "DataLoadingConfig": if pb2 is None: return None return cls( diff --git a/flytekit/models/types.py b/flytekit/models/types.py index 1414c8ed1c..89ad0ef6f8 100644 --- a/flytekit/models/types.py +++ b/flytekit/models/types.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import json as _json from flyteidl.core import types_pb2 as _types_pb2 diff --git a/flytekit/models/workflow_closure.py b/flytekit/models/workflow_closure.py index ffbfa7b3dc..412a52e958 100644 --- a/flytekit/models/workflow_closure.py +++ b/flytekit/models/workflow_closure.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.core import workflow_closure_pb2 as _workflow_closure_pb2 from flytekit.models import common as _common diff --git a/flytekit/plugins/__init__.py b/flytekit/plugins/__init__.py index 0a87cc37a7..0161fcfbbc 100644 --- a/flytekit/plugins/__init__.py +++ b/flytekit/plugins/__init__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.tools import lazy_loader as _lazy_loader pyspark = _lazy_loader.lazy_load_module("pyspark") # type: _lazy_loader._LazyLoadModule diff --git a/flytekit/sdk/exceptions.py b/flytekit/sdk/exceptions.py index 6753d4182e..89874a5518 100644 --- a/flytekit/sdk/exceptions.py +++ b/flytekit/sdk/exceptions.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.exceptions import user as _user diff --git a/flytekit/sdk/tasks.py b/flytekit/sdk/tasks.py index 00d06135d1..26bdb65cd2 100644 --- a/flytekit/sdk/tasks.py +++ b/flytekit/sdk/tasks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime import six as _six @@ -1044,9 +1042,7 @@ def dynamic_sidecar_task( can be defined in the PodSpec by defining a container whose name matches the primary_container_name. These container attributes will be applied to the container brought up to execute the primary task definition. - .. code-block:: python - def generate_pod_spec_for_task(): pod_spec = generated_pb2.PodSpec() secondary_container = generated_pb2.Container( @@ -1060,10 +1056,8 @@ def generate_pod_spec_for_task(): mountPath="/data", ) secondary_container.volumeMounts.extend([shared_volume_mount]) - primary_container = generated_pb2.Container(name="primary") primary_container.volumeMounts.extend([shared_volume_mount]) - pod_spec.volumes.extend([generated_pb2.Volume( name="shared-data", volumeSource=generated_pb2.VolumeSource( @@ -1074,12 +1068,10 @@ def generate_pod_spec_for_task(): )]) pod_spec.containers.extend([primary_container, secondary_container]) return pod_spec - @outputs(out=Types.Integer) @python_task def my_sub_task(wf_params, out): out.set(randint()) - @outputs(out=[Types.Integer]) @dynamic_sidecar_task( pod_spec=generate_pod_spec_for_task(), @@ -1090,45 +1082,32 @@ def my_task(wf_params, out): for i in xrange(100): out_list.append(my_sub_task().outputs.out) out.set(out_list) - .. note:: - All outputs of a batch task must be a list. This is because the individual outputs of sub-tasks should be appended into a list. There cannot be aggregation of outputs done in this task. To accomplish aggregation, it is recommended that a python_task take the outputs of this task as input and do the necessary work. If a sub-task does not contribute an output, it must be yielded from the task with the `yield` keyword or returned from the task in a list. If this isn't done, the sub-task will not be executed. - :param _task_function: this is the decorated method and shouldn't be declared explicitly. The function must take a first argument, and then named arguments matching those defined in @inputs and @outputs. No keyword arguments are allowed. :param Text cache_version: [optional] string representing logical version for discovery. This field should be updated whenever the underlying algorithm changes. - .. note:: - This argument is required to be a non-empty string if `cache` is True. - :param int retries: [optional] integer determining number of times task can be retried on :py:exc:`flytekit.sdk.exceptions.RecoverableException` or transient platform failures. Defaults to 0. - .. note:: - If retries > 0, the task must be able to recover from any remote state created within the user code. It is strongly recommended that tasks are written to be idempotent. - :param bool interruptible: [optional] boolean describing if the task is interruptible. - :param Text deprecated: [optional] string that should be provided if this task is deprecated. The string will be logged as a warning so it should contain information regarding how to update to a newer task. :param Text storage_request: [optional] Kubernetes resource string for lower-bound of disk storage space for the task to run. Default is set by platform-level configuration. - .. note:: - This is currently not supported by the platform. - :param Text cpu_request: [optional] Kubernetes resource string for lower-bound of cores for the task to execute. This can be set to a fractional portion of a CPU. Default is set by platform-level configuration. TODO: Add links to resource string documentation for Kubernetes @@ -1140,11 +1119,8 @@ def my_task(wf_params, out): TODO: Add links to resource string documentation for Kubernetes :param Text storage_limit: [optional] Kubernetes resource string for upper-bound of disk storage space for the task to run. This amount is not guaranteed! If not specified, it is set equal to storage_request. - .. note:: - This is currently not supported by the platform. - :param Text cpu_limit: [optional] Kubernetes resource string for upper-bound of cores for the task to execute. This can be set to a fractional portion of a CPU. This amount is not guaranteed! If not specified, it is set equal to cpu_request. diff --git a/flytekit/sdk/test_utils.py b/flytekit/sdk/test_utils.py index c31a6c3933..766774bf2c 100644 --- a/flytekit/sdk/test_utils.py +++ b/flytekit/sdk/test_utils.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from wrapt import decorator as _decorator from flytekit.common import utils as _utils diff --git a/flytekit/sdk/types.py b/flytekit/sdk/types.py index bcc158bd90..924fc54733 100644 --- a/flytekit/sdk/types.py +++ b/flytekit/sdk/types.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.types import blobs as _blobs from flytekit.common.types import containers as _containers from flytekit.common.types import helpers as _helpers diff --git a/flytekit/sdk/workflow.py b/flytekit/sdk/workflow.py index 48287d3f2a..d4cf8aedea 100644 --- a/flytekit/sdk/workflow.py +++ b/flytekit/sdk/workflow.py @@ -1,7 +1,9 @@ -from __future__ import absolute_import +from typing import Dict import six as _six +import flytekit.common.local_workflow +from flytekit.common import nodes as _nodes from flytekit.common import promise as _promise from flytekit.common import workflow as _common_workflow from flytekit.common.types import helpers as _type_helpers @@ -24,7 +26,7 @@ def __init__(self, sdk_type, help=None, **kwargs): super(Input, self).__init__("", _type_helpers.python_std_to_sdk_type(sdk_type), help=help, **kwargs) -class Output(_common_workflow.Output): +class Output(flytekit.common.local_workflow.Output): """ This object should be used to specify outputs. It can be used in conjunction with :py:meth:`flytekit.common.workflow.workflow` and :py:meth:`flytekit.common.workflow.workflow_class` @@ -59,18 +61,18 @@ class MyWorkflow(object): :param T _workflow_metaclass: Do NOT specify this parameter directly. This is the class that is being wrapped by this decorator. - :param cls: This is the class that will be instantiated from the inputs, outputs, and nodes. This will be used - by users extending the base Flyte programming model. If set, it must be a subclass of - :py:class:`flytekit.common.workflow.SdkWorkflow`. :param flytekit.models.core.workflow.WorkflowMetadata.OnFailurePolicy on_failure: [Optional] The execution policy when the workflow detects a failure. :param bool disable_default_launch_plan: Determines whether to create a default launch plan for the workflow or not. + :param cls: This is the class that will be instantiated from the inputs, outputs, and nodes. This will be used + by users extending the base Flyte programming model. If set, it must be a subclass of + :py:class:`flytekit.common.local_workflow.PythonWorkflow`. :rtype: flytekit.common.workflow.SdkWorkflow """ def wrapper(metaclass): - wf = _common_workflow.build_sdk_workflow_from_metaclass( + wf = flytekit.common.local_workflow.build_sdk_workflow_from_metaclass( metaclass, on_failure=on_failure, disable_default_launch_plan=disable_default_launch_plan, cls=cls ) return wf @@ -80,7 +82,7 @@ def wrapper(metaclass): return wrapper -def workflow(nodes, inputs=None, outputs=None, cls=None, on_failure=None): +def workflow(nodes: Dict[str, _nodes.SdkNode], inputs=None, outputs=None, cls=None, on_failure=None): """ This function provides a user-friendly interface for authoring workflows. @@ -112,11 +114,13 @@ def workflow(nodes, inputs=None, outputs=None, cls=None, on_failure=None): :param dict[Text,Output] outputs: [Optional] A dictionary of output descriptors for a workflow. :param T cls: This is the class that will be instantiated from the inputs, outputs, and nodes. This will be used by users extending the base Flyte programming model. If set, it must be a subclass of - :py:class:`flytekit.common.workflow.SdkWorkflow`. + :py:class:`flytekit.common.local_workflow.PythonWorkflow`. :param flytekit.models.core.workflow.WorkflowMetadata.OnFailurePolicy on_failure: [Optional] The execution policy when the workflow detects a failure. - :rtype: flytekit.common.workflow.SdkWorkflow + + :rtype: flytekit.common.local_workflow.SdkRunnableWorkflow """ - wf = (cls or _common_workflow.SdkWorkflow)( + # TODO: Why does Pycharm complain about nodes? + wf = (cls or flytekit.common.local_workflow.SdkRunnableWorkflow).construct_from_class_definition( inputs=[v.rename_and_return_reference(k) for k, v in sorted(_six.iteritems(inputs or {}))], outputs=[v.rename_and_return_reference(k) for k, v in sorted(_six.iteritems(outputs or {}))], nodes=[v.assign_id_and_return(k) for k, v in sorted(_six.iteritems(nodes))], diff --git a/flytekit/tools/lazy_loader.py b/flytekit/tools/lazy_loader.py index f15b5cf78e..fa6f797943 100644 --- a/flytekit/tools/lazy_loader.py +++ b/flytekit/tools/lazy_loader.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import importlib as _importlib import sys as _sys import types as _types diff --git a/flytekit/tools/module_loader.py b/flytekit/tools/module_loader.py index d852bf57b7..2a522eaa21 100644 --- a/flytekit/tools/module_loader.py +++ b/flytekit/tools/module_loader.py @@ -1,13 +1,11 @@ -from __future__ import absolute_import - import importlib import pkgutil import six from flytekit.common.exceptions import user as _user_exceptions +from flytekit.common.local_workflow import SdkRunnableWorkflow as _SdkRunnableWorkflow from flytekit.common.mixins import registerable as _registerable -from flytekit.common.workflow import SdkWorkflow as _SdkWorkflow def iterate_modules(pkgs): @@ -49,19 +47,21 @@ def _topo_sort_helper( ) recursion_set[obj] = len(recursion_stack) - 1 - for upstream in obj.upstream_entities: - if upstream not in visited: - for m1, k1, o1 in _topo_sort_helper( - upstream, - entity_to_module_key, - visited, - recursion_set, - recursion_stack, - include_entities, - ignore_entities, - detect_unreferenced_entities, - ): - yield m1, k1, o1 + if isinstance(obj, _registerable.HasDependencies): + for upstream in obj.upstream_entities: + if upstream not in visited: + for m1, k1, o1 in _topo_sort_helper( + upstream, + entity_to_module_key, + visited, + recursion_set, + recursion_stack, + include_entities, + ignore_entities, + detect_unreferenced_entities, + ): + if not o1.has_registered: + yield m1, k1, o1 recursion_stack.pop() del recursion_set[obj] @@ -110,7 +110,7 @@ def iterate_registerable_entities_in_order( if isinstance(o, _registerable.RegisterableEntity): if o.instantiated_in == m.__name__: entity_to_module_key[o] = (m, k) - if isinstance(o, _SdkWorkflow) and o.should_create_default_launch_plan: + if isinstance(o, _SdkRunnableWorkflow) and o.should_create_default_launch_plan: # SDK should create a default launch plan for a workflow. This is a special-case to simplify # authoring of workflows. entity_to_module_key[o.create_launch_plan()] = (m, k) diff --git a/flytekit/tools/subprocess.py b/flytekit/tools/subprocess.py index a437b61a1a..01ed6e2bd2 100644 --- a/flytekit/tools/subprocess.py +++ b/flytekit/tools/subprocess.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - import logging import shlex as _schlex import subprocess as _subprocess diff --git a/flytekit/type_engines/common.py b/flytekit/type_engines/common.py index 5ab9dfca0e..33927325ce 100644 --- a/flytekit/type_engines/common.py +++ b/flytekit/type_engines/common.py @@ -1,11 +1,7 @@ -from __future__ import absolute_import - import abc as _abc -import six as _six - -class TypeEngine(_six.with_metaclass(_abc.ABCMeta, object)): +class TypeEngine(object, metaclass=_abc.ABCMeta): @_abc.abstractmethod def python_std_to_sdk_type(self, t): """ diff --git a/flytekit/type_engines/default/flyte.py b/flytekit/type_engines/default/flyte.py index b7c329d8a3..4b58019003 100644 --- a/flytekit/type_engines/default/flyte.py +++ b/flytekit/type_engines/default/flyte.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import importlib as _importer from flytekit.common.exceptions import system as _system_exceptions diff --git a/pyproject.toml b/pyproject.toml index 55ec8d784c..339d113e41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,16 @@ [tool.black] line-length = 120 +exclude = ''' +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | _build + | build + | dist + )/ +) +''' diff --git a/setup.cfg b/setup.cfg index c19835b199..af302b7fb3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,7 +9,7 @@ line_length = 120 [flake8] max-line-length = 120 extend-ignore = E203, E266, E501, W503, E741 -exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,venv/*,src/*,tests/unit/common/protos/* +exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,venv/*,src/*,tests/unit/common/protos/*,build max-complexity=16 [tool:pytest] diff --git a/setup.py b/setup.py index daaa5b6fbe..2512403b74 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from setuptools import find_packages, setup # noqa import flytekit # noqa @@ -62,7 +60,7 @@ "scripts/flytekit_sagemaker_runner.py", ], license="apache2", - python_requires=">=2.7", + python_requires=">=3.6", classifiers=[ "Intended Audience :: Developers", "Intended Audience :: System Administrators", diff --git a/tests/flytekit/common/parameterizers.py b/tests/flytekit/common/parameterizers.py index 13d6704e96..e08057d54d 100644 --- a/tests/flytekit/common/parameterizers.py +++ b/tests/flytekit/common/parameterizers.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from datetime import timedelta from itertools import product diff --git a/tests/flytekit/common/task_definitions.py b/tests/flytekit/common/task_definitions.py index 0381b203d1..56534fd7ea 100644 --- a/tests/flytekit/common/task_definitions.py +++ b/tests/flytekit/common/task_definitions.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.sdk.tasks import inputs, outputs, python_task from flytekit.sdk.types import Types diff --git a/tests/flytekit/common/workflows/batch.py b/tests/flytekit/common/workflows/batch.py index ed460adc92..d3e095dd82 100644 --- a/tests/flytekit/common/workflows/batch.py +++ b/tests/flytekit/common/workflows/batch.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - from six import moves as _six_moves from flytekit.sdk.tasks import dynamic_task, inputs, outputs, python_task diff --git a/tests/flytekit/common/workflows/dynamic_workflows.py b/tests/flytekit/common/workflows/dynamic_workflows.py index 97f5e81343..fabab6367b 100644 --- a/tests/flytekit/common/workflows/dynamic_workflows.py +++ b/tests/flytekit/common/workflows/dynamic_workflows.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - from flytekit.sdk import tasks as _tasks from flytekit.sdk import workflow as _workflow from flytekit.sdk.types import Types as _Types diff --git a/tests/flytekit/common/workflows/failing_workflows.py b/tests/flytekit/common/workflows/failing_workflows.py index 50d8aa0c63..260e460b17 100644 --- a/tests/flytekit/common/workflows/failing_workflows.py +++ b/tests/flytekit/common/workflows/failing_workflows.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - from flytekit.models.core.workflow import WorkflowMetadata from flytekit.sdk.tasks import python_task from flytekit.sdk.workflow import workflow_class diff --git a/tests/flytekit/common/workflows/gpu.py b/tests/flytekit/common/workflows/gpu.py index a1c432211f..7e7621ac69 100644 --- a/tests/flytekit/common/workflows/gpu.py +++ b/tests/flytekit/common/workflows/gpu.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - from flytekit.sdk.tasks import inputs, outputs, python_task from flytekit.sdk.types import Types from flytekit.sdk.workflow import Input, Output, workflow_class diff --git a/tests/flytekit/common/workflows/hive.py b/tests/flytekit/common/workflows/hive.py index 05f640467c..b3cb56fe5d 100644 --- a/tests/flytekit/common/workflows/hive.py +++ b/tests/flytekit/common/workflows/hive.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - import six as _six from flytekit.sdk.tasks import inputs, outputs, python_task, qubole_hive_task diff --git a/tests/flytekit/common/workflows/nested.py b/tests/flytekit/common/workflows/nested.py index 99a5ff89a9..a364356ff6 100644 --- a/tests/flytekit/common/workflows/nested.py +++ b/tests/flytekit/common/workflows/nested.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - from flytekit.sdk.tasks import inputs, outputs, python_task from flytekit.sdk.types import Types from flytekit.sdk.workflow import Input, Output, workflow_class diff --git a/tests/flytekit/common/workflows/notebook.py b/tests/flytekit/common/workflows/notebook.py index bd61c6a94f..e8af3b7467 100644 --- a/tests/flytekit/common/workflows/notebook.py +++ b/tests/flytekit/common/workflows/notebook.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - from flytekit.contrib.notebook.tasks import python_notebook, spark_notebook from flytekit.sdk.tasks import inputs, outputs from flytekit.sdk.types import Types diff --git a/tests/flytekit/common/workflows/notifications.py b/tests/flytekit/common/workflows/notifications.py index 79031753e6..d09bc14fad 100644 --- a/tests/flytekit/common/workflows/notifications.py +++ b/tests/flytekit/common/workflows/notifications.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - from flytekit.common import notifications as _notifications from flytekit.models.core import execution as _execution from flytekit.sdk.tasks import inputs, outputs, python_task diff --git a/tests/flytekit/common/workflows/presto.py b/tests/flytekit/common/workflows/presto.py index 317df496c7..e681a3e953 100644 --- a/tests/flytekit/common/workflows/presto.py +++ b/tests/flytekit/common/workflows/presto.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.tasks.presto_task import SdkPrestoTask from flytekit.sdk.tasks import inputs from flytekit.sdk.types import Types diff --git a/tests/flytekit/common/workflows/python.py b/tests/flytekit/common/workflows/python.py index d9cd9a9c7d..ff8f41f29d 100644 --- a/tests/flytekit/common/workflows/python.py +++ b/tests/flytekit/common/workflows/python.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - from flytekit.sdk.tasks import inputs, outputs, python_task from flytekit.sdk.types import Types from flytekit.sdk.workflow import Input, workflow_class diff --git a/tests/flytekit/common/workflows/raw_container.py b/tests/flytekit/common/workflows/raw_container.py index ed3f786174..60edbcc737 100644 --- a/tests/flytekit/common/workflows/raw_container.py +++ b/tests/flytekit/common/workflows/raw_container.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - from flytekit.common.tasks.raw_container import SdkRawContainerTask from flytekit.sdk.types import Types from flytekit.sdk.workflow import Input, Output, workflow_class diff --git a/tests/flytekit/common/workflows/scala_spark.py b/tests/flytekit/common/workflows/scala_spark.py index 37a6abe0c7..7df3b0fb32 100644 --- a/tests/flytekit/common/workflows/scala_spark.py +++ b/tests/flytekit/common/workflows/scala_spark.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - from flytekit.sdk.spark_types import SparkType from flytekit.sdk.tasks import generic_spark_task, inputs, python_task from flytekit.sdk.types import Types diff --git a/tests/flytekit/common/workflows/sidecar.py b/tests/flytekit/common/workflows/sidecar.py index 9343b29755..aa01094cf6 100644 --- a/tests/flytekit/common/workflows/sidecar.py +++ b/tests/flytekit/common/workflows/sidecar.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - import os import time diff --git a/tests/flytekit/common/workflows/simple.py b/tests/flytekit/common/workflows/simple.py index 1914bcf7bf..2e21a7f300 100644 --- a/tests/flytekit/common/workflows/simple.py +++ b/tests/flytekit/common/workflows/simple.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - import pandas as _pd from flytekit.sdk.tasks import inputs, outputs, python_task diff --git a/tests/flytekit/common/workflows/spark.py b/tests/flytekit/common/workflows/spark.py index 2754395d25..b3f381f6ba 100644 --- a/tests/flytekit/common/workflows/spark.py +++ b/tests/flytekit/common/workflows/spark.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import random from operator import add diff --git a/tests/flytekit/loadtests/cp_orchestrator.py b/tests/flytekit/loadtests/cp_orchestrator.py index e3b99a7d0b..776afecd59 100644 --- a/tests/flytekit/loadtests/cp_orchestrator.py +++ b/tests/flytekit/loadtests/cp_orchestrator.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - from six.moves import range from flytekit.sdk.workflow import workflow_class diff --git a/tests/flytekit/loadtests/cp_python.py b/tests/flytekit/loadtests/cp_python.py index b3a0efd24b..f26cbb51f7 100644 --- a/tests/flytekit/loadtests/cp_python.py +++ b/tests/flytekit/loadtests/cp_python.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import time from six.moves import range diff --git a/tests/flytekit/loadtests/cp_spark.py b/tests/flytekit/loadtests/cp_spark.py index b513c028f8..b79fbc3509 100644 --- a/tests/flytekit/loadtests/cp_spark.py +++ b/tests/flytekit/loadtests/cp_spark.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import random from operator import add diff --git a/tests/flytekit/loadtests/dynamic_job.py b/tests/flytekit/loadtests/dynamic_job.py index 8d5fc4f06e..520ff401a7 100644 --- a/tests/flytekit/loadtests/dynamic_job.py +++ b/tests/flytekit/loadtests/dynamic_job.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import time from six.moves import range diff --git a/tests/flytekit/loadtests/orchestrator.py b/tests/flytekit/loadtests/orchestrator.py index c976171ef3..5981502319 100644 --- a/tests/flytekit/loadtests/orchestrator.py +++ b/tests/flytekit/loadtests/orchestrator.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - from six.moves import range from flytekit.sdk.types import Types diff --git a/tests/flytekit/loadtests/python.py b/tests/flytekit/loadtests/python.py index 332b9846f5..e6da8df722 100644 --- a/tests/flytekit/loadtests/python.py +++ b/tests/flytekit/loadtests/python.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, division, print_function - import time from six.moves import range diff --git a/tests/flytekit/unit/bin/test_python_entrypoint.py b/tests/flytekit/unit/bin/test_python_entrypoint.py index 1060a3658e..3a08de20b4 100644 --- a/tests/flytekit/unit/bin/test_python_entrypoint.py +++ b/tests/flytekit/unit/bin/test_python_entrypoint.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os import mock diff --git a/tests/flytekit/unit/cli/auth/test_auth.py b/tests/flytekit/unit/cli/auth/test_auth.py index a3fba13d7b..2deecaefaa 100644 --- a/tests/flytekit/unit/cli/auth/test_auth.py +++ b/tests/flytekit/unit/cli/auth/test_auth.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import re from multiprocessing import Queue as _Queue diff --git a/tests/flytekit/unit/cli/auth/test_credentials.py b/tests/flytekit/unit/cli/auth/test_credentials.py index 7a5805ee44..f1ae57a016 100644 --- a/tests/flytekit/unit/cli/auth/test_credentials.py +++ b/tests/flytekit/unit/cli/auth/test_credentials.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.clis.auth import credentials as _credentials diff --git a/tests/flytekit/unit/cli/pyflyte/conftest.py b/tests/flytekit/unit/cli/pyflyte/conftest.py index 4dc1b10d12..9036367cff 100644 --- a/tests/flytekit/unit/cli/pyflyte/conftest.py +++ b/tests/flytekit/unit/cli/pyflyte/conftest.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os import sys diff --git a/tests/flytekit/unit/cli/pyflyte/test_basic_auth.py b/tests/flytekit/unit/cli/pyflyte/test_basic_auth.py index 421713ee16..d18f21dfa5 100644 --- a/tests/flytekit/unit/cli/pyflyte/test_basic_auth.py +++ b/tests/flytekit/unit/cli/pyflyte/test_basic_auth.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import json from mock import MagicMock, patch diff --git a/tests/flytekit/unit/cli/pyflyte/test_launch_plans.py b/tests/flytekit/unit/cli/pyflyte/test_launch_plans.py index 6669ce350c..e19fea82a0 100644 --- a/tests/flytekit/unit/cli/pyflyte/test_launch_plans.py +++ b/tests/flytekit/unit/cli/pyflyte/test_launch_plans.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.clis.sdk_in_container import launch_plan diff --git a/tests/flytekit/unit/cli/pyflyte/test_register.py b/tests/flytekit/unit/cli/pyflyte/test_register.py index a16fc8e59a..937d29972f 100644 --- a/tests/flytekit/unit/cli/pyflyte/test_register.py +++ b/tests/flytekit/unit/cli/pyflyte/test_register.py @@ -1,48 +1,40 @@ from mock import MagicMock -from flytekit.engines.flyte import engine +from flytekit.common.launch_plan import SdkLaunchPlan +from flytekit.common.tasks.task import SdkTask +from flytekit.common.workflow import SdkWorkflow def test_register_workflows(mock_clirunner, monkeypatch): - mock_get_task = MagicMock() - monkeypatch.setattr(engine.FlyteEngineFactory, "get_task", MagicMock(return_value=mock_get_task)) - mock_get_workflow = MagicMock() - monkeypatch.setattr( - engine.FlyteEngineFactory, "get_workflow", MagicMock(return_value=mock_get_workflow), - ) - mock_get_launch_plan = MagicMock() - monkeypatch.setattr( - engine.FlyteEngineFactory, "get_launch_plan", MagicMock(return_value=mock_get_launch_plan), - ) + + mock_register_task = MagicMock(return_value=MagicMock()) + monkeypatch.setattr(SdkTask, "register", mock_register_task) + mock_register_workflow = MagicMock(return_value=MagicMock()) + monkeypatch.setattr(SdkWorkflow, "register", mock_register_workflow) + mock_register_launch_plan = MagicMock(return_value=MagicMock()) + monkeypatch.setattr(SdkLaunchPlan, "register", mock_register_launch_plan) result = mock_clirunner("register", "workflows") assert result.exit_code == 0 - assert len(mock_get_task.mock_calls) == 4 - assert len(mock_get_task.register.mock_calls) == 4 - assert len(mock_get_workflow.mock_calls) == 1 - assert len(mock_get_workflow.register.mock_calls) == 1 - assert len(mock_get_launch_plan.mock_calls) == 1 - assert len(mock_get_launch_plan.register.mock_calls) == 1 + assert len(mock_register_task.mock_calls) == 4 + assert len(mock_register_workflow.mock_calls) == 1 + assert len(mock_register_launch_plan.mock_calls) == 1 def test_register_workflows_with_test_switch(mock_clirunner, monkeypatch): - mock_get_task = MagicMock() - monkeypatch.setattr(engine.FlyteEngineFactory, "get_task", MagicMock(return_value=mock_get_task)) - mock_get_workflow = MagicMock() - monkeypatch.setattr( - engine.FlyteEngineFactory, "get_workflow", MagicMock(return_value=mock_get_workflow), - ) - mock_get_launch_plan = MagicMock() - monkeypatch.setattr( - engine.FlyteEngineFactory, "get_launch_plan", MagicMock(return_value=mock_get_launch_plan), - ) + mock_register_task = MagicMock(return_value=MagicMock()) + monkeypatch.setattr(SdkTask, "register", mock_register_task) + mock_register_workflow = MagicMock(return_value=MagicMock()) + monkeypatch.setattr(SdkWorkflow, "register", mock_register_workflow) + mock_register_launch_plan = MagicMock(return_value=MagicMock()) + monkeypatch.setattr(SdkLaunchPlan, "register", mock_register_launch_plan) result = mock_clirunner("register", "--test", "workflows") assert result.exit_code == 0 - assert len(mock_get_task.mock_calls) == 0 - assert len(mock_get_workflow.mock_calls) == 0 - assert len(mock_get_launch_plan.mock_calls) == 0 + assert len(mock_register_task.mock_calls) == 0 + assert len(mock_register_workflow.mock_calls) == 0 + assert len(mock_register_launch_plan.mock_calls) == 0 diff --git a/tests/flytekit/unit/cli/test_flyte_cli.py b/tests/flytekit/unit/cli/test_flyte_cli.py index adcd07dff9..0da78624b1 100644 --- a/tests/flytekit/unit/cli/test_flyte_cli.py +++ b/tests/flytekit/unit/cli/test_flyte_cli.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import mock as _mock import pytest diff --git a/tests/flytekit/unit/cli/test_helpers.py b/tests/flytekit/unit/cli/test_helpers.py index 3ed7e05daf..10022778e5 100644 --- a/tests/flytekit/unit/cli/test_helpers.py +++ b/tests/flytekit/unit/cli/test_helpers.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.clis import helpers diff --git a/tests/flytekit/unit/clients/test_raw.py b/tests/flytekit/unit/clients/test_raw.py index e42ab5136f..a143866494 100644 --- a/tests/flytekit/unit/clients/test_raw.py +++ b/tests/flytekit/unit/clients/test_raw.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import json import os diff --git a/tests/flytekit/unit/common_tests/exceptions/test_base.py b/tests/flytekit/unit/common_tests/exceptions/test_base.py index 58ef24b6dd..f4ede26b74 100644 --- a/tests/flytekit/unit/common_tests/exceptions/test_base.py +++ b/tests/flytekit/unit/common_tests/exceptions/test_base.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.exceptions import base diff --git a/tests/flytekit/unit/common_tests/exceptions/test_scopes.py b/tests/flytekit/unit/common_tests/exceptions/test_scopes.py index 37ac077268..f14ced33f9 100644 --- a/tests/flytekit/unit/common_tests/exceptions/test_scopes.py +++ b/tests/flytekit/unit/common_tests/exceptions/test_scopes.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.common.exceptions import scopes, system, user diff --git a/tests/flytekit/unit/common_tests/exceptions/test_system.py b/tests/flytekit/unit/common_tests/exceptions/test_system.py index ed574eea3a..4610703efa 100644 --- a/tests/flytekit/unit/common_tests/exceptions/test_system.py +++ b/tests/flytekit/unit/common_tests/exceptions/test_system.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.exceptions import base, system diff --git a/tests/flytekit/unit/common_tests/exceptions/test_user.py b/tests/flytekit/unit/common_tests/exceptions/test_user.py index ee4d02bf29..f8851bb122 100644 --- a/tests/flytekit/unit/common_tests/exceptions/test_user.py +++ b/tests/flytekit/unit/common_tests/exceptions/test_user.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.exceptions import base, user diff --git a/tests/flytekit/unit/common_tests/mixins/sample_registerable.py b/tests/flytekit/unit/common_tests/mixins/sample_registerable.py index a2e3ede79f..b0ecb08ca9 100644 --- a/tests/flytekit/unit/common_tests/mixins/sample_registerable.py +++ b/tests/flytekit/unit/common_tests/mixins/sample_registerable.py @@ -1,12 +1,10 @@ -from __future__ import absolute_import - -import six as _six - from flytekit.common import sdk_bases as _sdk_bases from flytekit.common.mixins import registerable as _registerable -class ExampleRegisterable(_six.with_metaclass(_sdk_bases.ExtendedSdkType, _registerable.RegisterableEntity)): +class ExampleRegisterable( + _registerable.RegisterableEntity, _registerable.TrackableEntity, metaclass=_sdk_bases.ExtendedSdkType +): def __init__(self, *args, **kwargs): super(ExampleRegisterable, self).__init__(*args, **kwargs) diff --git a/tests/flytekit/unit/common_tests/mixins/test_registerable.py b/tests/flytekit/unit/common_tests/mixins/test_registerable.py index 8f0d82b19e..030d5bc1e9 100644 --- a/tests/flytekit/unit/common_tests/mixins/test_registerable.py +++ b/tests/flytekit/unit/common_tests/mixins/test_registerable.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from tests.flytekit.unit.common_tests.mixins import sample_registerable as _sample_registerable diff --git a/tests/flytekit/unit/common_tests/tasks/spark/test_spark_task.py b/tests/flytekit/unit/common_tests/tasks/spark/test_spark_task.py index 29e9e3254a..9389fdc7ea 100644 --- a/tests/flytekit/unit/common_tests/tasks/spark/test_spark_task.py +++ b/tests/flytekit/unit/common_tests/tasks/spark/test_spark_task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from six.moves import range from flytekit.sdk.tasks import outputs, spark_task diff --git a/tests/flytekit/unit/common_tests/tasks/test_sdk_runnable.py b/tests/flytekit/unit/common_tests/tasks/test_sdk_runnable.py index 7c4763046a..80d5b20421 100644 --- a/tests/flytekit/unit/common_tests/tasks/test_sdk_runnable.py +++ b/tests/flytekit/unit/common_tests/tasks/test_sdk_runnable.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest as _pytest from flytekit.common import constants as _common_constants diff --git a/tests/flytekit/unit/common_tests/tasks/test_task.py b/tests/flytekit/unit/common_tests/tasks/test_task.py index 1c6d52b9fc..9212211b0e 100644 --- a/tests/flytekit/unit/common_tests/tasks/test_task.py +++ b/tests/flytekit/unit/common_tests/tasks/test_task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os as _os import pytest as _pytest @@ -18,23 +16,27 @@ from flytekit.sdk.types import Types -@_patch("flytekit.engines.loader.get_engine") -def test_fetch_latest(mock_get_engine): +@_patch("flytekit.engines.flyte.engine._FlyteClientManager") +@_patch("flytekit.configuration.platform.URL") +def test_fetch_latest(mock_url, mock_client_manager): + mock_url.get.return_value = "localhost" admin_task = _task_models.Task( _identifier.Identifier(_identifier.ResourceType.TASK, "p1", "d1", "n1", "v1"), _MagicMock(), ) - mock_engine = _MagicMock() - mock_engine.fetch_latest_task = _MagicMock(return_value=admin_task) - mock_get_engine.return_value = mock_engine + mock_client = _MagicMock() + mock_client.list_tasks_paginated = _MagicMock(return_value=([admin_task], "")) + mock_client_manager.return_value.client = mock_client task = _task.SdkTask.fetch_latest("p1", "d1", "n1") assert task.id == admin_task.id -@_patch("flytekit.engines.loader.get_engine") -def test_fetch_latest_not_exist(mock_get_engine): - mock_engine = _MagicMock() - mock_engine.fetch_latest_task = _MagicMock(return_value=None) - mock_get_engine.return_value = mock_engine +@_patch("flytekit.engines.flyte.engine._FlyteClientManager") +@_patch("flytekit.configuration.platform.URL") +def test_fetch_latest_not_exist(mock_url, mock_client_manager): + mock_client = _MagicMock() + mock_client.list_tasks_paginated = _MagicMock(return_value=(None, "")) + mock_client_manager.return_value.client = mock_client + mock_url.get.return_value = "localhost" with _pytest.raises(_user_exceptions.FlyteEntityNotExistException): _task.SdkTask.fetch_latest("p1", "d1", "n1") diff --git a/tests/flytekit/unit/common_tests/test_interface.py b/tests/flytekit/unit/common_tests/test_interface.py index 2adaf237dc..e3eb65f081 100644 --- a/tests/flytekit/unit/common_tests/test_interface.py +++ b/tests/flytekit/unit/common_tests/test_interface.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.common import interface diff --git a/tests/flytekit/unit/common_tests/test_launch_plan.py b/tests/flytekit/unit/common_tests/test_launch_plan.py index 83376e4949..ffef6da28a 100644 --- a/tests/flytekit/unit/common_tests/test_launch_plan.py +++ b/tests/flytekit/unit/common_tests/test_launch_plan.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os as _os import pytest as _pytest @@ -297,8 +295,9 @@ def test_serialize(): "default_input": _workflow.Input(_types.Types.Integer, default=5), }, ) - workflow_to_test._id = _identifier.Identifier(_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v") + workflow_to_test.id = _identifier.Identifier(_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v") lp = workflow_to_test.create_launch_plan(fixed_inputs={"required_input": 5}, role="iam_role",) + with _configuration.TemporaryConfiguration( _os.path.join(_os.path.dirname(_os.path.realpath(__file__)), "../../common/configs/local.config",), internal_overrides={"image": "myflyteimage:v123", "project": "myflyteproject", "domain": "development"}, @@ -318,7 +317,7 @@ def test_promote_from_model(): "default_input": _workflow.Input(_types.Types.Integer, default=5), }, ) - workflow_to_test._id = _identifier.Identifier(_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v") + workflow_to_test.id = _identifier.Identifier(_identifier.ResourceType.WORKFLOW, "p", "d", "n", "v") lp = workflow_to_test.create_launch_plan( fixed_inputs={"required_input": 5}, schedule=_schedules.CronSchedule("* * ? * * *"), diff --git a/tests/flytekit/unit/common_tests/test_nodes.py b/tests/flytekit/unit/common_tests/test_nodes.py index fe051de186..7d2b97e81c 100644 --- a/tests/flytekit/unit/common_tests/test_nodes.py +++ b/tests/flytekit/unit/common_tests/test_nodes.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime import pytest as _pytest diff --git a/tests/flytekit/unit/common_tests/test_notifications.py b/tests/flytekit/unit/common_tests/test_notifications.py index 7124e82726..8a2278097a 100644 --- a/tests/flytekit/unit/common_tests/test_notifications.py +++ b/tests/flytekit/unit/common_tests/test_notifications.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common import notifications as _notifications from flytekit.models.core import execution as _execution_model diff --git a/tests/flytekit/unit/common_tests/test_promise.py b/tests/flytekit/unit/common_tests/test_promise.py index 223373b80b..bed6d2c2a6 100644 --- a/tests/flytekit/unit/common_tests/test_promise.py +++ b/tests/flytekit/unit/common_tests/test_promise.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.common import promise diff --git a/tests/flytekit/unit/common_tests/test_schedules.py b/tests/flytekit/unit/common_tests/test_schedules.py index d860019576..14d9203504 100644 --- a/tests/flytekit/unit/common_tests/test_schedules.py +++ b/tests/flytekit/unit/common_tests/test_schedules.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime import pytest as _pytest diff --git a/tests/flytekit/unit/common_tests/test_workflow.py b/tests/flytekit/unit/common_tests/test_workflow.py index 2c2e55a9b5..ae1ef6df34 100644 --- a/tests/flytekit/unit/common_tests/test_workflow.py +++ b/tests/flytekit/unit/common_tests/test_workflow.py @@ -1,10 +1,11 @@ -from __future__ import absolute_import - import pytest as _pytest from flyteidl.admin import workflow_pb2 as _workflow_pb2 -from flytekit.common import constants, interface, nodes, promise, workflow +from flytekit.common import constants, interface +from flytekit.common import local_workflow as _local_workflow +from flytekit.common import nodes, promise, workflow from flytekit.common.exceptions import user as _user_exceptions +from flytekit.common.local_workflow import build_sdk_workflow_from_metaclass from flytekit.common.types import containers, primitives from flytekit.models import literals as _literals from flytekit.models.core import identifier as _identifier @@ -14,7 +15,7 @@ def test_output(): - o = workflow.Output("name", 1, sdk_type=primitives.Integer, help="blah") + o = _local_workflow.Output("name", 1, sdk_type=primitives.Integer, help="blah") assert o.name == "name" assert o.var.description == "blah" assert o.var.type == primitives.Integer.to_flyte_literal_type() @@ -55,8 +56,10 @@ def my_list_task(wf_params, a, b): nodes = [n1, n2, n3, n4, n5, n6] - w = workflow.SdkWorkflow( - inputs=input_list, outputs=[workflow.Output("a", n1.outputs.b, sdk_type=primitives.Integer)], nodes=nodes, + w = _local_workflow.SdkRunnableWorkflow.construct_from_class_definition( + inputs=input_list, + outputs=[_local_workflow.Output("a", n1.outputs.b, sdk_type=primitives.Integer)], + nodes=nodes, ) assert w.interface.inputs["input_1"].type == primitives.Integer.to_flyte_literal_type() @@ -131,9 +134,9 @@ class my_workflow(object): n5 = my_list_task(a=[input_1, input_2, n3.outputs.b, 100]) n6 = my_list_task(a=n5.outputs.b) n1 >> n6 - a = workflow.Output("a", n1.outputs.b, sdk_type=primitives.Integer) + a = _local_workflow.Output("a", n1.outputs.b, sdk_type=primitives.Integer) - w = workflow.build_sdk_workflow_from_metaclass( + w = _local_workflow.build_sdk_workflow_from_metaclass( my_workflow, on_failure=_workflow_models.WorkflowMetadata.OnFailurePolicy.FAIL_AFTER_EXECUTABLE_NODES_COMPLETE, ) @@ -151,7 +154,7 @@ class my_workflow(object): assert w.nodes[3].inputs[0].binding.promise.node_id == "n1" # Test conversion to flyte_idl and back - w._id = _identifier.Identifier(_identifier.ResourceType.WORKFLOW, "fake", "faker", "fakest", "fakerest") + w.id = _identifier.Identifier(_identifier.ResourceType.WORKFLOW, "fake", "faker", "fakest", "fakerest") w = _workflow_models.WorkflowTemplate.from_flyte_idl(w.to_flyte_idl()) assert w.interface.inputs["input_1"].type == primitives.Integer.to_flyte_literal_type() assert w.interface.inputs["input_2"].type == primitives.Integer.to_flyte_literal_type() @@ -219,13 +222,15 @@ def my_list_task(wf_params, a, b): nodes = [n1, n2, n3, n4, n5, n6] wf_out = [ - workflow.Output( + _local_workflow.Output( "nested_out", [n5.outputs.b, n6.outputs.b, [n1.outputs.b, n2.outputs.b]], sdk_type=[[primitives.Integer]], ), - workflow.Output("scalar_out", n1.outputs.b, sdk_type=primitives.Integer), + _local_workflow.Output("scalar_out", n1.outputs.b, sdk_type=primitives.Integer), ] - w = workflow.SdkWorkflow(inputs=input_list, outputs=wf_out, nodes=nodes) + w = _local_workflow.SdkRunnableWorkflow.construct_from_class_definition( + inputs=input_list, outputs=wf_out, nodes=nodes + ) # Test that required input isn't set with _pytest.raises(_user_exceptions.FlyteAssertion): @@ -258,9 +263,9 @@ def my_list_task(wf_params, a, b): assert n.inputs[1].binding.scalar.primitive.integer == 10 # Test that workflow is saved in the node - w._id = "fake" + w.id = "fake" assert n.workflow_node.sub_workflow_ref == "fake" - w._id = None + w.id = None # Test that outputs are promised n.assign_id_and_return("node-id*") # dns'ified @@ -343,13 +348,15 @@ def my_list_task(wf_params, a, b): nodes = [n1, n2, n3, n4, n5, n6] wf_out = [ - workflow.Output( + _local_workflow.Output( "nested_out", [n5.outputs.b, n6.outputs.b, [n1.outputs.b, n2.outputs.b]], sdk_type=[[primitives.Integer]], ), - workflow.Output("scalar_out", n1.outputs.b, sdk_type=primitives.Integer), + _local_workflow.Output("scalar_out", n1.outputs.b, sdk_type=primitives.Integer), ] - w = workflow.SdkWorkflow(inputs=input_list, outputs=wf_out, nodes=nodes) + w = _local_workflow.SdkRunnableWorkflow.construct_from_class_definition( + inputs=input_list, outputs=wf_out, nodes=nodes + ) serialized = w.serialize() assert isinstance(serialized, _workflow_pb2.WorkflowSpec) assert len(serialized.template.nodes) == 6 @@ -362,6 +369,6 @@ class MyWorkflow(object): input_1 = promise.Input("input_1", primitives.Integer) input_2 = promise.Input("input_2", primitives.Integer, default=5, help="Not required.") - w = workflow.build_sdk_workflow_from_metaclass(MyWorkflow, disable_default_launch_plan=True,) + w = build_sdk_workflow_from_metaclass(MyWorkflow, disable_default_launch_plan=True,) assert w.should_create_default_launch_plan is False diff --git a/tests/flytekit/unit/common_tests/test_workflow_promote.py b/tests/flytekit/unit/common_tests/test_workflow_promote.py index 22d9f53d9a..bbfd5e627e 100644 --- a/tests/flytekit/unit/common_tests/test_workflow_promote.py +++ b/tests/flytekit/unit/common_tests/test_workflow_promote.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from datetime import timedelta from os import path as _path @@ -65,8 +63,6 @@ def get_workflow_template(): This function retrieves a TasKTemplate object from the pb file in the resources directory. It was created by reading from Flyte Admin, the following workflow, after registration. - from __future__ import absolute_import - from flytekit.common.types.primitives import Integer from flytekit.sdk.tasks import ( python_task, diff --git a/tests/flytekit/unit/common_tests/types/impl/test_blobs.py b/tests/flytekit/unit/common_tests/types/impl/test_blobs.py index aa9931df8e..3b61837e44 100644 --- a/tests/flytekit/unit/common_tests/types/impl/test_blobs.py +++ b/tests/flytekit/unit/common_tests/types/impl/test_blobs.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os import pytest diff --git a/tests/flytekit/unit/common_tests/types/impl/test_schema.py b/tests/flytekit/unit/common_tests/types/impl/test_schema.py index 3576676b3e..90733564d7 100644 --- a/tests/flytekit/unit/common_tests/types/impl/test_schema.py +++ b/tests/flytekit/unit/common_tests/types/impl/test_schema.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import collections as _collections import datetime as _datetime import os as _os diff --git a/tests/flytekit/unit/common_tests/types/test_blobs.py b/tests/flytekit/unit/common_tests/types/test_blobs.py index f5047da966..4fdee08b3b 100644 --- a/tests/flytekit/unit/common_tests/types/test_blobs.py +++ b/tests/flytekit/unit/common_tests/types/test_blobs.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.types import blobs from flytekit.common.types.impl import blobs as blob_impl from flytekit.models import literals as _literal_models diff --git a/tests/flytekit/unit/common_tests/types/test_containers.py b/tests/flytekit/unit/common_tests/types/test_containers.py index cdba800058..dcb8730b08 100644 --- a/tests/flytekit/unit/common_tests/types/test_containers.py +++ b/tests/flytekit/unit/common_tests/types/test_containers.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from six.moves import range as _range diff --git a/tests/flytekit/unit/common_tests/types/test_helpers.py b/tests/flytekit/unit/common_tests/types/test_helpers.py index 875a0bc40c..425feef30f 100644 --- a/tests/flytekit/unit/common_tests/types/test_helpers.py +++ b/tests/flytekit/unit/common_tests/types/test_helpers.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.types import base_sdk_types as _base_sdk_types from flytekit.common.types import helpers as _type_helpers from flytekit.models import literals as _literals diff --git a/tests/flytekit/unit/common_tests/types/test_primitives.py b/tests/flytekit/unit/common_tests/types/test_primitives.py index 9b7d32d898..b161c751dd 100644 --- a/tests/flytekit/unit/common_tests/types/test_primitives.py +++ b/tests/flytekit/unit/common_tests/types/test_primitives.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime import pytest diff --git a/tests/flytekit/unit/common_tests/types/test_proto.py b/tests/flytekit/unit/common_tests/types/test_proto.py index 9fab12fca6..959dd2d56d 100644 --- a/tests/flytekit/unit/common_tests/types/test_proto.py +++ b/tests/flytekit/unit/common_tests/types/test_proto.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import base64 as _base64 import pytest as _pytest diff --git a/tests/flytekit/unit/common_tests/types/test_schema.py b/tests/flytekit/unit/common_tests/types/test_schema.py index 512e814fe6..02bfb8f55e 100644 --- a/tests/flytekit/unit/common_tests/types/test_schema.py +++ b/tests/flytekit/unit/common_tests/types/test_schema.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.types import primitives, schema from flytekit.common.types.impl import schema as schema_impl from flytekit.sdk import test_utils diff --git a/tests/flytekit/unit/configuration/conftest.py b/tests/flytekit/unit/configuration/conftest.py index 9feb0c8738..700805ee96 100644 --- a/tests/flytekit/unit/configuration/conftest.py +++ b/tests/flytekit/unit/configuration/conftest.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os as _os import pytest as _pytest diff --git a/tests/flytekit/unit/configuration/test_common.py b/tests/flytekit/unit/configuration/test_common.py index 8210753e92..a9088bdba6 100644 --- a/tests/flytekit/unit/configuration/test_common.py +++ b/tests/flytekit/unit/configuration/test_common.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os import pytest diff --git a/tests/flytekit/unit/configuration/test_internal.py b/tests/flytekit/unit/configuration/test_internal.py index 3d28e47abc..73cbf62b49 100644 --- a/tests/flytekit/unit/configuration/test_internal.py +++ b/tests/flytekit/unit/configuration/test_internal.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.configuration.internal import look_up_version_from_image_tag diff --git a/tests/flytekit/unit/configuration/test_resources.py b/tests/flytekit/unit/configuration/test_resources.py index c50a82cac4..3564ebe363 100644 --- a/tests/flytekit/unit/configuration/test_resources.py +++ b/tests/flytekit/unit/configuration/test_resources.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os from flytekit.configuration import resources, set_flyte_config_file diff --git a/tests/flytekit/unit/configuration/test_temporary_configuration.py b/tests/flytekit/unit/configuration/test_temporary_configuration.py index 4bf7e8ae95..a41f131ec8 100644 --- a/tests/flytekit/unit/configuration/test_temporary_configuration.py +++ b/tests/flytekit/unit/configuration/test_temporary_configuration.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os as _os from flytekit.configuration import TemporaryConfiguration as _TemporaryConfiguration diff --git a/tests/flytekit/unit/configuration/test_waterfall.py b/tests/flytekit/unit/configuration/test_waterfall.py index 07460e434d..2b35d641c0 100644 --- a/tests/flytekit/unit/configuration/test_waterfall.py +++ b/tests/flytekit/unit/configuration/test_waterfall.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os as _os from flytekit.common.utils import AutoDeletingTempDir as _AutoDeletingTempDir diff --git a/tests/flytekit/unit/contrib/sensors/test_impl.py b/tests/flytekit/unit/contrib/sensors/test_impl.py index f84b4287b3..e14d4f75da 100644 --- a/tests/flytekit/unit/contrib/sensors/test_impl.py +++ b/tests/flytekit/unit/contrib/sensors/test_impl.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import mock from hmsclient import HMSClient from hmsclient.genthrift.hive_metastore import ttypes as _ttypes diff --git a/tests/flytekit/unit/engines/flyte/test_engine.py b/tests/flytekit/unit/engines/flyte/test_engine.py index b9fb0c25b2..452f9beeba 100644 --- a/tests/flytekit/unit/engines/flyte/test_engine.py +++ b/tests/flytekit/unit/engines/flyte/test_engine.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os import pytest diff --git a/tests/flytekit/unit/engines/test_loader.py b/tests/flytekit/unit/engines/test_loader.py index 248e35dd18..5f6ee2d46c 100644 --- a/tests/flytekit/unit/engines/test_loader.py +++ b/tests/flytekit/unit/engines/test_loader.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.engines import loader diff --git a/tests/flytekit/unit/interfaces/test_random.py b/tests/flytekit/unit/interfaces/test_random.py index ee247a515c..6b3d99948c 100644 --- a/tests/flytekit/unit/interfaces/test_random.py +++ b/tests/flytekit/unit/interfaces/test_random.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import random as global_random from flytekit.interfaces import random diff --git a/tests/flytekit/unit/models/admin/test_common.py b/tests/flytekit/unit/models/admin/test_common.py index 5ff6fc54f2..0e1a8abf70 100644 --- a/tests/flytekit/unit/models/admin/test_common.py +++ b/tests/flytekit/unit/models/admin/test_common.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest as _pytest from flytekit.models.admin import common as _common diff --git a/tests/flytekit/unit/models/core/test_errors.py b/tests/flytekit/unit/models/core/test_errors.py index 935886b763..fd55ba5ebf 100644 --- a/tests/flytekit/unit/models/core/test_errors.py +++ b/tests/flytekit/unit/models/core/test_errors.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.models.core import errors diff --git a/tests/flytekit/unit/models/core/test_execution.py b/tests/flytekit/unit/models/core/test_execution.py index c8bc7839f4..a0294967a7 100644 --- a/tests/flytekit/unit/models/core/test_execution.py +++ b/tests/flytekit/unit/models/core/test_execution.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime from flytekit.models.core import execution diff --git a/tests/flytekit/unit/models/core/test_identifier.py b/tests/flytekit/unit/models/core/test_identifier.py index 633aa1df66..75ae18cf23 100644 --- a/tests/flytekit/unit/models/core/test_identifier.py +++ b/tests/flytekit/unit/models/core/test_identifier.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.models.core import identifier diff --git a/tests/flytekit/unit/models/core/test_types.py b/tests/flytekit/unit/models/core/test_types.py index 55ce9abf16..744e1f90d5 100644 --- a/tests/flytekit/unit/models/core/test_types.py +++ b/tests/flytekit/unit/models/core/test_types.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flyteidl.core import types_pb2 as _types_pb2 from flytekit.models.core import types as _types diff --git a/tests/flytekit/unit/models/core/test_workflow.py b/tests/flytekit/unit/models/core/test_workflow.py index c906c3f40a..b64b87eb0b 100644 --- a/tests/flytekit/unit/models/core/test_workflow.py +++ b/tests/flytekit/unit/models/core/test_workflow.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from datetime import timedelta from flytekit.models import interface as _interface diff --git a/tests/flytekit/unit/models/test_common.py b/tests/flytekit/unit/models/test_common.py index eb41871eb0..7982f0570f 100644 --- a/tests/flytekit/unit/models/test_common.py +++ b/tests/flytekit/unit/models/test_common.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.models import common as _common from flytekit.models.core import execution as _execution diff --git a/tests/flytekit/unit/models/test_dynamic_job.py b/tests/flytekit/unit/models/test_dynamic_job.py index 146607d533..0a9dff117f 100644 --- a/tests/flytekit/unit/models/test_dynamic_job.py +++ b/tests/flytekit/unit/models/test_dynamic_job.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from datetime import timedelta as _timedelta from itertools import product diff --git a/tests/flytekit/unit/models/test_dynamic_wfs.py b/tests/flytekit/unit/models/test_dynamic_wfs.py index 5a4ce948e3..302e87e705 100644 --- a/tests/flytekit/unit/models/test_dynamic_wfs.py +++ b/tests/flytekit/unit/models/test_dynamic_wfs.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - from flytekit.common import constants as _sdk_constants from flytekit.sdk import tasks as _tasks from flytekit.sdk import workflow as _workflow diff --git a/tests/flytekit/unit/models/test_execution.py b/tests/flytekit/unit/models/test_execution.py index d5170c3619..362ea11349 100644 --- a/tests/flytekit/unit/models/test_execution.py +++ b/tests/flytekit/unit/models/test_execution.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.models import common as _common_models diff --git a/tests/flytekit/unit/models/test_filters.py b/tests/flytekit/unit/models/test_filters.py index 5c4c432a9e..11be4848b8 100644 --- a/tests/flytekit/unit/models/test_filters.py +++ b/tests/flytekit/unit/models/test_filters.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.models import filters diff --git a/tests/flytekit/unit/models/test_interface.py b/tests/flytekit/unit/models/test_interface.py index 316b970d83..03f89f69e4 100644 --- a/tests/flytekit/unit/models/test_interface.py +++ b/tests/flytekit/unit/models/test_interface.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.models import interface, types diff --git a/tests/flytekit/unit/models/test_literals.py b/tests/flytekit/unit/models/test_literals.py index 4790881888..b6eec52cd7 100644 --- a/tests/flytekit/unit/models/test_literals.py +++ b/tests/flytekit/unit/models/test_literals.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from datetime import datetime, timedelta import pytest diff --git a/tests/flytekit/unit/models/test_matchable_resource.py b/tests/flytekit/unit/models/test_matchable_resource.py index 229f0797e8..01274c148f 100644 --- a/tests/flytekit/unit/models/test_matchable_resource.py +++ b/tests/flytekit/unit/models/test_matchable_resource.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.models import matchable_resource diff --git a/tests/flytekit/unit/models/test_named_entity.py b/tests/flytekit/unit/models/test_named_entity.py index 20168ba7a4..bb3db95439 100644 --- a/tests/flytekit/unit/models/test_named_entity.py +++ b/tests/flytekit/unit/models/test_named_entity.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.models import named_entity diff --git a/tests/flytekit/unit/models/test_project.py b/tests/flytekit/unit/models/test_project.py index 53bd733015..d53ffe81b7 100644 --- a/tests/flytekit/unit/models/test_project.py +++ b/tests/flytekit/unit/models/test_project.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.models import project diff --git a/tests/flytekit/unit/models/test_qubole.py b/tests/flytekit/unit/models/test_qubole.py index 1421221fcf..34655379f8 100644 --- a/tests/flytekit/unit/models/test_qubole.py +++ b/tests/flytekit/unit/models/test_qubole.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.models import qubole diff --git a/tests/flytekit/unit/models/test_schedule.py b/tests/flytekit/unit/models/test_schedule.py index 1889cd76c7..ac405e4a7c 100644 --- a/tests/flytekit/unit/models/test_schedule.py +++ b/tests/flytekit/unit/models/test_schedule.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.models import schedule as _schedule diff --git a/tests/flytekit/unit/models/test_tasks.py b/tests/flytekit/unit/models/test_tasks.py index 31d4c84270..6bd6a3277e 100644 --- a/tests/flytekit/unit/models/test_tasks.py +++ b/tests/flytekit/unit/models/test_tasks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from datetime import timedelta from itertools import product diff --git a/tests/flytekit/unit/models/test_types.py b/tests/flytekit/unit/models/test_types.py index dd539db528..27b5ddf595 100644 --- a/tests/flytekit/unit/models/test_types.py +++ b/tests/flytekit/unit/models/test_types.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flyteidl.core import types_pb2 diff --git a/tests/flytekit/unit/models/test_workflow_closure.py b/tests/flytekit/unit/models/test_workflow_closure.py index 0a2a6e09b2..3368387a7f 100644 --- a/tests/flytekit/unit/models/test_workflow_closure.py +++ b/tests/flytekit/unit/models/test_workflow_closure.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from datetime import timedelta from flytekit.models import interface as _interface diff --git a/tests/flytekit/unit/sdk/conftest.py b/tests/flytekit/unit/sdk/conftest.py index 0be301949f..874a288263 100644 --- a/tests/flytekit/unit/sdk/conftest.py +++ b/tests/flytekit/unit/sdk/conftest.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest as _pytest from flytekit.configuration import TemporaryConfiguration diff --git a/tests/flytekit/unit/sdk/tasks/test_dynamic_tasks.py b/tests/flytekit/unit/sdk/tasks/test_dynamic_tasks.py index 4fcabc1877..44ded2e3bd 100644 --- a/tests/flytekit/unit/sdk/tasks/test_dynamic_tasks.py +++ b/tests/flytekit/unit/sdk/tasks/test_dynamic_tasks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - from six import moves as _six_moves from flytekit.common.tasks import sdk_dynamic as _sdk_dynamic diff --git a/tests/flytekit/unit/sdk/tasks/test_hive_tasks.py b/tests/flytekit/unit/sdk/tasks/test_hive_tasks.py index 48475ca18e..63542e3331 100644 --- a/tests/flytekit/unit/sdk/tasks/test_hive_tasks.py +++ b/tests/flytekit/unit/sdk/tasks/test_hive_tasks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, print_function - import logging as _logging from datetime import datetime as _datetime diff --git a/tests/flytekit/unit/sdk/tasks/test_pytorch_task.py b/tests/flytekit/unit/sdk/tasks/test_pytorch_task.py index 33bff0766f..8472cde292 100644 --- a/tests/flytekit/unit/sdk/tasks/test_pytorch_task.py +++ b/tests/flytekit/unit/sdk/tasks/test_pytorch_task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime from flytekit.common import constants as _common_constants diff --git a/tests/flytekit/unit/sdk/tasks/test_sagemaker_tasks.py b/tests/flytekit/unit/sdk/tasks/test_sagemaker_tasks.py index fd137306d1..1a2a82d8e6 100644 --- a/tests/flytekit/unit/sdk/tasks/test_sagemaker_tasks.py +++ b/tests/flytekit/unit/sdk/tasks/test_sagemaker_tasks.py @@ -187,7 +187,8 @@ def test_simple_hpo_job_task(): assert simple_xgboost_hpo_job_task.metadata.deprecated_error_message == "" assert "metricDefinitions" in simple_xgboost_hpo_job_task.custom["trainingJob"]["algorithmSpecification"].keys() assert len(simple_xgboost_hpo_job_task.custom["trainingJob"]["algorithmSpecification"]["metricDefinitions"]) == 1 - """ These are attributes for SdkRunnable. We will need these when supporting CustomTrainingJobTask and CustomHPOJobTask + """ + These are attributes for SdkRunnable. We will need these when supporting CustomTrainingJobTask and CustomHPOJobTask assert simple_xgboost_hpo_job_task.task_module == __name__ assert simple_xgboost_hpo_job_task._get_container_definition().args[0] == 'pyflyte-execute' """ diff --git a/tests/flytekit/unit/sdk/tasks/test_sidecar_tasks.py b/tests/flytekit/unit/sdk/tasks/test_sidecar_tasks.py index 503202706f..a0fbf1f8a4 100644 --- a/tests/flytekit/unit/sdk/tasks/test_sidecar_tasks.py +++ b/tests/flytekit/unit/sdk/tasks/test_sidecar_tasks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import mock from k8s.io.api.core.v1 import generated_pb2 diff --git a/tests/flytekit/unit/sdk/tasks/test_spark_task.py b/tests/flytekit/unit/sdk/tasks/test_spark_task.py index 31418a6b46..c654d79ad2 100644 --- a/tests/flytekit/unit/sdk/tasks/test_spark_task.py +++ b/tests/flytekit/unit/sdk/tasks/test_spark_task.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime import os as _os import sys as _sys diff --git a/tests/flytekit/unit/sdk/tasks/test_tasks.py b/tests/flytekit/unit/sdk/tasks/test_tasks.py index 30a13eaee0..a21ffaed04 100644 --- a/tests/flytekit/unit/sdk/tasks/test_tasks.py +++ b/tests/flytekit/unit/sdk/tasks/test_tasks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import datetime as _datetime import os as _os diff --git a/tests/flytekit/unit/sdk/test_workflow.py b/tests/flytekit/unit/sdk/test_workflow.py index fd2dcf7d2a..2b604b5bf6 100644 --- a/tests/flytekit/unit/sdk/test_workflow.py +++ b/tests/flytekit/unit/sdk/test_workflow.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.common import constants diff --git a/tests/flytekit/unit/sdk/types/test_blobs.py b/tests/flytekit/unit/sdk/types/test_blobs.py index cc82ee27ce..78e99fe5a7 100644 --- a/tests/flytekit/unit/sdk/types/test_blobs.py +++ b/tests/flytekit/unit/sdk/types/test_blobs.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.common.types.impl import blobs as _blob_impl diff --git a/tests/flytekit/unit/sdk/types/test_primitives.py b/tests/flytekit/unit/sdk/types/test_primitives.py index 8691963307..e039b9780b 100644 --- a/tests/flytekit/unit/sdk/types/test_primitives.py +++ b/tests/flytekit/unit/sdk/types/test_primitives.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.sdk import types as _sdk_types diff --git a/tests/flytekit/unit/sdk/types/test_schema.py b/tests/flytekit/unit/sdk/types/test_schema.py index a8510b8ffa..95f70d6702 100644 --- a/tests/flytekit/unit/sdk/types/test_schema.py +++ b/tests/flytekit/unit/sdk/types/test_schema.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.common.exceptions import user as _user_exceptions diff --git a/tests/flytekit/unit/test_plugins.py b/tests/flytekit/unit/test_plugins.py index 611e50bcd3..1a5fb33ccf 100644 --- a/tests/flytekit/unit/test_plugins.py +++ b/tests/flytekit/unit/test_plugins.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit import plugins diff --git a/tests/flytekit/unit/tools/test_aws.py b/tests/flytekit/unit/tools/test_aws.py index 5107c8b1e7..93445b0752 100644 --- a/tests/flytekit/unit/tools/test_aws.py +++ b/tests/flytekit/unit/tools/test_aws.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.interfaces.data.s3.s3proxy import AwsS3Proxy diff --git a/tests/flytekit/unit/tools/test_lazy_loader.py b/tests/flytekit/unit/tools/test_lazy_loader.py index 4b29208c93..7801318408 100644 --- a/tests/flytekit/unit/tools/test_lazy_loader.py +++ b/tests/flytekit/unit/tools/test_lazy_loader.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest import six diff --git a/tests/flytekit/unit/tools/test_module_loader.py b/tests/flytekit/unit/tools/test_module_loader.py index d0ef966c63..dbcf246a00 100644 --- a/tests/flytekit/unit/tools/test_module_loader.py +++ b/tests/flytekit/unit/tools/test_module_loader.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import os import sys diff --git a/tests/flytekit/unit/tools/test_subprocess.py b/tests/flytekit/unit/tools/test_subprocess.py index 0b440e6ee9..c61a03873a 100644 --- a/tests/flytekit/unit/tools/test_subprocess.py +++ b/tests/flytekit/unit/tools/test_subprocess.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import mock from flytekit.tools import subprocess diff --git a/tests/flytekit/unit/type_engines/default/test_flyte_type_engine.py b/tests/flytekit/unit/type_engines/default/test_flyte_type_engine.py index 96d90ccfaa..90cbef2253 100644 --- a/tests/flytekit/unit/type_engines/default/test_flyte_type_engine.py +++ b/tests/flytekit/unit/type_engines/default/test_flyte_type_engine.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flyteidl.core import errors_pb2 as _errors_pb2 diff --git a/tests/flytekit/unit/use_scenarios/unit_testing/test_blobs.py b/tests/flytekit/unit/use_scenarios/unit_testing/test_blobs.py index 4ca56a1695..0530bdb865 100644 --- a/tests/flytekit/unit/use_scenarios/unit_testing/test_blobs.py +++ b/tests/flytekit/unit/use_scenarios/unit_testing/test_blobs.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from flytekit.common.utils import AutoDeletingTempDir from flytekit.sdk.tasks import inputs, outputs, python_task from flytekit.sdk.test_utils import flyte_test diff --git a/tests/flytekit/unit/use_scenarios/unit_testing/test_hive_tasks.py b/tests/flytekit/unit/use_scenarios/unit_testing/test_hive_tasks.py index 1a4ed7cb16..a137f254c1 100644 --- a/tests/flytekit/unit/use_scenarios/unit_testing/test_hive_tasks.py +++ b/tests/flytekit/unit/use_scenarios/unit_testing/test_hive_tasks.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from flytekit.sdk.tasks import hive_task diff --git a/tests/flytekit/unit/use_scenarios/unit_testing/test_schemas.py b/tests/flytekit/unit/use_scenarios/unit_testing/test_schemas.py index 93fed3eb3d..e7df18e86a 100644 --- a/tests/flytekit/unit/use_scenarios/unit_testing/test_schemas.py +++ b/tests/flytekit/unit/use_scenarios/unit_testing/test_schemas.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pandas as pd import pytest