From 4208da21d919b8e0ab0df5d6fac715804dd28ad3 Mon Sep 17 00:00:00 2001 From: Yicheng-Lu-llll <51814063+Yicheng-Lu-llll@users.noreply.github.com> Date: Fri, 15 Mar 2024 01:21:13 -0500 Subject: [PATCH 01/50] Add Ray Autoscaler to the Flyte-Ray plugin (#1937) Signed-off-by: Yicheng-Lu-llll --- .../flytekitplugins/ray/models.py | 40 +++++++++++++++++-- .../flytekit-ray/flytekitplugins/ray/task.py | 6 +++ plugins/flytekit-ray/tests/test_ray.py | 9 ++++- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/plugins/flytekit-ray/flytekitplugins/ray/models.py b/plugins/flytekit-ray/flytekitplugins/ray/models.py index 080f1239b4..06e36af186 100644 --- a/plugins/flytekit-ray/flytekitplugins/ray/models.py +++ b/plugins/flytekit-ray/flytekitplugins/ray/models.py @@ -10,14 +10,14 @@ def __init__( self, group_name: str, replicas: int, - min_replicas: typing.Optional[int] = 0, + min_replicas: typing.Optional[int] = None, max_replicas: typing.Optional[int] = None, ray_start_params: typing.Optional[typing.Dict[str, str]] = None, ): self._group_name = group_name self._replicas = replicas - self._min_replicas = min_replicas - self._max_replicas = max_replicas if max_replicas else replicas + self._max_replicas = max(replicas, max_replicas) if max_replicas is not None else replicas + self._min_replicas = min(replicas, min_replicas) if min_replicas is not None else replicas self._ray_start_params = ray_start_params @property @@ -127,10 +127,14 @@ class RayCluster(_common.FlyteIdlEntity): """ def __init__( - self, worker_group_spec: typing.List[WorkerGroupSpec], head_group_spec: typing.Optional[HeadGroupSpec] = None + self, + worker_group_spec: typing.List[WorkerGroupSpec], + head_group_spec: typing.Optional[HeadGroupSpec] = None, + enable_autoscaling: bool = False, ): self._head_group_spec = head_group_spec self._worker_group_spec = worker_group_spec + self._enable_autoscaling = enable_autoscaling @property def head_group_spec(self) -> HeadGroupSpec: @@ -148,6 +152,14 @@ def worker_group_spec(self) -> typing.List[WorkerGroupSpec]: """ return self._worker_group_spec + @property + def enable_autoscaling(self) -> bool: + """ + Whether to enable autoscaling. + :rtype: bool + """ + return self._enable_autoscaling + def to_flyte_idl(self) -> _ray_pb2.RayCluster: """ :rtype: flyteidl.plugins._ray_pb2.RayCluster @@ -155,6 +167,7 @@ def to_flyte_idl(self) -> _ray_pb2.RayCluster: return _ray_pb2.RayCluster( head_group_spec=self.head_group_spec.to_flyte_idl() if self.head_group_spec else None, worker_group_spec=[wg.to_flyte_idl() for wg in self.worker_group_spec], + enable_autoscaling=self.enable_autoscaling, ) @classmethod @@ -166,6 +179,7 @@ def from_flyte_idl(cls, proto): return cls( head_group_spec=HeadGroupSpec.from_flyte_idl(proto.head_group_spec) if proto.head_group_spec else None, worker_group_spec=[WorkerGroupSpec.from_flyte_idl(wg) for wg in proto.worker_group_spec], + enable_autoscaling=proto.enable_autoscaling, ) @@ -178,9 +192,13 @@ def __init__( self, ray_cluster: RayCluster, runtime_env: typing.Optional[str], + ttl_seconds_after_finished: typing.Optional[int] = None, + shutdown_after_job_finishes: bool = False, ): self._ray_cluster = ray_cluster self._runtime_env = runtime_env + self._ttl_seconds_after_finished = ttl_seconds_after_finished + self._shutdown_after_job_finishes = shutdown_after_job_finishes @property def ray_cluster(self) -> RayCluster: @@ -190,10 +208,22 @@ def ray_cluster(self) -> RayCluster: def runtime_env(self) -> typing.Optional[str]: return self._runtime_env + @property + def ttl_seconds_after_finished(self) -> typing.Optional[int]: + # ttl_seconds_after_finished specifies the number of seconds after which the RayCluster will be deleted after the RayJob finishes. + return self._ttl_seconds_after_finished + + @property + def shutdown_after_job_finishes(self) -> bool: + # shutdown_after_job_finishes specifies whether the RayCluster should be deleted after the RayJob finishes. + return self._shutdown_after_job_finishes + def to_flyte_idl(self) -> _ray_pb2.RayJob: return _ray_pb2.RayJob( ray_cluster=self.ray_cluster.to_flyte_idl(), runtime_env=self.runtime_env, + ttl_seconds_after_finished=self.ttl_seconds_after_finished, + shutdown_after_job_finishes=self.shutdown_after_job_finishes, ) @classmethod @@ -201,4 +231,6 @@ def from_flyte_idl(cls, proto: _ray_pb2.RayJob): return cls( ray_cluster=RayCluster.from_flyte_idl(proto.ray_cluster) if proto.ray_cluster else None, runtime_env=proto.runtime_env, + ttl_seconds_after_finished=proto.ttl_seconds_after_finished, + shutdown_after_job_finishes=proto.shutdown_after_job_finishes, ) diff --git a/plugins/flytekit-ray/flytekitplugins/ray/task.py b/plugins/flytekit-ray/flytekitplugins/ray/task.py index f0ee542b32..76688d74cd 100644 --- a/plugins/flytekit-ray/flytekitplugins/ray/task.py +++ b/plugins/flytekit-ray/flytekitplugins/ray/task.py @@ -34,8 +34,11 @@ class WorkerNodeConfig: class RayJobConfig: worker_node_config: typing.List[WorkerNodeConfig] head_node_config: typing.Optional[HeadNodeConfig] = None + enable_autoscaling: bool = False runtime_env: typing.Optional[dict] = None address: typing.Optional[str] = None + shutdown_after_job_finishes: bool = False + ttl_seconds_after_finished: typing.Optional[int] = None class RayFunctionTask(PythonFunctionTask): @@ -67,9 +70,12 @@ def get_custom(self, settings: SerializationSettings) -> Optional[Dict[str, Any] WorkerGroupSpec(c.group_name, c.replicas, c.min_replicas, c.max_replicas, c.ray_start_params) for c in cfg.worker_node_config ], + enable_autoscaling=cfg.enable_autoscaling if cfg.enable_autoscaling else False, ), # Use base64 to encode runtime_env dict and convert it to byte string runtime_env=base64.b64encode(json.dumps(cfg.runtime_env).encode()).decode(), + ttl_seconds_after_finished=cfg.ttl_seconds_after_finished, + shutdown_after_job_finishes=cfg.shutdown_after_job_finishes, ) return MessageToDict(ray_job.to_flyte_idl()) diff --git a/plugins/flytekit-ray/tests/test_ray.py b/plugins/flytekit-ray/tests/test_ray.py index 8bcebf7937..0c0ada1944 100644 --- a/plugins/flytekit-ray/tests/test_ray.py +++ b/plugins/flytekit-ray/tests/test_ray.py @@ -10,8 +10,11 @@ from flytekit.configuration import Image, ImageConfig, SerializationSettings config = RayJobConfig( - worker_node_config=[WorkerNodeConfig(group_name="test_group", replicas=3)], + worker_node_config=[WorkerNodeConfig(group_name="test_group", replicas=3, min_replicas=0, max_replicas=10)], runtime_env={"pip": ["numpy"]}, + enable_autoscaling=True, + shutdown_after_job_finishes=True, + ttl_seconds_after_finished=20, ) @@ -37,8 +40,10 @@ def t1(a: int) -> str: ) ray_job_pb = RayJob( - ray_cluster=RayCluster(worker_group_spec=[WorkerGroupSpec("test_group", 3)]), + ray_cluster=RayCluster(worker_group_spec=[WorkerGroupSpec("test_group", 3, 0, 10)], enable_autoscaling=True), runtime_env=base64.b64encode(json.dumps({"pip": ["numpy"]}).encode()).decode(), + shutdown_after_job_finishes=True, + ttl_seconds_after_finished=20, ).to_flyte_idl() assert t1.get_custom(settings) == MessageToDict(ray_job_pb) From 3f45131887602551036e83a86a71c9f7badf7b52 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 15 Mar 2024 16:40:10 -0400 Subject: [PATCH 02/50] Migrate over to using datetime.now(timezone.utc) (#2270) Signed-off-by: Thomas J. Fan --- flytekit/bin/entrypoint.py | 2 +- flytekit/core/context_manager.py | 2 +- flytekit/core/mock_stats.py | 6 ++++-- flytekit/core/utils.py | 4 ++-- flytekit/experimental/eager_function.py | 10 +++++----- flytekit/remote/remote.py | 4 ++-- plugins/flytekit-deck-standard/tests/test_renderer.py | 4 ++-- .../flytekitplugins/great_expectations/schema.py | 2 +- .../flytekitplugins/great_expectations/task.py | 2 +- .../tests/test_flytekitplugins_iap.py | 4 ++-- tests/flytekit/integration/remote/test_remote.py | 4 ++-- tests/flytekit/unit/core/test_context_manager.py | 4 ++-- tests/flytekit/unit/deck/test_deck.py | 4 ++-- tests/flytekit/unit/models/test_literals.py | 2 +- 14 files changed, 28 insertions(+), 26 deletions(-) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index bfb4fd860b..92f56409ec 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -249,7 +249,7 @@ def setup_execution( domain=exe_domain, name=exe_name, ), - execution_date=_datetime.datetime.utcnow(), + execution_date=_datetime.datetime.now(_datetime.timezone.utc), stats=_get_stats( cfg=StatsConfig.auto(), # Stats metric path will be: diff --git a/flytekit/core/context_manager.py b/flytekit/core/context_manager.py index 5c6341712a..f70f10bc94 100644 --- a/flytekit/core/context_manager.py +++ b/flytekit/core/context_manager.py @@ -937,7 +937,7 @@ def initialize(): default_user_space_params = ExecutionParameters( execution_id=WorkflowExecutionIdentifier.promote_from_model(default_execution_id), task_id=_identifier.Identifier(_identifier.ResourceType.TASK, "local", "local", "local", "local"), - execution_date=_datetime.datetime.utcnow(), + execution_date=_datetime.datetime.now(_datetime.timezone.utc), stats=mock_stats.MockStats(), logging=user_space_logger, tmp_dir=user_space_path, diff --git a/flytekit/core/mock_stats.py b/flytekit/core/mock_stats.py index dedce6ae7c..18763fa74a 100644 --- a/flytekit/core/mock_stats.py +++ b/flytekit/core/mock_stats.py @@ -57,8 +57,10 @@ def __init__(self, mock_stats, metric, tags): self._tags = tags def __enter__(self): - self._timer = _datetime.datetime.utcnow() + self._timer = _datetime.datetime.now(_datetime.timezone.utc) def __exit__(self, exc_type, exc_val, exc_tb): - self._mock_stats.gauge(self._metric, _datetime.datetime.utcnow() - self._timer, tags=self._tags) + self._mock_stats.gauge( + self._metric, _datetime.datetime.now(_datetime.timezone.utc) - self._timer, tags=self._tags + ) self._timer = None diff --git a/flytekit/core/utils.py b/flytekit/core/utils.py index 17cdfb3de9..b5a415d13d 100644 --- a/flytekit/core/utils.py +++ b/flytekit/core/utils.py @@ -311,7 +311,7 @@ def wrapper(*args, **kwargs): return wrapper def __enter__(self): - self.start_time = datetime.datetime.utcnow() + self.start_time = datetime.datetime.now(datetime.timezone.utc) self._start_wall_time = _time.perf_counter() self._start_process_time = _time.process_time() return self @@ -323,7 +323,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): """ from flytekit.core.context_manager import FlyteContextManager - end_time = datetime.datetime.utcnow() + end_time = datetime.datetime.now(datetime.timezone.utc) end_wall_time = _time.perf_counter() end_process_time = _time.process_time() diff --git a/flytekit/experimental/eager_function.py b/flytekit/experimental/eager_function.py index 7a3cee897d..d47a2baef2 100644 --- a/flytekit/experimental/eager_function.py +++ b/flytekit/experimental/eager_function.py @@ -2,7 +2,7 @@ import inspect import signal from contextlib import asynccontextmanager -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from functools import partial, wraps from typing import List, Optional @@ -179,9 +179,9 @@ async def __call__(self, **kwargs): self.async_stack.set_node(node) poll_interval = self._poll_interval or timedelta(seconds=30) - time_to_give_up = datetime.max if self._timeout is None else datetime.utcnow() + self._timeout + time_to_give_up = datetime.max if self._timeout is None else datetime.now(timezone.utc) + self._timeout - while datetime.utcnow() < time_to_give_up: + while datetime.now(timezone.utc) < time_to_give_up: execution = self.remote.sync(execution) if execution.closure.phase in {WorkflowExecutionPhase.FAILED}: raise EagerException(f"Error executing {self.entity.name} with error: {execution.closure.error}") @@ -208,9 +208,9 @@ async def terminate(self): ) poll_interval = self._poll_interval or timedelta(seconds=6) - time_to_give_up = datetime.max if self._timeout is None else datetime.utcnow() + self._timeout + time_to_give_up = datetime.max if self._timeout is None else datetime.now(timezone.utc) + self._timeout - while datetime.utcnow() < time_to_give_up: + while datetime.now(timezone.utc) < time_to_give_up: execution = self.remote.sync(execution) if execution.is_done: break diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 992685abb8..4fd17fe40b 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -1661,9 +1661,9 @@ def wait( :param sync_nodes: passed along to the sync call for the workflow execution """ poll_interval = poll_interval or timedelta(seconds=30) - time_to_give_up = datetime.max if timeout is None else datetime.utcnow() + timeout + time_to_give_up = datetime.max if timeout is None else datetime.now() + timeout - while datetime.utcnow() < time_to_give_up: + while datetime.now() < time_to_give_up: execution = self.sync_execution(execution, sync_nodes=sync_nodes) if execution.is_done: return execution diff --git a/plugins/flytekit-deck-standard/tests/test_renderer.py b/plugins/flytekit-deck-standard/tests/test_renderer.py index c114333d58..dbe157cefc 100644 --- a/plugins/flytekit-deck-standard/tests/test_renderer.py +++ b/plugins/flytekit-deck-standard/tests/test_renderer.py @@ -22,8 +22,8 @@ [ dict( Name="foo", - Start=datetime.datetime.utcnow(), - Finish=datetime.datetime.utcnow() + datetime.timedelta(microseconds=1000), + Start=datetime.datetime.now(datetime.timezone.utc), + Finish=datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(microseconds=1000), WallTime=1.0, ProcessTime=1.0, ) diff --git a/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py b/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py index 3413cdcdd3..4cd0d5a3e9 100644 --- a/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py +++ b/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py @@ -290,7 +290,7 @@ def to_python_value( run_id = ge.core.run_identifier.RunIdentifier( **{ "run_name": ge_conf.datasource_name + "_run", - "run_time": datetime.datetime.utcnow(), + "run_time": datetime.datetime.now(datetime.timezone.utc), } ) diff --git a/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/task.py b/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/task.py index bd04f18782..a39baacf34 100644 --- a/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/task.py +++ b/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/task.py @@ -216,7 +216,7 @@ def execute(self, **kwargs) -> Any: run_id = ge.core.run_identifier.RunIdentifier( **{ "run_name": self._datasource_name + "_run", - "run_time": datetime.datetime.utcnow(), + "run_time": datetime.datetime.now(datetime.timezone.utc), } ) diff --git a/plugins/flytekit-identity-aware-proxy/tests/test_flytekitplugins_iap.py b/plugins/flytekit-identity-aware-proxy/tests/test_flytekitplugins_iap.py index 766ff646ab..9b0136331d 100644 --- a/plugins/flytekit-identity-aware-proxy/tests/test_flytekitplugins_iap.py +++ b/plugins/flytekit-identity-aware-proxy/tests/test_flytekitplugins_iap.py @@ -1,5 +1,5 @@ import uuid -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock, patch import click @@ -58,7 +58,7 @@ def test_get_gcp_secret_manager_secret_not_found(): def create_mock_token(aud: str, expires_in: timedelta = None): """Create a mock JWT token with a certain audience, expiration time, and random JTI.""" - exp = datetime.utcnow() + expires_in + exp = datetime.now(timezone.utc) + expires_in jti = "test_token" + str(uuid.uuid4()) payload = {"exp": exp, "aud": aud, "jti": jti} diff --git a/tests/flytekit/integration/remote/test_remote.py b/tests/flytekit/integration/remote/test_remote.py index a1c137dc48..3398c771f3 100644 --- a/tests/flytekit/integration/remote/test_remote.py +++ b/tests/flytekit/integration/remote/test_remote.py @@ -81,10 +81,10 @@ def test_monitor_workflow_execution(register): ) poll_interval = datetime.timedelta(seconds=1) - time_to_give_up = datetime.datetime.utcnow() + datetime.timedelta(seconds=60) + time_to_give_up = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=60) execution = remote.sync_execution(execution, sync_nodes=True) - while datetime.datetime.utcnow() < time_to_give_up: + while datetime.datetime.now(datetime.timezone.utc) < time_to_give_up: if execution.is_done: break diff --git a/tests/flytekit/unit/core/test_context_manager.py b/tests/flytekit/unit/core/test_context_manager.py index ad27ad1852..6d519e2da6 100644 --- a/tests/flytekit/unit/core/test_context_manager.py +++ b/tests/flytekit/unit/core/test_context_manager.py @@ -1,6 +1,6 @@ import base64 import os -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from unittest.mock import Mock @@ -251,7 +251,7 @@ def test_exec_params(): ep = ExecutionParameters( execution_id=id_models.WorkflowExecutionIdentifier("p", "d", "n"), task_id=id_models.Identifier(id_models.ResourceType.TASK, "local", "local", "local", "local"), - execution_date=datetime.utcnow(), + execution_date=datetime.now(timezone.utc), stats=mock_stats.MockStats(), logging=None, tmp_dir="/tmp", diff --git a/tests/flytekit/unit/deck/test_deck.py b/tests/flytekit/unit/deck/test_deck.py index da23ed188b..45056ae283 100644 --- a/tests/flytekit/unit/deck/test_deck.py +++ b/tests/flytekit/unit/deck/test_deck.py @@ -32,8 +32,8 @@ def test_deck(): def test_timeline_deck(): time_info = dict( Name="foo", - Start=datetime.datetime.utcnow(), - Finish=datetime.datetime.utcnow() + datetime.timedelta(microseconds=1000), + Start=datetime.datetime.now(datetime.timezone.utc), + Finish=datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(microseconds=1000), WallTime=1.0, ProcessTime=1.0, ) diff --git a/tests/flytekit/unit/models/test_literals.py b/tests/flytekit/unit/models/test_literals.py index 0035f8ec1d..d32628d036 100644 --- a/tests/flytekit/unit/models/test_literals.py +++ b/tests/flytekit/unit/models/test_literals.py @@ -103,7 +103,7 @@ def test_boolean_primitive(): def test_datetime_primitive(): - dt = datetime.utcnow().replace(tzinfo=timezone.utc) + dt = datetime.now(timezone.utc) obj = literals.Primitive(datetime=dt) assert obj.integer is None assert obj.boolean is None From c8ac2764f2de8d7e98e56bcf5c7a04ab07e617ca Mon Sep 17 00:00:00 2001 From: Troy Chiu <114708546+troychiu@users.noreply.github.com> Date: Mon, 18 Mar 2024 10:52:34 -0700 Subject: [PATCH 03/50] cache ignore portion (#2275) Signed-off-by: troychiu --- flytekit/core/base_task.py | 23 ++++++++++++--- flytekit/core/local_cache.py | 28 +++++++++++++++---- flytekit/core/task.py | 7 ++++- flytekit/models/task.py | 13 +++++++++ plugins/flytekit-bigquery/tests/test_agent.py | 1 + plugins/flytekit-openai/tests/test_agent.py | 1 + .../flytekit-snowflake/tests/test_agent.py | 1 + plugins/flytekit-spark/tests/test_agent.py | 1 + tests/flytekit/common/parameterizers.py | 4 ++- tests/flytekit/unit/core/test_local_cache.py | 28 +++++++++++++++++++ tests/flytekit/unit/models/test_tasks.py | 3 ++ .../unit/models/test_workflow_closure.py | 1 + 12 files changed, 99 insertions(+), 12 deletions(-) diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index f910c3a3f2..7901618241 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -97,6 +97,7 @@ class TaskMetadata(object): cache (bool): Indicates if caching should be enabled. See :std:ref:`Caching ` cache_serialize (bool): Indicates if identical (ie. same inputs) instances of this task should be executed in serial when caching is enabled. See :std:ref:`Caching ` cache_version (str): Version to be used for the cached value + cache_ignore_input_vars (Tuple[str, ...]): Input variables that should not be included when calculating hash for cache interruptible (Optional[bool]): Indicates that this task can be interrupted and/or scheduled on nodes with lower QoS guarantees that can include pre-emption. This can reduce the monetary cost executions incur at the cost of performance penalties due to potential interruptions @@ -112,6 +113,7 @@ class TaskMetadata(object): cache: bool = False cache_serialize: bool = False cache_version: str = "" + cache_ignore_input_vars: Tuple[str, ...] = () interruptible: Optional[bool] = None deprecated: str = "" retries: int = 0 @@ -128,6 +130,10 @@ def __post_init__(self): raise ValueError("Caching is enabled ``cache=True`` but ``cache_version`` is not set.") if self.cache_serialize and not self.cache: raise ValueError("Cache serialize is enabled ``cache_serialize=True`` but ``cache`` is not enabled.") + if self.cache_ignore_input_vars and not self.cache: + raise ValueError( + f"Cache ignore input vars are specified ``cache_ignore_input_vars={self.cache_ignore_input_vars}`` but ``cache`` is not enabled." + ) @property def retry_strategy(self) -> _literal_models.RetryStrategy: @@ -151,6 +157,7 @@ def to_taskmetadata_model(self) -> _task_model.TaskMetadata: deprecated_error_message=self.deprecated, cache_serializable=self.cache_serialize, pod_template_name=self.pod_template_name, + cache_ignore_input_vars=self.cache_ignore_input_vars, ) @@ -281,13 +288,15 @@ def local_execute( # TODO: how to get a nice `native_inputs` here? logger.info( f"Checking cache for task named {self.name}, cache version {self.metadata.cache_version} " - f"and inputs: {input_literal_map}" + f", inputs: {input_literal_map}, and ignore input vars: {self.metadata.cache_ignore_input_vars}" ) if local_config.cache_overwrite: outputs_literal_map = None logger.info("Cache overwrite, task will be executed now") else: - outputs_literal_map = LocalTaskCache.get(self.name, self.metadata.cache_version, input_literal_map) + outputs_literal_map = LocalTaskCache.get( + self.name, self.metadata.cache_version, input_literal_map, self.metadata.cache_ignore_input_vars + ) # The cache returns None iff the key does not exist in the cache if outputs_literal_map is None: logger.info("Cache miss, task will be executed now") @@ -296,10 +305,16 @@ def local_execute( if outputs_literal_map is None: outputs_literal_map = self.sandbox_execute(ctx, input_literal_map) # TODO: need `native_inputs` - LocalTaskCache.set(self.name, self.metadata.cache_version, input_literal_map, outputs_literal_map) + LocalTaskCache.set( + self.name, + self.metadata.cache_version, + input_literal_map, + self.metadata.cache_ignore_input_vars, + outputs_literal_map, + ) logger.info( f"Cache set for task named {self.name}, cache version {self.metadata.cache_version} " - f"and inputs: {input_literal_map}" + f", inputs: {input_literal_map}, and ignore input vars: {self.metadata.cache_ignore_input_vars}" ) else: # This code should mirror the call to `sandbox_execute` in the above cache case. diff --git a/flytekit/core/local_cache.py b/flytekit/core/local_cache.py index 7806452c5e..7cd87e2a49 100644 --- a/flytekit/core/local_cache.py +++ b/flytekit/core/local_cache.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, Tuple from diskcache import Cache @@ -28,10 +28,14 @@ def _recursive_hash_placement(literal: Literal) -> Literal: return literal -def _calculate_cache_key(task_name: str, cache_version: str, input_literal_map: LiteralMap) -> str: +def _calculate_cache_key( + task_name: str, cache_version: str, input_literal_map: LiteralMap, cache_ignore_input_vars: Tuple[str, ...] = () +) -> str: # Traverse the literals and replace the literal with a new literal that only contains the hash literal_map_overridden = {} for key, literal in input_literal_map.literals.items(): + if key in cache_ignore_input_vars: + continue literal_map_overridden[key] = _recursive_hash_placement(literal) # Generate a stable representation of the underlying protobuf by passing `deterministic=True` to the @@ -61,13 +65,25 @@ def clear(): LocalTaskCache._cache.clear() @staticmethod - def get(task_name: str, cache_version: str, input_literal_map: LiteralMap) -> Optional[LiteralMap]: + def get( + task_name: str, cache_version: str, input_literal_map: LiteralMap, cache_ignore_input_vars: Tuple[str, ...] + ) -> Optional[LiteralMap]: if not LocalTaskCache._initialized: LocalTaskCache.initialize() - return LocalTaskCache._cache.get(_calculate_cache_key(task_name, cache_version, input_literal_map)) + return LocalTaskCache._cache.get( + _calculate_cache_key(task_name, cache_version, input_literal_map, cache_ignore_input_vars) + ) @staticmethod - def set(task_name: str, cache_version: str, input_literal_map: LiteralMap, value: LiteralMap) -> None: + def set( + task_name: str, + cache_version: str, + input_literal_map: LiteralMap, + cache_ignore_input_vars: Tuple[str, ...], + value: LiteralMap, + ) -> None: if not LocalTaskCache._initialized: LocalTaskCache.initialize() - LocalTaskCache._cache.set(_calculate_cache_key(task_name, cache_version, input_literal_map), value) + LocalTaskCache._cache.set( + _calculate_cache_key(task_name, cache_version, input_literal_map, cache_ignore_input_vars), value + ) diff --git a/flytekit/core/task.py b/flytekit/core/task.py index a99fbf599e..f528d451f0 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -2,7 +2,7 @@ import datetime as _datetime from functools import update_wrapper -from typing import Any, Callable, Dict, Iterable, List, Optional, Type, TypeVar, Union, overload +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type, TypeVar, Union, overload from flytekit.core import launch_plan as _annotated_launchplan from flytekit.core import workflow as _annotated_workflow @@ -91,6 +91,7 @@ def task( cache: bool = ..., cache_serialize: bool = ..., cache_version: str = ..., + cache_ignore_input_vars: Tuple[str, ...] = ..., retries: int = ..., interruptible: Optional[bool] = ..., deprecated: str = ..., @@ -122,6 +123,7 @@ def task( cache: bool = ..., cache_serialize: bool = ..., cache_version: str = ..., + cache_ignore_input_vars: Tuple[str, ...] = ..., retries: int = ..., interruptible: Optional[bool] = ..., deprecated: str = ..., @@ -152,6 +154,7 @@ def task( cache: bool = False, cache_serialize: bool = False, cache_version: str = "", + cache_ignore_input_vars: Tuple[str, ...] = (), retries: int = 0, interruptible: Optional[bool] = None, deprecated: str = "", @@ -213,6 +216,7 @@ def my_task(x: int, y: typing.Dict[str, str]) -> str: :param cache_version: Cache version to use. Changes to the task signature will automatically trigger a cache miss, but you can always manually update this field as well to force a cache miss. You should also manually bump this version if the function body/business logic has changed, but the signature hasn't. + :param cache_ignore_input_vars: Input variables that should not be included when calculating hash for cache. :param retries: Number of times to retry this task during a workflow execution. :param interruptible: [Optional] Boolean that indicates that this task can be interrupted and/or scheduled on nodes with lower QoS guarantees. This will directly reduce the `$`/`execution cost` associated, @@ -295,6 +299,7 @@ def wrapper(fn: Callable[..., Any]) -> PythonFunctionTask[T]: cache=cache, cache_serialize=cache_serialize, cache_version=cache_version, + cache_ignore_input_vars=cache_ignore_input_vars, retries=retries, interruptible=interruptible, deprecated=deprecated, diff --git a/flytekit/models/task.py b/flytekit/models/task.py index 1da786ea6d..b6e8222fb9 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -177,6 +177,7 @@ def __init__( deprecated_error_message, cache_serializable, pod_template_name, + cache_ignore_input_vars, ): """ Information needed at runtime to determine behavior such as whether or not outputs are discoverable, timeouts, @@ -197,6 +198,7 @@ def __init__( :param bool cache_serializable: Whether or not caching operations are executed in serial. This means only a single instance over identical inputs is executed, other concurrent executions wait for the cached results. :param pod_template_name: The name of the existing PodTemplate resource which will be used in this task. + :param cache_ignore_input_vars: Input variables that should not be included when calculating hash for cache. """ self._discoverable = discoverable self._runtime = runtime @@ -207,6 +209,7 @@ def __init__( self._deprecated_error_message = deprecated_error_message self._cache_serializable = cache_serializable self._pod_template_name = pod_template_name + self._cache_ignore_input_vars = cache_ignore_input_vars @property def discoverable(self): @@ -284,6 +287,14 @@ def pod_template_name(self): """ return self._pod_template_name + @property + def cache_ignore_input_vars(self): + """ + Input variables that should not be included when calculating hash for cache. + :rtype: tuple[Text] + """ + return self._cache_ignore_input_vars + def to_flyte_idl(self): """ :rtype: flyteidl.admin.task_pb2.TaskMetadata @@ -297,6 +308,7 @@ def to_flyte_idl(self): deprecated_error_message=self.deprecated_error_message, cache_serializable=self.cache_serializable, pod_template_name=self.pod_template_name, + cache_ignore_input_vars=self.cache_ignore_input_vars, ) if self.timeout: tm.timeout.FromTimedelta(self.timeout) @@ -318,6 +330,7 @@ def from_flyte_idl(cls, pb2_object): deprecated_error_message=pb2_object.deprecated_error_message, cache_serializable=pb2_object.cache_serializable, pod_template_name=pb2_object.pod_template_name, + cache_ignore_input_vars=pb2_object.cache_ignore_input_vars, ) diff --git a/plugins/flytekit-bigquery/tests/test_agent.py b/plugins/flytekit-bigquery/tests/test_agent.py index 5897b4b468..57d4b747cd 100644 --- a/plugins/flytekit-bigquery/tests/test_agent.py +++ b/plugins/flytekit-bigquery/tests/test_agent.py @@ -54,6 +54,7 @@ def __init__(self): "This is deprecated!", True, "A", + (), ) task_config = { "Location": "us-central1", diff --git a/plugins/flytekit-openai/tests/test_agent.py b/plugins/flytekit-openai/tests/test_agent.py index dd340bd1a7..1216fdedda 100644 --- a/plugins/flytekit-openai/tests/test_agent.py +++ b/plugins/flytekit-openai/tests/test_agent.py @@ -40,6 +40,7 @@ async def test_chatgpt_agent(): "This is deprecated!", True, "A", + (), ) tmp = TaskTemplate( id=task_id, diff --git a/plugins/flytekit-snowflake/tests/test_agent.py b/plugins/flytekit-snowflake/tests/test_agent.py index f3dcb0686d..adc699061f 100644 --- a/plugins/flytekit-snowflake/tests/test_agent.py +++ b/plugins/flytekit-snowflake/tests/test_agent.py @@ -46,6 +46,7 @@ async def test_snowflake_agent(mock_get_private_key): "This is deprecated!", True, "A", + (), ) task_config = { diff --git a/plugins/flytekit-spark/tests/test_agent.py b/plugins/flytekit-spark/tests/test_agent.py index 642755a351..83034be90e 100644 --- a/plugins/flytekit-spark/tests/test_agent.py +++ b/plugins/flytekit-spark/tests/test_agent.py @@ -31,6 +31,7 @@ async def test_databricks_agent(): "This is deprecated!", True, "A", + (), ) task_config = { "sparkConf": { diff --git a/tests/flytekit/common/parameterizers.py b/tests/flytekit/common/parameterizers.py index 84481a37ad..57bfcea08f 100644 --- a/tests/flytekit/common/parameterizers.py +++ b/tests/flytekit/common/parameterizers.py @@ -124,8 +124,9 @@ deprecated, cache_serializable, pod_template_name, + cache_ignore_input_vars, ) - for discoverable, runtime_metadata, timeout, retry_strategy, interruptible, discovery_version, deprecated, cache_serializable, pod_template_name in product( + for discoverable, runtime_metadata, timeout, retry_strategy, interruptible, discovery_version, deprecated, cache_serializable, pod_template_name, cache_ignore_input_vars in product( [True, False], LIST_OF_RUNTIME_METADATA, [timedelta(days=i) for i in range(3)], @@ -135,6 +136,7 @@ ["deprecated"], [True, False], ["A", "B"], + [()], ) ] diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py index 27814c739f..6a9570fd7a 100644 --- a/tests/flytekit/unit/core/test_local_cache.py +++ b/tests/flytekit/unit/core/test_local_cache.py @@ -1,4 +1,5 @@ import datetime +import re import sys import typing from dataclasses import dataclass @@ -597,3 +598,30 @@ def t2(n: int) -> int: @pytest.mark.serial def test_checkpoint_cached_task(): assert t2(n=5) == 6 + + +def test_cache_ignore_input_vars(): + @task(cache=True, cache_version="v1", cache_ignore_input_vars=["a"]) + def add(a: int, b: int) -> int: + return a + b + + @workflow + def add_wf(a: int, b: int) -> int: + return add(a=a, b=b) + + assert add_wf(a=10, b=5) == 15 + assert add_wf(a=20, b=5) == 15 # since a is ignored, this line will hit cache of a=10, b=5 + assert add_wf(a=20, b=8) == 28 + + +def test_set_cache_ignore_input_vars_without_set_cache(): + with pytest.raises( + ValueError, + match=re.escape( + "Cache ignore input vars are specified ``cache_ignore_input_vars=['a']`` but ``cache`` is not enabled." + ), + ): + + @task(cache_ignore_input_vars=["a"]) + def add(a: int, b: int) -> int: + return a + b diff --git a/tests/flytekit/unit/models/test_tasks.py b/tests/flytekit/unit/models/test_tasks.py index b4158c3852..b9685736b7 100644 --- a/tests/flytekit/unit/models/test_tasks.py +++ b/tests/flytekit/unit/models/test_tasks.py @@ -73,6 +73,7 @@ def test_task_metadata(): "This is deprecated!", True, "A", + (), ) assert obj.discoverable is True @@ -142,6 +143,7 @@ def test_task_spec(): "This is deprecated!", True, "A", + (), ) int_type = types.LiteralType(types.SimpleType.INTEGER) @@ -202,6 +204,7 @@ def test_task_template_k8s_pod_target(): "deprecated", False, "A", + (), ), interface_models.TypedInterface( # inputs diff --git a/tests/flytekit/unit/models/test_workflow_closure.py b/tests/flytekit/unit/models/test_workflow_closure.py index 64e5a57713..8181d0c256 100644 --- a/tests/flytekit/unit/models/test_workflow_closure.py +++ b/tests/flytekit/unit/models/test_workflow_closure.py @@ -42,6 +42,7 @@ def test_workflow_closure(): "This is deprecated!", True, "A", + (), ) cpu_resource = _task.Resources.ResourceEntry(_task.Resources.ResourceName.CPU, "1") From 4d42399079535c3582c9adfad5791253dcbd045e Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Tue, 19 Mar 2024 17:17:16 +0800 Subject: [PATCH 04/50] Fix CI Error by restricting s3fs version (#2283) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 09dceed631..2ea7997f8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ "requests>=2.18.4,<3.0.0", "rich", "rich_click", - "s3fs>=2023.3.0", + "s3fs>=2023.3.0,!=2024.3.1", "statsd>=3.0.0,<4.0.0", "typing_extensions", "urllib3>=1.22,<2.0.0", From f45dc392349f0bb39fd1a7a74cfe73757c0176da Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 19 Mar 2024 13:15:32 -0700 Subject: [PATCH 05/50] Add support for Union[FlyteFile, FlyteDirectory] as input (#2273) Signed-off-by: Kevin Su --- flytekit/types/directory/types.py | 7 +++++- flytekit/types/file/file.py | 3 +++ tests/flytekit/unit/core/test_type_engine.py | 23 +++++++++++++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index decd86c7c8..b2a21c4b50 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -14,8 +14,9 @@ from fsspec.utils import get_protocol from marshmallow import fields +from flytekit import BlobType from flytekit.core.context_manager import FlyteContext, FlyteContextManager -from flytekit.core.type_engine import TypeEngine, TypeTransformer, get_batch_size +from flytekit.core.type_engine import TypeEngine, TypeTransformer, TypeTransformerFailedError, get_batch_size from flytekit.exceptions.user import FlyteAssertion from flytekit.models import types as _type_models from flytekit.models.core import types as _core_types @@ -441,6 +442,10 @@ def to_python_value( self, ctx: FlyteContext, lv: Literal, expected_python_type: typing.Type[FlyteDirectory] ) -> FlyteDirectory: uri = lv.scalar.blob.uri + + if lv.scalar.blob.metadata.type.dimensionality != BlobType.BlobDimensionality.MULTIPART: + raise TypeTransformerFailedError(f"{lv.scalar.blob.uri} is not a directory.") + if not ctx.file_access.is_remote(uri) and not os.path.isdir(uri): raise FlyteAssertion(f"Expected a directory, but the given uri '{uri}' is not a directory.") diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index da6bc4d699..de4e49cdaf 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -463,6 +463,9 @@ def to_python_value( except AttributeError: raise TypeTransformerFailedError(f"Cannot convert from {lv} to {expected_python_type}") + if lv.scalar.blob.metadata.type.dimensionality != BlobType.BlobDimensionality.SINGLE: + raise TypeTransformerFailedError(f"{lv.scalar.blob.uri} is not a file.") + if not ctx.file_access.is_remote(uri) and not os.path.isfile(uri): raise FlyteAssertion( f"Cannot convert from {lv} to {expected_python_type}. " f"Expected a file, but {uri} is not a file." diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index d0cfdfc69c..b9c9dac8f2 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -56,7 +56,7 @@ from flytekit.models.literals import Blob, BlobMetadata, Literal, LiteralCollection, LiteralMap, Primitive, Scalar, Void from flytekit.models.types import LiteralType, SimpleType, TypeStructure, UnionType from flytekit.types.directory import TensorboardLogs -from flytekit.types.directory.types import FlyteDirectory +from flytekit.types.directory.types import FlyteDirectory, FlyteDirToMultipartBlobTransformer from flytekit.types.file import FileExt, JPEGImageFile from flytekit.types.file.file import FlyteFile, FlyteFilePathTransformer, noop from flytekit.types.pickle import FlytePickle @@ -2539,3 +2539,24 @@ def test_ListTransformer_get_sub_type(): def test_ListTransformer_get_sub_type_as_none(): assert ListTransformer.get_sub_type_or_none(type([])) is None + + +def test_union_file_directory(): + lt = TypeEngine.to_literal_type(FlyteFile) + s3_file = "s3://my-file" + + transformer = FlyteFilePathTransformer() + ctx = FlyteContext.current_context() + lv = transformer.to_literal(ctx, s3_file, FlyteFile, lt) + + union_trans = UnionTransformer() + pv = union_trans.to_python_value(ctx, lv, typing.Union[FlyteFile, FlyteDirectory]) + assert pv._remote_source == s3_file + + s3_dir = "s3://my-dir" + transformer = FlyteDirToMultipartBlobTransformer() + ctx = FlyteContext.current_context() + lv = transformer.to_literal(ctx, s3_dir, FlyteFile, lt) + + pv = union_trans.to_python_value(ctx, lv, typing.Union[FlyteFile, FlyteDirectory]) + assert pv._remote_source == s3_dir From 3642ec6771985f504c2b6a9dfd2598a84219ac58 Mon Sep 17 00:00:00 2001 From: Jan Fiedler <89976021+fiedlerNr9@users.noreply.github.com> Date: Wed, 20 Mar 2024 02:07:59 +0100 Subject: [PATCH 06/50] Update flyteinteractive Jupyter: Move to `ClassDecorator` (#2278) * reuse constants and functions for vscode & jupyter Signed-off-by: Jan Fiedler * convert jupyter to ClassDecorator Signed-off-by: Jan Fiedler * afjust & add tests Signed-off-by: Jan Fiedler * update docs Signed-off-by: Jan Fiedler * adjust cmd command Signed-off-by: Jan Fiedler * adjust jupyter default port Signed-off-by: Jan Fiedler * fix linting Signed-off-by: Jan Fiedler * fix linting Signed-off-by: Jan Fiedler --------- Signed-off-by: Jan Fiedler --- plugins/flytekit-flyteinteractive/README.md | 26 +- .../docs/jupyter_example.png | Bin 0 -> 135989 bytes .../docs/{example.png => vscode_example.png} | Bin .../flyteinteractive/__init__.py | 4 +- .../flyteinteractive/constants.py | 6 + .../flyteinteractive/jupyter_lib/constants.py | 3 - .../flyteinteractive/jupyter_lib/decorator.py | 228 +++++++++++++----- .../jupyter_lib/jupyter_constants.py | 1 + .../flytekitplugins/flyteinteractive/utils.py | 24 +- .../flyteinteractive/vscode_lib/config.py | 6 +- .../flyteinteractive/vscode_lib/decorator.py | 29 +-- .../{constants.py => vscode_constants.py} | 7 - .../tests/test_flyteinteractive_jupyter.py | 159 ++++++++++++ ...gin.py => test_flyteinteractive_vscode.py} | 24 +- 14 files changed, 405 insertions(+), 112 deletions(-) create mode 100644 plugins/flytekit-flyteinteractive/docs/jupyter_example.png rename plugins/flytekit-flyteinteractive/docs/{example.png => vscode_example.png} (100%) create mode 100644 plugins/flytekit-flyteinteractive/flytekitplugins/flyteinteractive/constants.py delete mode 100644 plugins/flytekit-flyteinteractive/flytekitplugins/flyteinteractive/jupyter_lib/constants.py create mode 100644 plugins/flytekit-flyteinteractive/flytekitplugins/flyteinteractive/jupyter_lib/jupyter_constants.py rename plugins/flytekit-flyteinteractive/flytekitplugins/flyteinteractive/vscode_lib/{constants.py => vscode_constants.py} (85%) create mode 100644 plugins/flytekit-flyteinteractive/tests/test_flyteinteractive_jupyter.py rename plugins/flytekit-flyteinteractive/tests/{test_flyin_plugin.py => test_flyteinteractive_vscode.py} (96%) diff --git a/plugins/flytekit-flyteinteractive/README.md b/plugins/flytekit-flyteinteractive/README.md index 7142475654..6c2bd76794 100644 --- a/plugins/flytekit-flyteinteractive/README.md +++ b/plugins/flytekit-flyteinteractive/README.md @@ -35,7 +35,7 @@ def train(): 4. You can access the server by opening a web browser and navigating to `localhost:8080`. VSCode example screenshot: - + ## Build Custom Image with VSCode Plugin If users want to skip the vscode downloading process at runtime, they have the option to create a custom image with vscode by including the following lines in their Dockerfile. @@ -103,3 +103,27 @@ def wf(): t_short_live() t_vim() ``` +## Jupyter + +FlyteInteractive Jupyter offers an easy solution for users to run Python tasks within a Jupyter Notebook server, compatible with any image. `@jupyter` is a decorator which users can put within @task and user function. With `@jupyter`, the task will run a Jupyter Notebook server instead of the user defined functions. + + +## Starter Example +```python +from flytekit import task +from flytekitplugins.flyteinteractive import jupyter + +@task +@jupyter +def train(): + ... +``` + +## User Guide +1. Build the image with Dockerfile.dev `docker buildx build --push . -f Dockerfile.dev -t localhost:30000/flytekit:dev --build-arg PYTHON_VERSION=3.8` +2. Run the decorated task on the remote. For example: `pyflyte run --remote --image localhost:30000/flytekit:dev [PYTHONFILE] [WORKFLOW|TASK] [ARGS]...` +3. Once the code server is prepared, you can forward a local port to the pod. For example: `kubectl port-forward -n [NAMESPACE] [PODNAME] 8888:8888`. +4. You can access the server by opening a web browser and navigating to `localhost:8888`. + +Jupyter example screenshot: + diff --git a/plugins/flytekit-flyteinteractive/docs/jupyter_example.png b/plugins/flytekit-flyteinteractive/docs/jupyter_example.png new file mode 100644 index 0000000000000000000000000000000000000000..4021e522ef4adc69f0af8f0dc0a20ba1b44e6af0 GIT binary patch literal 135989 zcmeFZc|4Tu-#;voBDBecM5&auEF;7ym8)b6V;zz`X2{MMONvUUY}rZK$3BA@gF?zK z%M4>o$U4SYXE4k>({(@heSfd-^}D-YzrUa74>RX6=XoB>=Qxhf^8UOJFN_WK_<2Qn zIXF1@@BDf5J_iR6jDv&g^uYt{Uj|+pv%k0m+;w%0J#_VSy*#~qOuZc(o%Nid&OYvr z_w{aYa9oXxGY7k!GC!f6(NH6Mt~Y9^px~_lQ#Gm9^Zvlov3aI*4P;y{(X;mY>u*cM zEcXk_)4LzsFKM5+e&kcpkHjA=n@`eb9+gsJ<=@vDzvqg&xYfU-m-k)x5V=9Q29sFt zf$bXKp3S6&%}w|&Pw>62m$ONkIL|33S}s2`I91az+t#Mx_X*z55ATP7`I@G4=nJGDS ztQE;&@kh4cN_Ah_8>Q#=x8yFCykfQyp7YE}(X6?6d{+qZ*HnuTm(F(G`UWUXH8Xgeaqgn#=GLr2X}@R2iR?$(SQK=0C^%o>wTi*R&sDIXAc;T9x7eV;Z0N z@UCjNIhw;7PtE%km#7W2$QBNzxgH@!1^Iuv*7;@m<}UJzeI~De*!|4Ivx28NeoV>8 zSBB7~PFEVZ%RKLjvv31+tUceVpFQcq3160Xb=Sc4tABKPd0lQfc6#k7s8GAjqBuMC z#-6==-H9*fwW94Z5Bq(+?7zFY6JU{*c5v#gx-g&UtMiH=Fj1wgiDYn{eLhs_f>3n(`yd-q_!SS1~B?a9>Z64$o@`}#8<;{O| z%i9|Mu|)cOX`9JWXZDEsl{`Oi`9=XtYNfyAGu6#ps_}4Guv@8r{~;G|si7-(6z{x^ z+B$VM@#kfcGx?HlrN5avy>c4RxGSLAk#g48X}w`!D0rXL{T#K}#@PA5>tC3zPVauy z_O;*5Q9FX~!}ob~CgY%X8gQsZEao;S2Q%Wk;CIed^pU*Z*Ri22pFY-S`#D^t{hXTh zVJ1)5{%|q@RHsT-=?3w_(Ox!fdh1$-PTC*k%{?7z^50vjf_6A1x;WqsL70QC2W)nK z(!T-UdVHh5mADW-J@Hd}ve!4qC+9IK;w$)vf{(qA{bLE=9KAnd&yf zQ3+GJ`q}Fab5EEM{L$Gr=fKrnX-&E{N?q_r+5UcpTNfb0bs9_MDAO(oqTLUYZhJ`L zygD4!`pLpbw69cxylE2zJwL&L55?NC7<(b`_5DL6j{02Ie%ARQzIMlvF1CX3bhft{~g`8y&@0f z;Lzc?bMv}c(7q)iPefAtcE{@6GHuySPw~m0CR{@Yi>ic|uDpCQ&@2jzJ^yG4+jh)D zZlLX09Qy)O6~PTFQxQ5Mbp6P}UdVQzSG+ncg+L%;um?zYXcKT!;6(7;-qLdL(z>>? z2#i|g(XL5_yPZFOzTkVFE)RLw15O?)jzzi;emVQ*xwEtLHar=qH2(hVBW$^muBJnp zr)Kd;0uJgXEjK+JOU!<>ho5c@VV7EIH@nQp9Uh9c$5U1p+o49tu%%T8Eq2KP_TIT< za~df;cBm_8^I=_~HITMdd753S)EZD4-dP>vqdawP45=cRA|zm;mV)dOKHDwr(OwYc zS-@u{J1m~UZc|IUEQVe3G^)La3~Qv_&~y-~uv?8fXX0%c^{3 zp%?%w*>p@0B(qD03CHExRaap94K2Tv3S+c}cd9yKHrb_U%8Qg=!?FAShx7l**=Zcn z6?3pM%}Pgov~Df{)XM)K7w_)Q$B!RDAdt)U_BP7t+t~~)7)lIUtPj~xGGD%2yQ$>>kF}wphr~}X%CjLkh25=@8o^WX zzpwh&-PZ^9Ij^I7Du^cxWmyrhcfexjfslpl+79Vsg1$QeX;K0fuE6|M4& zzQKn_Cw%q4b%x^?#a7Kr#`puBC(pycF+x$YBt`_)y0p}6r zx^Xq7!i$3!S(8{V!zJS;8&$2MZeY%+)hYf$s4Qtv(4EIUtbuZmhB#}unuFDa@+u?=B{4Ot@< z{Q(qPr!m|CDDs55-jUS!wAcYtHs45R%c1RPOvn+Vu4{~sZzL_C(RZh9zLd};n zfY2XQifL~VL-`gtA4;OTC$cn>M^*QySdssn>8g3J(v^}@((EEWjtY3wNtX8}D6q=0 z@XD)QRQ;sUjI1ojj#KcbTFf?6knwA3Lh*tgsR<6rfS{^fvxCbEh@P=W@eUY>ysT{3 z1ZyS#omLr2C9(n7=lNP8Cvc*niJ0Nl^zr&wp~-Kwf+d~nd-}1GR}<1ag4JgzV`g{J zEoGe8yA<;aCHLK8P#)x{Popo#FI?=Lhp&wb0BXZi__|%tfFZMJ^DXJ*zPfpds=_`G zuhBPcluP+_>&av*r)gW&>6c#pXQ6&h{I6}7N01c+b8^}uz+7acX^5%h{S4lyf>X#I znV{`8CQ`Pocr@K?VXS^%$e{LcL%{S*f#cgt;u^Jee0{(!bRZSvv5BRP6Pcl-w~!;n z*dpBi9_?+Zz|MrHl<8 z%w+`vv@ZTmsp_D)zKbqtRqLqiesSc7etX zPZ|`w%OYwupDDro3*_puktQglZ|gO+TkbQ_9;Zk1+v2nPdlj}(ao8Hk`_to%=@k|X zv$>)d!OTCz31{!7d=M-5Y+U7F^i04;s9elFo8}8w(T};~wPr7B(lddG$M+$cv+tyE z_}&A*nQNWgk9;3g7XvJDV(dFOQe~PHuCP4K;Qd(HRSC^ zgKS|UJ2_J|+B&6jw*0&ZeubZ8nnbmuGkEy9#n{q7FK(E580YpZyr@E9o}68J%isPZ zGNZ5V(p|fNXL~aQ5TK^g$8y$bCl{@8yQ9Ebp z`#xt~T@I(k98V}kDT>QBGdEM1yLpHa1OcB^(X8S;N28MZfX%dYS;(T=wC9?IJ>Edg zj$At7J10(C<1>nB^_=fN6UO>#!hA3)L55kasd)V3Nh&H~4JKbpnKL1|Tk-BNGs>#z6-n>&7o1^S4q3_K3co z9y8U!iBG2+UmdQS1%+NlFIFDzI{nTRIr7eER=9%qRbkNmP5X|5`)rZAcA&Rmti7S| zrS6*Ey7BQ-41#=e6M*RP4GLEnSDC#+E4E<_<^A?I;DXEL*}8pb&b z6JqHMz|?Hmh_)8Vi-e^-;Zv-W&JO(|a7z8N0C3kB2I0y?J>HXM@g1{YD(QhHluhcAzj|I*>+Clsqf7Uct8bbivqdsCSzn-3BHp6@J@8lmK^=) z>&jRxtpGN=*O`>>ibTJ(lN`1#GWWZ2{~61RgbLG4i!jSL`# zrM4m4!!(kGeVwppmD(Sz6K1^hj$W@unRwAjfZQ zfw}S`v8lSstJx((;IyZG__{&kh0$!6ecYkuNj;@&eK3j23@+8!X zf;Z`zuzhHrwZ%j@3y#^IMfY6+Bd&mc<2U4v9=zt)hY0AAXRTca`!5pfm}JIDPOP;_ z)=pCG0a!;xcO7Ny(%^j)Gyd46mwY8M^%)BVrMtVG$<|?E+SqZF*u0My z*a^@op8v|0lnwmi%*W8FB>bG^WQG&~ZR8PY$)Pr$>KU^;8bVA<;Y`pE{HFdDckbXl zbg2o=(F6fO@M|UuF@8=9j^$q851;j4QcX@t+0pUBH$C_iDt6Ud#}Z*}7xbwCH5m;r z9ieV9v;uHibPxhZ|K^$#ZZ+H{NHVI+(*@M=s@>ZXwvVrg5*R0asrQ@^5Ff*o^@f`d z$A(qMi5XhqcE0IRZ+Zo!bs=`q<+5pSzk2j$HkkKxGq+vo*(H^EtpI7f&BxEjf2Xq_ z5RcVUscLwM-ukaRp@reV;T`u-sVRXvPeu-{SJiI?#nhyI4~An1qqfCqf+M>2a1DHGJz0F^H&&n#;z3 zz!v9@RgGt!lJ(I&R_eEe9|+s|kzcYm`K^f|@kjPZy_qJ@>7WQKbLS76rqBj|%6M^! zUD||HxnDq9d+bU15|pCZ{fOX+#^5ldMc<|;bGP%y(w?C9qd?X)K$E$ZPh@fzaCjIv ze|}m~2BSl1gri7$O@U*6-TAe}asN$!|E2x`p8n>&0BgVF*SsD?mgHrCZ&xt*P0}XG z&|~gP4w=ctVi^cAr4YUNQ^PI1Dha<)7CpBf%qEbm4sOZJi@ij-+rzs`YzaF6)8Ji4 z1xdBX&y)hds4~9V`qfA{sKO}2Kkc3N7#R>gvseUuzlMi#1013?TaYs~LBtv*;aBEQ zDi{k<&Ac0iTUFy`Y9>}!T#C)_$yS}h34w#0-wntUFKvw1k;i12d2J~PRw+dd2S!Of z{)YSJ%wppQs*0092pMG?*WPr>+!Qg81RuG&o5!<^gc8xC?ZVlv6x$ILC#pC_i9Tiz z+Y}1eG*8_|smv|EJ+BW}PP||)@Hlj#ea^Pl+szdqaWxO~etN`FMjB^N1?Ef<&tZIjSUCQSLTg=tJOiU>0OG1vN>AI%}N94RtMKrWL zy_GK5F&aEKqAlUO-!JCEZ$nkZBb-f!w#4%x2Ex$!Gn9qu&*ceEpL1Alt zcF$FS5*Rokk)>Z6~ zK!bByG4%cY$@@=)ZMS3nn|!ol4}Y~1U-91?Jkrbi?j+peVXUqAFyv)*IXG2$p~R!lK?&`M#xbn>WBBM%#vMtV+X;1D^6-^az?}-|CZS_L<>@t@l&~v@jVv z9dw6sQW3plNgYVg^a8~@Xou~dQ==~fOM3UJGExJD)$FNQx|Ssp@-{<}p4#9`Ny(_F zi35!FLos_)RBHeMZyw&%R^GIBe{f|;*v{|PuNkuS`=mY$un*qcTt0<_^<-)G8D5*T z)ZD90+*R+z{7`Bx8#yA&f1%QQ@}XK1eNX$VnBlS66>z1u3BK*v?;ClU+wPin+s4(O$xWC*BKN}P zYza<#olZZ$)pSxw5+_Eb9Qn#r+FLxMJt~OW6vXB-MGoVI(v#0HdARLWz9NDZy2BN3 z3O^sRv6wh?Mx+;N$+(Y!ukbRw@5IMhB{?2XaTz@;%oVSYRYMG|6z}w`=xs;o79P{| z@UanVmYdpNJh1sTxV#(eJeHC)bnOHx+R_f?X@%XNs?m2qYy{w6ZjtpQ8`=zUsb5 zo{aWo&ZG|elFJ_(9}_zk>`$G*KEf}YOV7WneEc0_z5Bqz(blVK2W8{V{*S;_r6Hfn zpOt}GC*yoqi@9LKZH8H$OX>T%%KSelooxOqR0JIe)vSk5&Zax#%UU3qT?~!(X{uB1 zfpbr}{CN$0>ER#qZ!-v;+N_`Ou5T~!NA+?Zb{%W7?o2MF~2(+%WQ>bo{N~!BmfqN#bo1@9y_3 z)vr|ZlR7*)u8ix6T8nd@i%M}bD@fVt25YrUSn zf4qOb6RI)s^kcyT$_=&TWx~4ZJ>MDQo>H;mYzihsvyY8PbP)Y!unn z>_K{dozJ|Bc0|w5nLM%I)cyL2&beo3GlLABXu->uzJ6#q&X$9y{ew;ZlUyNU_BqP` z^RA50(cn5rE6}YngqR(82jaY*YM;!J&!ID>E z`#%2fofTqt*2i8m0k2qKXy?;u5xN> zN<+Ns`ai{?)5>dBFE-dU9w;1g!p6`rex~*jv9j=k|9@=Jk$u0$+!EV;{%l0cn4Uv9*rHBvh)@bSv;kNbezE+IMbsk7 zxudO1@_0RoIl%u5f{jGp$OXq3reqMtc-K-1)$M#`Lk7BTLOnr?pI#k~t#?1*V~wm9 z$UG*EUR@jbyfdBaUe~{9VKL)Pg;>qbA zHg%oV<7J(QTh1;yLMfwBd@q-h&wIGsLAn!C)I;}6>a*qcK>p}_e?vxcYVzosvx4PV z=KjXjM(_`{3b-_#R48YIb_;8??s=&BX~Jbyss9lBja(>a_n?iMTKo-mU;Y*xMsD!g zj@88N6~mwie$thSujN^Z_6UJ>I%xXAFD2!+%Go*JZ@-vBTPH7a#LVVt`aJQ-P`a-rZrE1;LqkB>M|x>s4*%ZyjRnjb``*k|Y)|^O zu70!J4qH7O;@TJ$%wdd`gl#b2jRh1|Rvcm@WjClrFaOPw>*$}TU-``ESebO^HMLQP4HuT} z`Zh>0wAq9hg^WHjPpGSi(|8arrs4ScU<*j~SS-1GC+3)*LWMZzl)|h!9cs2ZTz`cBKby!(lQ}c2|bGIRFXVA{5xhP=FUy)1*e-H<0X3@xB6sttt z@y~T)h5P4N&!-J*LV-HvK|wmGKPBe!Z~?7^Nz@jq@oYOr*2K0`S3#gkU;+mtLqz4U5%c{>X|u3A{V2aYGVq#Nr(vIeCuxm(}V0weF8?o_vW~z!`Yddblb}~FMjAhuay6F1& z3q=`rN(ka8YcY!{SzQJh+DgmHK2%p%2N~xEJW_bESPLb7**i+Y5s~buYTR4brrtYG z-m1EO_RD)zIj$zv86EDoyX!khG!_~b&;Gj^;Xmsg(Y2FtEx0;-o-2fjA;ndTc{LVa z@3pYWclEne6GrJoctZ=4vwY+IH5Pegv(<-{d~K{swZ$^@r;wi)?5%F5)o7_zeAuqw%xzoCareb*NEv zk}Ir1XU=iDyF6sT^BKnV^D|KC&R{Qp6m#wkVhQb0Q4NlvdUoUl-M3~>dEJw`GtV2o&$Em$)yvM54oJmSV_|Eml9b3}c z95O945DS!EFL~glo5Oge+*{cmD@<@q&^p;;K?y$h7D9m3Wig`AM#Hp)!9qe|PS_*o z(q*WAfnheIW3^Y4`cPamV1tsBoIJX@ul?6AJ=m1+h1<_Nm#U*v4Sn7Ghz@5h#! z7>;%J2FT0JJusDMb6Lv=ym@A&jGgrHkqb&ZfzR6o>GGoO-xJ46_RD zTOwFh?hw6%(-6Lsa^3)#RUMRAVpuht((0L~IomJ?HfzkjbtUmgH^0F%or;mRpdP-2nc~@-!i)DFg0# z{LswA;CA*rGbqtE)J<82vIk8N0Gmyqjk;2!WrkA%N&a;ZrJ&aoG-G)f)x5RK2eLGC zi#Z+eE{^q`I9ydS@4L}8Zojm3+^|;NHw*<2XV?UqcK8R(BwL{K2o}v9)W;-g`uqe@ z;M#oPWPh#Prk#Gfwe+y-*);kcPoBZkOOO&BulmXY_L9>*wihe2AstXGd`A(w)iN@3 zD;ARY%ZZe<+ULw(pCRGzm_$*6nV$b(@??f5%lbanw@<5UwkN~FqcV`4s;#oahK#cJ z`XpzGewlu%0*U~|Slzkv0?x*OnLdfdHS~$*Dj``AK31{mNA#RbK==nb5>iJT@nVx> zQoN+oG)jt?-C3GAiB`eC+m^_?k!;4vymAndHtwrv<~UH9R|rsUW-jbNjY z+w00Hrgl#42!eGxp>OO7th~-J0shdrrqFtFTkjo(X%l-G-#vD-x@@UaT8_Mx1#!yo zfk7TSKPVk*|KNUJqf|{rT^{%_1Emke_xAC=pdC<4;g`;R(^Fo*`!HvqRL4tJtz%P& zCtaLDepS+F7uQVtG3SE2=B53|E}kQG2G)KaKqCeYD#cU?{zSj|ZdRiGW_5&Ua7saD ziCiFsFI5lm5Jg)__tW)(dmD-HwQtgZhiwPLdslQj8rOR3bHKi)`c0<{2=p{U(7vl(7`2Nwn!>MdhcQ{(*+Q4OeaSHlWzONHobB!&k`RA%_ z$a94f(85;i$k*h?3Lwq**(8VGy+33m)255qA^5=|3(1iA+5Qn2^tm%X@sy33S~pn? zQ&CRVLu*R05$tM);KEtD#7_|#y_sZ(C^i6$wh znpb(>0nkb4#TSc35)BOv1+RW2%UbN{tkOvq%)BFGcM1%z+6M-vj%BJtebQC$M(;gZ zKNHY@orgdT53gtFhKG~K{5P|*@^((zK;2_bhrFw(n@O>0=rL!R9WuB4yP1@~t7Fxe zo?&q-a>9ME2{FRDT(x9UL4bA`$N<6%$zce@5btAzdX zg9ht~X(Cl?twtyX;R#nt3}tqN=-t-_Ej!Yakz45#YLl0dSxLB`npW|C^X>PCHcOmFj6D}4#;{A& zqpZ)eqZ6LZ;2FB5>afMJ$d?7%-{}Tb&UM7-4(LO;{4wKW532>a$+`#p&XEsg9BNIVMCg`7 zYQ$hqwbU{+!Q9!Ic^9)zWgRT#N0l|EoeB29Uhv0`o`HOtm_0VnUvby*Kxk`d<2R>; zF9!m3$3o}Teua0rd0a`wHm2&dw#3pNl8tGWpC`dNHH*SFecoypUB<)%cU$?Tt_}b4 zDAlL~n@z#ocSyl=24q}Ch1N#D6x$>GgW%I8zeK5%oePuD@GHW|k{_ieYmlH#+n~U} znR}Y6Pp3V9etq*8Q#Y3l!?RNwUUcBfYc@A_l&$gZ3telPO)uSB9jjm#jX&;ypgK;& zw&v98!zf5xHf`R~2rm@g)Y-#B8gU-co}+^Ui3T7`OHXzd$!CvM3SWrk8L&lbF4yPx z)o&(`3+{a?yJrm-$t4q<5)Hr{$WroPVBX!&8Z*m+EQ#>lf#|~`54DJm4Q#VsEgpdX zBms2%a=7_kLA*?f;7-gJ#pW`RrQuzyE`T)N#F~VKTNE-^^YTJBLY{rk*ApbyT$f-> z#VUrbSbbW~YmJU3%Ka^=5qVN}C5N@!(bpv=Hb2b$EHAT}xkD~lVkZjw2ezS2eKXvk z$2(q-cERI&m-w{dL?!8P_cU>g^-l{;>ut?G>q#S1xLe=?%MMw_XXMR_m#rF*trqpM9iT zyd6I=w2W`@i!3i^Y5Zb5Yjm70t~+IJ5QfRHv{7?7>`QXzCk0D)^#-W9)l-ARDA9K( zh&}w{X!G*qq~!Wdc}O4~YXybIBOhgvn;bj+TWc*`G%kc!t9l5WuAS|HqD#Fas!Ahkqyw+^s}iinZ8B;!|wICv6f648+tF zzefTBmMkd&9kx#Sx8v6OR1?0gcCbF}Niog?hq?{Iv7@s+d!sahm%Z>)FnlhhqH(c9 z5V-5cShh*?XHbT`CKj8C_6P@8Zh;ugFq*-evHV*@HI?Ei-C%e7mPw z37&f%Ik#X7=To;amvZ*{YQmUcRrp*Mt8%>$NN}Z0_zl{Z8g6gTWle7GZHM8v#Wn2b8V*k@*|5_d!;dGw z*{X%NmsT`ctAQ3Y(rgWY-vEQsm|;dM5O zUEZyl!iGUOr9Q3CWA!Zt4!7HyER{_Z<%#j9o7D3SJUjxws|WO#0g(+bI@}VY8<`;<2F!cBPqx@g93PZM_@t5 z_Tr$GP{2y1!=21LnhdvsW8l}>jM4q9`eEyaDRK#RMUU;Fy#q``$CxFG%4bWFgg z;N){x-1^far^b5C2b+#C`VSV0rUL+g`-4}~Ih7BSZ3n|ps$~(JjWSiO*9tZ9rFrSe z?V295blPC|f`{!8O_O4!n(4dL%b+%sXhETYYCT{HHgaB7iX+4^x9T3THSLZY!LO#r zlZ-&29G2-V^^d1ZD6EYdBMPF9gz8ejbPLYeLR`}=9v;^@uA0+5_n`pr{n!$M{Kt1y z|Drq;(gWAThCGTl7+*?zH(S+ko4535Y=?erh9?o6=+(|w5@AJG@THUmnf*LaPvGul ztW1At02{90*Ie&bwjJ@*6w?lsX6Y3$TMo~l$)umKI#-a!SV*pq8`n#g2a_)drJ0pqAXgaVt_8m;Iez9(Hd9$(?iE8s~tI& zJhsu^kQstkA&7vuN1DLG2tmS!%};t;1mZqay9({@b?=$vy)80K5li=IhSS%>=fjtx zWiDNm^Q$lq&$(`EX7+Qg@Xvzy-j5B|Dt+Dtw{XpNeFCWA+hW_fu({qWIPy=Y%^Y6d zV=+1v0ln#}UC#1eOUoJ%Fa%~msp0>H6aIn`|KmeaHS*gtAwVC|gMf{*8u1Z+_qerX zmLLCJgS1^P52V&99BGX7*QDE-UQ-S4EIj#5vG!%-;f(rogJGAwvn(a4+Ltlnh@YEJdX8ItL?*GeY$eoF3n>IeGI+Gw5Flit3o==+mO@O*vqwdInY z667ND`TQeySd;hU4Pad{Cx(0Zb{KpvpE{DXPz2#OIk95Sp_bMVB=o@Sf#tVnXN<>> zz~A?5ajw4A*c1iaG|3I5l}VJUV#BCm!y}FsO=-Jvtgvl-enOS~cvA5B^X*WluXQEn zgom;6kX(J^OOJK66GW896q-KxltjdONHe{B!o_7GP zfJ@KLH=f7GhX*1>qk}&`J)}wtUO%G?@l^|b@#OnDG4tfF6FhlY`LWec^7VtTe)NA& znn}Vr)0T5OM7tKIpF9_TJw3ZS)L*W(lfg6xO-1|kzMCo7w5r0t1iw~ut;|TR>EMvH zkVL(2n~*KqRjXir1c^MnYt;O{CZu_uKtcgG%SxgJ(JSX*i{EO>g6`yzP^KP1+{${9!EQjgumtR_u$43AZ%tFBK!ki^L zoMCLlK33Q=^Ne9ls(9q!CgQ#Wde?}7j(Ryh1!7i+sJP&Dh~^sm^dn!Y++%ngOWev? z@i;3}eO)@|@=@@5U^4@}QIc_=BF@+tG=d+ghY*Gj37HZP1}|Lfd}PLP1z=iRfD-y) zUUutofs=Bi$D_h>!ijhJD)B94qrfe8j(ugsOi4!Rp8r(x6|bg+7$3VTp9c8asH=yW zCV8pGa|uH&uhI&Mn*F9tZSQC_aC=X!P|3(`|E z1#0nt_xrJ=@YJ9`Qt;EJR)U|Pvm}FCacDN2Z_D+lYin->TjfqJnEfJt`9VaNF&LgO zwKCaiFj^cdtcYS;;;%e#HKL#YWq%f%_R`Gs&!M&+f`-}kNPQn9HZs4SwQ`pKogrDQK&9x+KeKX|sXr8q==!lzVAPM6=sK7vV?-kH03T%lhp;y+d1ONLUx^-|VM9w2#kt;7Edzc!_}zN3BL_ z&%_=EEWxO0Z;$Zuy&SLpss#Q5fp;YpMIhcDWbRSJGYYt46$qZw(36jE`MxF3vW#8o zbsocGpA^0n#SX6tLY zVg9#z&i^31i=0@w8^+E7m_PkN1>dNR4X^Cf`m$e$Gv+SHzz@=7aU&&vAx7rVUK zMIt1j6LTt`?yf-cOMy|beLY6sJ10aK1sSPy4$K~{dHdC9;DiX-XnpIr(N4td{6NL% zQN2^p!mI$a*6vCa^xZUNkE!lEyH!T0>D3-&!&nlGt!crsrsPB;wjSQnPe{iP4>f)<=*73FJz31!z9!+jZAKc1k`f)5vmikW$H^l*>^t{yH^ zVgtJWK(T4}?o4#Z?hyTQ8dRlrDH8;}TQ$ODhfCl5MVdpL5O<&Vum5=tr{O>UprTRx z3%6Y%+GJvUi?S(4ZLHDAEjT)Fwn4P@GXYRtmKoy7^J1+0gt+pYMc=rC8J;frO?v#k zUoODzRqqYZYaU?=sP00Rpx36W@>(kVb7B&Yl=8W~R&v*g$2I*qRv3&DbLM-Oo(3l)0Snre_1vrt0Ml!{Ds!#DeUF z)?7gYQ~rhV6OncU3-QPIZWK`rh4!kygOAGiyk}Dm#$=CZQwLaQ?$ISK`45;Dlq&@b z{XiggFYjsUTYxH-AD9KNeJi|YRTaKFeF-{2jDi1HO*GO*vQ6ta71!U!BkuQ{6i6oq z?m*V(6f2KP-QJ#K^L-;BY+x#2IN4&$pT97FJFtr=(VO?wcevKBszkzJelnlfC{WSN zED@BG@E>}k-MYnIDJVA3HY5?YG0-vAr3mz*PrwrFExc*n%6(N215-bC_Nb8_3TkgD zVkjt^P#3at-!<|a8iGS*kt!s91t0;V!NNdAFMLt%CXYm;1nBt4z_uNXqMG{b+Bh4~ z5)98QzrJo7s_OpM-l(8=aXZ8UX72x4YalD@Q8*PVyPsv>ps69$&2Lvys-Eb4YLShe z2F7XL$S3^#l?49ph1-9hYlCX;ynaC@QCe;3;1lyX`%NM};g%2pajx|MQl{p;VcJM9 z{~^p;?mpDitSiFvlU!HhPr(sDc!Ln?g(Vd90*=0hOSyEvEPMySWHy$JqOYkJnnH(d zvBC<=P)aJ)Ly$%4Q3WIuFwH$VyZQWbFW*(edfHR}=xw+y0{}dUfEqXNq?0MXGBAP7 zrIi-iJ(c=7N{l%?FQL)PnMcKseQdz|wx_9s9`w!REWJ@1qE<(SC9jwqCRMe8>8eDI zUK5hPIjtHB6vl`#^%8=+XcMJtjO-P-p{Iw`?N{EU=D4DeeTHxV_j&D?lD-z3HE|Q?DS5RrqxF>Ql*3OQQ}YE?*pVuupKI44~>) zQynIB*wBoIFJ05@#)q`p`WQ8?EqYX>pC0&FS=5grIK<2wxWi6QP>WS82VJGorjDHzQRLVKThNMH(!Bjyu8=5ReXbFOeS2#HYN7M=2&S9y zNHg@np=PlS*mcJDP0Zu=5_MbC`e;xJdo z2>YycVqYk@>fx;=g&gyehM;NZqVmT&Pxw2(Qbv8X80@gwqpWx`!C!4=yum{tyv&in zPWF{Dih(%XZePiwyCK-QZ=}{<+OQx#+PJs(8%^YsMnS=`irzCZrykkEH-R4c8-KdW?=VY5(&afz_H{c2D=I$-jmZ9FZyVm8)O&Q`Wi^k*wW0;A6#`pB^|j z7aJ;FpG+4;lXK{J2_x-~1z=U^U?aA(mnhx0^dgL#nUnZ&MnS*QIFdBtvP^{U~Z<6gE8-77VD6>r=2%YD+OYd~dPk`&%l6@%e0Q z{iWyS@r{W?S19ApEl+TLm(T4Ubf@-pto) zc4ue*-Zqb7o8Sv0t`RKe&Cy2fd2$Le*;ra`u5Y&-wRnfB$><)0^ymK6#!P>Ss|%l? z>wJgcKqjOZ8HHn0XC@gYaMu$2n@#IeAW1HcP~npbFT=lRE=|P|_?zg{;y(iY{$ZE<(vux|sS1Y7aSi>=5k>Z*{f;n`4`xy?7BbwH7PSPTqgPF!q3H=E0_IDC{rHgPk_9 zYDv(hSz2>q7h$OrVmo;my6mh2`H;VA_1I^tZ#>h-kbCUB{mx<1*y5#pIy*t9a?RlU z4(7h9i1?2f1$kc%?YM>pvS%$eISDoOju zJZ@UnQmYJZ?)G76S^KY>Tn*N@LuaU>FOj@5Qn&sGUNwymHQ1*}531xwFdft8 z6ZUb#O}7&qAw-0Six&h^?L|3mTv_MU%%sJMrK{Vytd0c^+aCS9N^ay@Jh9}TdPFI7 zT}gTA{bncSr(t_a+T`s{pDG+@Y7cC>(G%lwy&xHVk)1j&mnB_v#e}F0pUFqUt&*-{ zQXhC>i`^w7b$%U`{K=H^oUc5CYOi!?LKPg!xE%IAC*Pt~yRfFVll%ap>0g|lXeQEx zWTXA&GDBJ@+=0!sV?nEwn zZv?&{ARxVFUr~>TRg{Djr_6Jm5-%+qnJ=3#BjkrYjO-`?O&I0%{y4T(078z0hgO7(_KT$~ANj{e!8+CXwaoxVWSs-Erk2 zv{wp$@zy@b9QoFZ6i_II=T$$zMdxLLL1DtFh{hr3@zj(PA3FywZu*=4@fX`P_ZjM1IvJvyVS8GcDOUSX=|6KI@TFfgXL| z5LU`U$Iohn`c_Ke#m@bed6GBQS?Bus^Ji^K_Mr3M_WWNxY}uuY?u14Cb(Fs{;E)_x zg(Nf1QsJteZH&8VJ9Xbza2L%i&RMKrTYs`fI?SPWVwTA}0A$xBc_{MpZZgQ)I{l_W z$-np<|LJ}7=t>@YVaeu!W$3xibN|`{I#CDR1V|vrSbYY}xDzg%v(q^ zv`mrz)7`)QR#c;N`U*CP+8(+NOms;$qRDF2k`|8>0k zxMaB5=bYg?3xyPNMf0z-4^y%KsQ4cO?vLPn69kdfx0rb@6~EF|eM!RRlYV&3)Heyn z=8BZ-|3leVKvlVI-z$oOfRuoMw1g6e?i2(OkPxK18>Cy5Zt3ps?gr`ZPU-H3|33HK zdw(yy_r3oZIL5)jIp_QK-rrtp%{kZHAv@F#3(Vmhn@TmS(S}Atslv=!9YKXLr2jF8 z|2b6;;98MR1ru`C-4%^id@F=Zv+eU23h+tioelVDY<+l5!w|*)IT-)jaQ)*~R}b`# zOtZA|RsKX-?Aq5|Oa6#+O+1je!|$rE2XN_~Ih4*rwUP33w`>j{fLYSJ=W2Jf%EX&P zUD3HaTT?O9Hw=BEe)-Yh&oe`K^P=jW9KR02g?z#NT3wa%A@34rvkI?ij=k>r3%CC| zelID5tkNhCUK_^GF_X_2L@Bd)$)-(NYhUYFZ5U%9=#V;zmOWa^b=+@ye7aLHKLt=L zi4!>>FUtd*q4k)&?qu>?U!$X5mI_%_r%{lCuubb{#4VME-u5z`7?0)3k5!n?h2(EK!I{SAh~tJD@VmI<4MP0Elf&wc%)A!I|sj%Wetot8IoDI!zDbLcY%l z31&)85fyXxIiRy>{1Gy~(tLV$fXmfR#SS1e3heQcd3?SPx+*X@SO~?% z97ge%n|3`O``k!ycP{jCfSx_E^arNmL<=&H5ul-OlUgtaB-IM(M8upFP3uk^c`MSL zDb!8@Vt1vC!&j4g^#G0pX_=ah{uzhP&O$<$!EW z5fPD`c$PErqP@`pRXd~Vjr8Sh1(3ALnd0HEOwWS%{r1_Wh}an7F4S9xe_vKaP{!KS&3 zqqXUpy?R4&&I30n{Hfx2@Z&N*yD306QI`4jX2sWr>Ts|zF;B2<4exsrIIlkdNSRCB z4tURre)i;xD(?1iMgAZd16mlnbW9cdnX>v*crgSNpW?3)DQK_ z$QqSniE_MHi2zywIT`UL6Fl6mLiLtypesSl`B%>V-!^bpLG(;{v4GqAga(`pSU@jpFiVknuSHcl#biIz1ll#kudy5 zkHqw|fFAB_Qa2%+eZ%&!X8xuRx#x2q+FM%U6p8ik*_1%UZjm?7om)avXX&S9aK$)t;zjuF=OxXD_<^ zS?2VflxEL3oy}UWM(JMi&L{GHyS`eiJDWoQ{A4W9o+4AlTlBHjCoDdsA9Rp$8YrB7 zQtX*bWIn1|w>wpkY}{2kVwhq4MA~5er$ls%1IO{(8}=nHl4}R0y{H(I>P{eyZ!sZA zl+~%?_PnH`P~-^JA5OpMPe0uNgoHD@?_7yUI8~AA&8DnOuR+I|5WZdOBAhApU+Hjj;@#yHtiBL6xWY#iqOG?sA?!I#wkHH8< zf>XYlg=L{!g2U)M7<5NMg*^-;o`PhH+eKs-TG*h`fa3I+?3Y*g!wx4vS62d_;+cjv z`~KHgQGaK@gajGT05oMdg&8{jleSC>uX3ZwVy0B*%MEgl!wwRsfuCS33zPUgFBcRo zCCvQ4ZC3K#r$i#a!&i`%L=m-EyuBQ6U_Mdrrmo1v{P_{(eN^^qWX}CB1le+~U~{f& zHx(HVL}Hm4VB9W)kx2sO|J^}TRBkR+gRsgI=`1COoigWJO>MEQq+C^Ph~YXAC14kG zVw4V>gDoeVXV>NKDuDpA0;CnrbxF=0@~#F*5qY8?Pl)VUjbKMG5I+sl)^-_;(YEw+ z9{CZJUjh}1Nl<=oEZgv71dM&@dyr>!vM1aF4zf)(U>~2!I8STEVwi8aoJVD3qdZugLBPp*Iv6zFl{l)) z%O~nY!}PP-XLV+5D{~>y19$vNlm#_4r_^w~Z+^gRQafTY&Sq}IRCFM=j^6{ZXrEgq zQEE>;P7(R!2&f~@Tt#4QWh|BLoji~Uc2#+l_kesyn*#u?cQflul9FUa&h52_U6kAu zxkTSj1)Fuv}& z%7;uEb{098LnN2)e6}|~f0uu=5~2FW;_n{sMAwH3qJ^@qgxahgp7d)5lur`%K1gbR z%!kI@DgBI#O&krD*|+?a2KYVuaQxM;8;2`QRe4m`LF@FY=c!C#W_ogxxnh~Y%7c3Q z_}yg0H6R(DdYtmC1DnMpwjU>NgY(I2gL?Vi`KWxY0=lgt`B&uW{EDD0WhV}dkRi6gHn7Gr#sWK zy##sV^bklv8~5g!KTKB_<{ZdVsI)>T;qs~Uv`;r5J+z}Rd(m~c((Qyy$-oTW z56qsDJN$8IqQA#P6ln`vnn?UnW8>_8+B9PS~_V&IgY!@SA@mHxQYOnNibJeveC@b@b z42PAUan1m#uTu&8d{H~TVv0@e(d!9<_XR3-*Td^-MiUVMX+{{SK-=T%$SY~=F@9Ci zaAQ3BN(+apv=GL-i1b)y+TnCtKFn@5a&KpXd$jcSXNdSEqO?=_vBJhw=pv5K+)gZ0m_0!%C=DvGd#G7#mrERqxxIiAI`VHf&q+nNd> z*KOOR?vrRCZJ}t>^aJ4oL0+yZQXIYy{T7p(6-8IJY;r2c-(1_6R}wjO!=m~P)V+Xdb%*B z)AhvOwlL;UOiQ}XP9$G^kYQ{sTzw(K`@X3c$|!`o;^HX?N`)2W z5z4=my1@LTraJc;XH0P~qKyXUv+R8Vq?SCkWTx1X;_p|-8_PAxRsI-=d8YbL_jb&X zpv3S|A4mUV5hmRnm>hc^frtqz10>V@g?t|m=JA0x923O^nOQA>;Nlhx3R*g3agGm6 z;V8&WR@)a?ZVdfEPRd18JjwZ(@u8j#*5$-GfLh(_+|r3AOgq0nA@^8iOrjk}itU;9 z14FHQak5bz)tzr!hz&pNwNYi&w}(;__~crC`vx=mi_Wz_OAtLdSGNrd-+(R6w-D*4 z;);s0&weVK4<{jkM%ta)gHwH)UU+n}^C~lpAgT0d`b|bo{c7LhlhCZ^_(YXy+!R46 zqH?_?IeyK9DMIaBb@cbxHUqeIkJ$h6Lm3Y-sqU~$)grj_VGnWLD^8=`(-Q?8kE)n(MOnN(W`jhE(nR2UE^GB})o6N&vV_!6gwj<9HCiCX!hB@UE zrC?l-Iw-DvY`}=Uou0p4U7Hx&^8rQd++#PZTeD%{^T+<_?H-%<-D}%|#f*=yaH19W zB#|id7t=7@o~Ar=OUh-UD(8G=CNmh+;jau<`wd80PxI-+%bZP(DrT+yXN~**+D;F2 zc}o};%F5Rn3&rbGs@BfF8IW~V<$5+VDs^9lD}2s2vt7hS&@@^0rM;KdDAE?r>rWQ_ zuY*#Jq2_L1X-5IhlBZH(umoP}lD!jy6;U$QqThb3G;_kPEH-$9QU)zHLfgnTkoVkZ_XC`ACG5_rTad$isu>$7SV%cpU+dZ) zb!;?9DJ>l_sa$--BUGfK3R@-4zVXsDtt!yUqHR!!U^bI)c%NGYgo_y3Q1{#|`^~Hj z$5odOJ!G=K+qMXAV6aU(dPhSH+w@iWlwlnWDcNF-Om}E(v&&zLm78`3lZHn`=stPI z{1&yyBWqNm%iD*ZRE7spJT}vre^)j=y%?fBKI2Po$8>B=-zH{9{p9*};#jrSR;gu5 z*8KD$@v}EWcr0JB1O+(M8c-gQ#EzUc`MMM+N%g4(Rk3_~lk=rM%ZiO{?+~sd@ z+BaZfU_`;R7LjVv3f(=GBr~zLw)P)K2kyP&wPRgyW4-s-KgrKQrKk7j)?+x2uihlo zL|T}#E0BvdmJcx^qm^zU-j>7A^&^~;2Q}cSLttcR-9rA$X6hXaTF|s>BcE{ z>I!|8P87nFYT@KrV3Mh3O8jMA%{00CwY~W!nIOU&F{!DRHRM*4pMPyYwn@D4*EsHfQUt)YWCXddP!(KMew z{q;BfaS5(+t_q&!0eL;_P`Fd+p~5C@AJWRWH1PQtIT@h&8|dpLW{0SfJI>z}*ngDT zy<)&C)WYHII`ZFq+&^wD$c^`>JC@43WE`14uJCV$D>~$!#VK=!?6>-VynDb6X6D|Q z+j%U`uivCl+W3@Xd*J1b?(;P=^|jRhdhNV70LC2a4@~*5kfbXaDu7^LZpS|R^;iGX z-DD;J1o-m8obkVo#=uKpWf9{R7xFi`B{ZanPRHfDF|`&(z*--dcb9w5Q&!{&tlxQDIq zMhLQTjfpa?tS>JKTrFtZ_BX@)@GD2V!ty65+scZn^7#m^j0mx2&G7mr-&axexM9xb zU0jYf5V`)^xdeE4e616HBMD@$J#bzJG;+AGmEYBk9c8M>{$i!8&7!FF%~>s5t-k(m zQ>aD_K=hjMx<|HWU5N0D)vbnyrZ?EaioTwA5tqGCVf4F0DPtJ#WN4t6 zd_5CaDMZo^M7HhkLik1Bkhv$uR9&2)@D~)I6}F6=zsumc=@bmakCRrGc}*?0n_dL4 z;{ho8mO3q_`vcnH3aTEh5}n>`gHQK`xS|!avI@#h=)%U!%dHKjBHmAT+JA_m|M7wA zMEYvH;Z_^iOvSsbSC#7%{6^mWnUDUy@4B`<(;JMc(>iW*$x%a}_u%?%s*xZP>?22#X>Q@Oibq2m8_2EW)7e@GzR}FfZf0Z6hfWWO>lC=fL+h8TE&?* z$jQ522ELB@U0yP3w(WQ}$0wm|xz^8eap|?ZTy6*WkuRKngJ<5@1IW8xyr1W;cAndG zj`O3-BVaxZ#d*07TG5?roMUa5ysZK53Utm5T@yu`PIW*Fe|4&Azd_od%=3oFsyDm9 z8jD&%3q-UMAj>+p@3O4lF7Cdz*m2n|YTpLk2xm?T)s|WF`2>f+1XlCf-qv`?&kMvc2{G4Rv)RL%jr17c!oq2yihe?*RMVGsI&vpbDqaw!bX=V& z4iG%}8mEP_^m=I-xf6k%_2|&JJoRQxk^K>C|M4Xtqq^aN(C3s$Lo}9k z5aC&mDXZ0{wCB5KXU@F+@RpK%^%jqpkm5ybuXkU7Biw96;5Z?{-Uq;H&+PzaKGq~U*f^M4ieI)l(K6jG# zH8YW1TJ21YBBUxBmxbjhRTX1Bk#FCvzcSgg^I7mA5>>I8wJN%cH;>)&ycF(tDcyfA z$Xd1*T;_cjOC(Bfx=gk_MKas{VQ}2sPTdCnyz~r#fNj#Rk2I0vJE_bjzy(J6WpGr?! z!A#Bd^gck-s({?N$D+w0%mtVBmS0p6Wf)caW10Ig;)}*^9LearX(?IuHo}*;adPu* zWq8bTYU*?xj!Ij^gfPGSsS-}q?he1e83U||;}!kvcj;|^^ka%^&}zSL6-vfO^^w9| zcs@znzicv|Ms#;5gxuuby}b4snoV`xI{JgbatnR^rJ%CTO9`BOQ@5vxk}%85l_%|& z>T{~xeDHToMp44Jn`U_ObL3klgOPm| z8{3x7_mKPf2kRy7SK)^7E)BMygtvSBk7~?wW|hNQ`p^93PkMTH`#1J5s?=?p(8eNN zN#Jh1>M%dCDX`8(KaWqARk2G`ZAZXF=y%z&$-VkE;Je5qDsYd}d3hVO^BvXzQSCyj zde@vu%?{eM?L{<*DyqAHP&(*07$QQ^^lA{mQJV)uiD745CN73G`ePDGz|o|hWv;OK zysHK}-K%Ri+qU2`)IAy2=tWXLJzpl1nF(u$r$&w2AXJ`y)n_GcVFv1bklk=V9+K1N zDGGaj1aYY@)7TDv&F^u(d6FXo`ZQ{>T{y{Hx&cBnT^mzK{(y-6)R!Q9=G!Cx%7m`s z1A=4em90oPuP?v);oTrXIo(|0z<6+dk>6QS#qD%idH=zqN>)17xcD}oI@`EZdhX(w z$Y&8Wm-o48{7=0bFUi!J^-A%qjn|47M5eKAo4yemjp=OhpUVjs9Px2ceRkWr142wE z*5Uof=B=S1_KuTldavOpKUlhd^Rnml%HqLqtgNMN5mE4dQ^GX}=dobxmru@yvRafLS{Gkm9|fvB$QDD6Lg?n~QLn{W(o%Ab#XNcheQV2_@GX|>Y1Z1I#jcY;5Vba)ru zUYEEER}HNqmEpnFjY8}h2X8j1ZERdb<#5wq7L^21UaNi!YO-xz67EV2C_FG~gqsdP z9gAEOV(#4q+L2pUYSe?Wag`&LHcrDxp6BKjAhOg8mxip7#g{0xJa5OMl3Qr*f4>zW zj&jK#4I;zEyGKwe8o8=x@$1yPNFC>>=lDIP{%eTM9BoGrnlK6s7z_>hj0w5=Up8Nn z6l$9Bjxf=1^7JyqSov8Ko~93DMlNqSf+wfyqd3>-8e|>W*axRy^ma#>*%X5Kjs*Qz z!5td$U9}!c=_7O=rcO1S`YL2Ub7%DkQX^PK_^vGfp6-6FXz%by^>^Q7gmLnnV&u)S zPvK%d}vu8I^^Nj_%`sC%z^LCG%LrUIGqSbGDigpJL-B|$>#&Ae3iOttdA&m``McHZJDxkFb^z5iE#+b8t<#n z+5zAVE&eN5JgheB15GEJqusvPsY2&sI&gJxIbJV*#sTW}EchEja}f5T3@VK%V$luN z27!8}KEo1y@?#ZSH<}+P1-ZR$k!b{Q*gazKhKJuDeUB*2^{SUuwhA-70|MlwmehKX z8Pw-~mAevPJ0*`=GnsLU(-fO0jQ*H4{e|;QfmQSkhN5+8?;8kpswq%ED)+~8-88~c z`rXH#`R2B>a?wb^#%Tb5*4_B{%~OkhEdw|^IA7pdbI?!frf^CW{#lh+LMq>V(m%YP z0ZtPIe$f+t(nyfwpne5Ej@zl)g=T;SVbpN04Usbm$y{*OjWvB9Q;>a_xeb`#wYv!B z&2#G71`J{y;Rlp6#W4?%uYPJn$CZq=^TB@k04u)xg(mIOevv+cCFcI#uTyWPu?brf z?>)Ll2RQ`!_fAdgQONxVO6cYuUyLm>C(JpWqMm%Y74mm_;cNduXc)sPrrz`My=P^d zD_DQAI4GQ9SRE#3Y8@?>aFgH8Xg08YJ3#7j-Y9=PR>CsdgZr6%bzRKOLQ!+r37ex+hR7m;05Q5xvc)11+|S}DIAfLC#qwf^i3_Wka2|=tAp))i zx59bvzVp}xYrs2CTAwpam0EN%(fv*o&V5^cqaTVGMGp#85g|&e^IGxkndSipjz5;e ztR4ojwhTXSS_0aRcZl~0XBdq6o|QR{2+P}|Is>m070v+4KquQh1q#_#1h-F9f$Lu+ zTCF_CX__G`OJ%sai?@>a#E+BF98oIHo(LG_JZ`!MvV~)n;ZKi%~XyL}5 zC&MSBS@_YDzY6y$6A$8dsl?jm?n~0idIe6hL9so@)S4lHb5cie_<7XB`?C&kA2(`2 zX6&3SS7&>BaM=B^rPuA9tFfeb{m)msel%-e;=Q^xy8|}wCc99*mqpcQ2rB&yr(b8+ zO>4M^FDdRWP@zsb_O}2vjxy$nu`A+ff;r--rw-UR+ zVq~_HJYhS*e|v2*GTT3#r8;%n?4rCcm_ADNc{=Ulg=hncR8eI)=E$S?Z-`Gd%$}^E zR=3@9@xSxvcD~wK>(2P_s8|iQ%$;$}diBGqUH^$|qTd?L1HDY&wX8G}v zhq${ymho9o4(&ZgDI_?+2%KBOw9lfvyu&>0{B=sm`>^YgYvGZrcMI%^sD>$@n;;!BSKe{`A?$bFf)bC|@?qSEA5 zVE)FGm>-7{Ga%&Rt5{;&*3%Rk&h~wj`rYcST5@a{L?X?}h@R$T=8J=8WT~%`6vebA zEQiJF_RRp#OiI~iT=WCe`<94T({^VG+Y5^T-^$IQEhUJ+qtH;t!i1Q;DWN=a&^gAJ z5pOKq)CpoYMoUX*qN1|vs};YuRWpnG{sxz6el+jJ7(?D}Fmu$~6oFWf$p@%TRPvxo z?66m8dYJ$d_QZy@3D5Hm7r#oI{f-I;Z%sG(io!<3_(H1yb=&b6qMuwBKSSnnBAPIs zTWo(uUW)2+`jqw_Lu9w*hK%_uM!?|H7|KX%qP}SJ7pM`|hs(ATH1+38ZW)j(eUS;c ztQW>S?H|kNVtH!0^#&66)3MZAw9)Ak%-W7)26`pGGfa(18^a|ic4-Frx({TanmuPS z2qjSC*Ycobn(_~>(&bB+YZg94LlolG7>P2%+MMbt% zyhMb8IT|vmwis)Nk_+LtYiRq1w=+iw-mgh;kX0M4UeaDnvWA>b0QBJwi_dR_|0(Dwt-?@lE$E+#r z@dCYDt?8&zI(h8Do2)KBL&NOj(6~)j7&EzteW}!grR8|zf*k2-zsk9OOT@Hdc;!l& zTq8_l&F9%tu3GF`c7i8x)00l#pwqhkc;aO;eXFDt-gd!-T7gm?4BxOH7&FX*b&+9! zp5zk+!9wOj*V~`ba-KBu3G9LOLw$;kOy_5oY$gz@t|C<10GHPlX!imh!B^IJrq=&w zhCj0>ZHftstynH8B9(evlqs6{yz~;R_7801L}ehZ`ruaT4BF2er8vD9%7QiG^lO6C z)~0l6zabel#$YilVi84JAv}o5eX#?Hd-(h(U*<6=wk}+srj=kBSka)X@p{{-t$pSg zW&Q5+i(lNU!|7Vcof$^fY|eLfJq;6^F~-!YN>`G9>T|@CAu@S&Jh!@CL5T zLhWklaBY+Os8Hk3y!FMZ_Dj%)`I>%@3T6|itG3o!dvpj;t`9~uomE#We_WloKE~gF zQ_vgNN#~~eP#xjQk2kt!|2B=P4?da7!ZDhTg|wXqC@IGaRC%W328WslhQa|b(c(MI z+)fwY)kZ)2M#RUedPkTAZ`^LCEMUM8#%$)mvRk%#<|XgXFTnqQMQ3>=o1X8B9z6YD z+~0jv3KJMND`s`9?=}mKuJ!NTWW zG+cwZhgt3Cm9Ey>pg4A`&(}Qw9a+1ovfPluik}`2z$#7Mu|*=V9N^mp0EiV7JJW69 zOe-}9$3)a$TXxYQ>tD(#8Q1;m7e*nM^Z5;I7t!n9PG<_{l?dIHR^Q$s@ufCe8c1Lxup4PgGzLqBOG zqf)~v?+MmYo?zAq?tdhZ8r@L+vS3Gas-Al&+QzoHdPOY8&+pWicM;xf9tbFl_F+@M zhg{@o|9Q9n_Tq;~BJI5Gxiuc9LvKk$xFDB92z>qFHuc<*0+|Tp17A_xP~Y-W~z_|Aem#^;vKw3?F6`IPr& z#lKrRICpuo|CF245p>nX>}bfuL-!f2urI9>xl7XO|8_mU|0?ksjuz~db&Ss;Di2jS z?NbD+(Xbr4qtBnYXB$J;H3gz;6oYQHY5hgp$?oj*Xx7IJ+P4(q9-wVg(xWj{>7hWP z0DOmLy?}YnUpxmCXs%zANpz%ayThrKZTnQZeb;fz{MNQtTFwE9ywm+kPi0t7+h}d+ zMV|T22={Mast*HI$qCD5UH(Mm_~pL$O=qpWa0H_t zLlVSdmN`dqkXrsgHlM|!t`wR1FRt?6qnqmt>6ZatqDzk!_UT*cH;{|>WyH3fgJLsG zfxRIM-BF0xZy-imzaG^8fI_Oo@0nwn&WYwp`_^%BjM`lPuWuxlh=vDE4rERg3Bnp_ z7=Mv?|NFJyN9{m@L36NJ=FG(SAFq%&2-Q^zuZ=Aw{&f+Y7w>oYSit7LrQwJDHA3*O zuz{B6m)5y$GwD~v607htz*0=RK5#SfwM@XT7|r}~SycL|Un`uTIrz4TSH1lMSY$SXNpQZ*x&nfGsNbII z_RGvIfG~*5_k&dxcTz5~M-s>xoQaiG0TomVImope$)i!B`49U1D*W;mp{-Aq8P*?y z&)jglD&>FpIj#{%%huvuZtYlHZm&7TW*v`ul_O}BXFx3&tm~pb88o4&&=)%O>UZEP z&Ffx#LLk?vTo)z@qKZW?)2rd1mwYPL>yMYnZ^TZMEkHCoXMBsHff|;z7pWQWN|I5X zFQQ(sw^cR#`sJx0ymq;NWk9DUJA=u4>i+=KBv@fO>z5~No+_p@z}3-GFVw)#vu|@v zJc^4r-4k?IPvZQ+zW^-2tjARx@~(hJWCj=@EIvYfz7Wl9yh$0uVUrKVm}dZKvy=WI zZWA;MY`>Pyl;A{ql`chkiis|VLHYZa;?m!D6L;$c5ObRbFQ>q>pFBrDE(mN?J{2yZ z5abyd@;EJOx2lNdE|VP-59#FKjqoXZHJIyMmBiN)h+!3%_0axo zcc&`?qB@fNncZcky9^0s;rZFe&y`S)cU^!46eRgP?d_0Duff5;{%V6A|11>`HG;hPl&3HXOc_2vvE}d2>{!5| zMjO#Arn^sB7w*}BE`l1UN)r!YQg-Otl~$mgfQ5<9@pH>0v z6x2^@R+v2QC%g4_B$fQwX}f#I6D$*w=$0#=>b|nC2TNbg@uJSVMQFl}_T6|Qn+R|z zQ1B3gj9ebQ#fm%z=a`s-mX=J1MR%+jCghN(Bc&6L5`ZkUICyn);(g z&g&ky3eO_~E}_L`g?|GKI%r%aQv|DkthNgXU|t0#x)-)PEC4`QPR)I<_9y@=1+b~g zY6@tM$Ady9z0pkiX}yf7dkP`s;(d&YMwf*HG23`#%$m_07h`JkblZwT%ndw*3q+?T z<&!!lfa==?V{8S^8@2|}T4fHv(p>X6&t@!cYN(c8g^QKL%Jkhkhqi3&6t+Cw1&ZO% zh}?hxBv%4bA&rv0NV*mv%r)KxO7`RS1HBbl>E%nFmC%nq3Zh9zP!Mw$D)PEO-P|C5 z@%*J~5Nb7c;2dibODt9sG|kI*xw{SEOna4D(mJ;vHB$Y@Zo})1UU=6UVf*84ZcqfS zV4MPlt`EXl8mqg?CjH)qmIlfj$x5!Dud)Z%q#;(d*BBAtgyf09i5ZZY)JB-@fHPim zve6*peKOJl)_13^m(4=$b|Y{rjDJ0Gk531Hia0697x8lqh!koCV3u~)I`bkhd$b$2MGrUQw7pjVq5FFQ{8~O~;+y_0k zjLTLMxLDfX{vbo%j*^CIZ(6m2Y4h{D5c?wu3nzM7qNoX9R%okL4GxpR`I6lljj#2H zq)-)jjJc?bBJSh5nF#h!cf8|QI1D8#sxWGArd-e6sQhy%i%@Bap3yO0QkK6LsZ?ua z!{yUUk?7?QNpJOLC7o%dm&Jq3=*C%nQXt6|5m~cyE!(1)(K*^i+Us6rKlyAbB|Z_n zgPiedn>{HNYKTMbrqZ=BUXZ|Xb3BZf6h~L<)X%ZUSpk}o&an=mG+?00>Emxae)%h; zF=mRO^XXu5Wczt{#k})09mSVrU~O@lQqj_Gp&b|a(vfO$`R#d?`Wwi&05ebO&-V?Z zPyeMp=+K6#USnAE44TpbykreQkRQuF z=q_8X(=Vvrd~E~#W988Z4xt8{r)yRqu)R}y{8Fx1OL!^P!f2vUT_tm60@`lS54#p@ z2tx$zr}-(tO_F;Kw!bjAqu%3B0-Z}Zz6`Ak5i0hb4d~P6m&~9fX%L=ui&VBq`ojyr zs@$4keX1Uw5KZW@8z8wR=Wxfmib++9kGG(Zc+L8QamZegQpiNMOwP3xIIfH3igc&Y z>Daq^r|uj~!I9m0%Z=+}CZkIrGL|QM>hY;kHfm@C)B^}Ry$$czS3#@O6-w_#VA~#1 zEQ4K#q5^=K^w~ZdWc<0DTpykzzkcI=WBZDh=*Q`#iv|J}Rqw-Vq2~ph&gxC$r)lzj z!P>lEB5+hBY!`tWugW7f3rLn7zmRIC-m5EH*K$>Jco(l4^Ru_bH|}X(Hp~QKwvquj zDZv-(Ux;qGFV|CcK~Kto(;dj&uQRm*pi&plz9s(0BsuU~PIotF79F@pr!Q<5T%Eaz zaqavGl?J*KRv;T`brT*5A9IhszOWGswY-u}_~VLyOTUct9^$Z=Y`^81V-tfzWFT}j z1-f@fI5hz#&U^iAYcC*Ulh>JmyFFThDq*%-*5bh?rxLr&wFZz-GhjF;N&JFZo1~R4 zRGy{b8UKD$5#@;J5bt8roxz#H{^V&cu zGVPejFLqb;8dttK&do&VW7x~H!Dkw+Qium(HgbIkd$FnfwgRb)G3-{^(;%=pQRp37 z;Gq!X%poeMfI|q+l<U*uaPp0#gIb2)@o;HVA`0*k4M2HX=6D$s)joW|LVqLfFZp|_!g4S%9Z8Q@H|3>j{w#UG z3+9(FZYJ9I$Y3^nFenc#@g#GB$GNitdA$X#9tNGsQmeP~?zjgM?&UEcuFG=dOVWXr zd3m5lYbbMV%KmM0O&^d~#ZcG)t&z2@gcSaU*Jro&wky=RX1IFh9i*>B4-|pkl zmT^(@0U{U3cU~;v;#W^bqIttfPSVbOYtcU7*809E$D*k!IUjYPBgol`fZHz_SqriP zmC&?!Xd{?IP%|d~&(t{(SiZn<5VVCW$+MqN6_Lq_boC46}?P8<8~>RW7_Ez^$OkRkim?_ zec`mUok434p=rUu$?{o-S|dzPdDi^(?vkDUPjS?W<;QOimbyjEW9baS_r?6on3vtr zuRxYDd%Q7>cQkP*rQ6oM4vuQMCbPKDaLKB~y)xOM)g59eg6?P+E|8#EHTJ1Y=wHMZ z%nxB|k%0=%S!JZm2JO<{tNrTzd0Z-Jw-(L_B$7gHdy%apkh8?q)-xA2K_&{Bu4P&5 z6;^0IjV2=2B%{C_!K@Xh6JVN4v*_IB^3rdltF(K3temig{9VnQqh2X?JcFw~b*!1fyH^xLX>ohIMt9_> zjVC_8UjfOVEM)Ee;+BaGIYqynbTc{HFH2i#NvQ)aMCDO(L*kLC#UR5!ZwdY07hCs| zj9SO;Mlj&+`$Wt#`MciV8BNz>UP3CBLKP(qQ{fVgP;$uqQ<&!Jo5xs50Kz)$hfCqZ zW{dM5?3grFUVN`b`w&vx>86@bGNd)#JZ@OIt##F+|D>dkZJ9J6Nb>ZJ@d~Z1B*+@C zeYL>%1- z?rA`Y?K&IsWMv>Qaw@QeTmTKx(Q(j*d-Ia)d7_=#VL-5`QJD>Xce|QQ@(ZHi@17Jw zwqG@JHEp@lr!fUYYDwWA_>-@B=#?3ae&%vGwZweS7I1%Wwnw+yvEDXm7ufjQ1uyEy z1(=(la7HOk)+-Y_f5uxRiuFF=x>q3EDvQ_@;`{HHXLj9xyu*%v}M zK}$4zYH>4V)%9Y;c6cf33RLiGQSFh(`%Ml}d-Ran{jUYzQ2N{=0;1vj0qk9V;HZ1= zfyEOc>Rm2^wy`xm`)NVBHN}VDT81$kghk>DDNI*ZHl@^KElzqem4DwwB|PBVxE?L9 zKBD$lE^{W_rC&mKr83&qO@Oao)O1%CqNtF}cc)G%UDaeHO_2qZS;DbbaVo!1oIh?F&1;+oZ5sUDpC92 zZg?L0{44Ge>ICB!Mal`;!`E@xF7gayZ*@SDbdNOfxSv<+s|IHe=VZ%)#{PWK$4Qiq ze_ead+_r##+p!X3<1NFg_zK>mL8HD{wg7uup5z&z9hnkhc(bnk8kf@>b?u1(0tPK1 z=0UK{%Vs{3g%zKJK3$c%Qtv%-qa)2(zny&>(&>a9(xhU#H)tf89$X3+WUlx0)Pp~c z_Fm=0Sv2dSdz=I9Q2+95+-sNf=WodDfZ7wEKq}S{ZZO`E@Wank6~8U2C?d`ylMkx4 zvShWTNF9QNZ3wZq3=5650eMubWMpm(_+@4q&Kj%7+A5l@y7>9N$htaarpCCdnL`ne3dHZ0hAD7g&EGhsjJ zwI5+YI-qPidP7YV#Tq~-4r#xjZAQTB-oFQTd9MY8#nHgpcJ$bYk)hLI%$4-t(9Zuu z(p-Z9J9VD<{X*KCT}#~*zQ_@T%4+lFwu=rdKt&wR z@F?*p!?2@%Uend4R_h*2w@I$f^Jf{*wYOmXu$M(C<136HRoFv2eb9CxI zLBxNgRK-GHB&upHTCr8?Kpkw3pM;czX*#QQltsLRYBJ$gU{=)w+kJ&PJl86pUEZ1& zxZCZ%$2#_?-b!uqJRWsNmv7oDp(l*fdV9iA_&{StnK8shw2v(*#<-BT{td%K6Ng#a z`>LZnNQ zXa;|*87^AL*t2oWd&zlb&7P<2AMCR)SqPWFjTcz`OHuph zbJO8=kEj^celU;hL+pB;3PWdth<+3$77;qKW$jQ4CSZTAjN=A+j$PIVlAKjkrV|WB z+*bPHI2`yKz~zvOan^eLsi(M+yVb z09Jqo7>`Q=vcmoy&>>yrT>lS(<;(1Mp}5e7vpxL*Luzrb#@8|$Ae?eY8ch3dnAeANOVqtwGr(rbS+ zo?XnYP%%EBMk1mkzWPP@f(J(cM$0~dO zFXhi)-c4d8yl4z_FSTktr#Lkfb+|Sd%iVz9+L~OxvDV8|DD+!#2W(=jdSh75wxFaU zsLy)JoU^$8>x%vT4}yXlfk;36*fv{Jm?3Q6wX}5(YDpm$%StCc3>NZAh4EdIC;n!& z^-eUoKl5K+>mMxF?|(oj3x5r$0^ykOO}QaaQpZHc?92V1c-)S*+(|4n$jOl8Lec+i z5Woi$GTryhe(?dL0^-P79WS+wN2lS&fIO$yR%!fj9;)>f#YL43KmEUr?C&RwSN`Rf z7HS0}Ia%#ErxzmQ!5-!`J`pq)mj?Lrmg25M|KsNvy?Y2HQQ}NcZRFc~;K$2{7f~rs z+j<9MKB$L#ct{BHS)BL9R$0%fc4z#xg67XB_#ZF8hl>6zW>{n1(W&ddgFne{9nfb}+*kF7k(A*@C~0bN|>kCwSr9_?g=` ziahR{&KEMryiWeiUBsf~jnRRY#3I9Ij+C4$ZP&bFsQkaqP1o!D>COK32V`56QTz@Il{q0UVL>2(Wam2Ad!-U|l zFPQQQ`8jQ4Fsbq?xshXtF|Je|4}Mrf zcNn(93&4C<;jdWqKd(S$OPCf<SU1Ld*{%*%!+cA`^9X* zza`{guFp;tkUWQxZ$D3y*FpmjNlb@K%9y#uF<;jLYrfX}Fmv!3d_0S-;V&zS6w^=v@Yu7tK9((Kw2tK(2E zRu8?4KN8-KUZ{q<;|2{%^VXGscd&E50c@>hJAsR1($z889BZHkC9@y^?gHMx0&v40 zLNW5mi#}5R{ob7xeos9LS{>hsr2Y8ZGlcQ1i|y04JIX;4g5e~J{cx6p-r47TP29Ek zO3wdB+FM6e)vj&diYOom3s4YgkdO}PZX`{*Q@UZ%Eg%vCg0u)oNOvq2-Q7qxNG-bK zz1H5(zW4pSd*Ao>j`5AL{#b$oSaV);&ht9YR7(!I1)tXb%h*#O9s<_LQac-ec=~C$24dI&(IF=WH};xL5wPS9jeOuU8-b% zdzB*vFTuow&xq10DSUaRe@?LVJ=KG2KQuIu$s&fwcZE{n1re(Z@QWjS$SmF*J=S=+ z;TPcz%$e35k(A*R9miB%1_MYT5<^*0COh9U2C+8EEl5uE{Nei<_ShUoh)u*mFy235TzM~lRv0dNm&sK?DhF;K}1#N-jgh+L$zdBAtbc4e> zbC*zSexhIE6ewzrskQ;?4*!m>o<|F=Ovcs)h$x6zI|bQS`LX&og9TMoZps#RdlCx3r z+*n_|*4b)2`&D`iju^pGbqZ6Dh`(zU_WXzOLqLrp2weo7$m8+*DRoecI7Y%AK62>M zqL9Mud!ft3WvwouB!T8orpkkQ+RVhy49qnNj2D$>er4B4G zUog4mg`o~i=tDtr;&>A`^`VY1{Jl8qiVbY9Z3k>u905PGUf|~PUGlfU5*~ z&np0mN#PMrGMB^DGh)!{NR~ZmxPt>_)d_$I9_{V}5NN!J7nmUNu9`eeVsLdoTwkbh z{Jj_<1b);G@C_(H)oAaJd_;eUkHnvzZ9dTN>Os|YJ$>@IT>hT1ZT}o}Da?AIDNu*R zcpk0){>~-{Y3##vgqlZHfVDg;pD+MJcSJALM-0F+1zeDf!8II*dpwpC1O3IwsX;A> z5Ir{y&a6Kik|(8No2^1hsCcGr{RNO|Pe~$msDSB!k0qrxoexO&R ztY!qV*g*KizMp{p*M)u|sTj0^5a21|N*ck0VLcbe(y6eF7hy$0=78H(lS;daY!$(U zKp@F!+CU?v^NG$b_uw61t4O+P;QrUA)4#q*?~Xcz=bk?mT+_V?9>kbw*0J@Mw*R(jRiz;ge{z<|3U@`Q% z3E<3K@b)sCAs6)Cqlyj@wZm(8^XN8CTF5T^RdX|88t?^Iej=NssH6jP`HL`BVS{HW zMOcRoUw;xT&9*JsgVvoF@t#v*-SPe=;G-TLl$?is6@i|JfS7CH0M@m@mtda(f_LWv zCPGzN#Sxv2Bqh)=gPunD>^UT}(G-Vl5{s=^77INbz2+6t%{~oWnd>etp#6>`t6pty zoMI%&H-8q0#^?+bH|Ou~24t?D#ae5--e(RZMxVXx0Wz6ww7b8Y9bc58d0lGy*>)oU zjnp34BHn}I0fw~srt;s4u()@r7erzoO!#qo4!553)Nt+7fg50?)&laEn||sNxRA7i zRKjEb{yAwJDk^P&INvz;8qc$fW_h;8UtxDGBQOzXCGnIGC_iA?cq|vd_w0;b2i>D= zV5_8Al+njZyDPyaUWJjky7gdqz|9VK{>jr%RZK!C=w=`gDmv9-%05q{ysMDvocvpRHm*_3)d&oHMWb5*T9P6#$L9V!f**@j^2fqFFvh^(PqRMaJtlIx zy}7UyU)E@feO2ts3~CVh_k|8U>ef_76&qb|#KNs8;6bgg4O8;s2{sZosYsEt=(>r} z69n*DucEK@9;sAXYa^8^BIcOGq2M+8%w zV9_f>FMB3UC#mlKOsl3{nY)O$+e>Z@5ChDvAlq;m*0qG&k1wYhaFq4LuiJ^z(g0lJ zU1}SZOeD+b*+)j1i@)Ekc47Oz&SauG4M$)7?dc`$T<6wI<`_&%a6S^C3fqRfL_#N< zr;v`O>-+RTR@bJALIw@6odqmM4|;7_*;j7|prgifMHMN80yJ<=J*ieA74iO$ zr~^2g5MZRav!-9JvDYEA{>^{d?=Gemki}>L_4*|Rb~xwPJLxc>BWNk4yJ+V0v2O$I z@VK5CV(yez0VkPWO>#(CQ#hXo?&ViQBxBR-3fwMTdQL)bDN6b^P2NT$nVaHXvm&v{++#-n+> z>Q;=cf4U*7QVEpg1-SD!>;z#?RVUl@=u$9CZgA=QOm}b02NV9I=_aa98#$a(r)VDI zaG83~7$o}@_1i4hUOOxVOOp``XkD=1ur$f5$$~I#m;HyJ;CA8lGC|JQ zmrUEVa=<4|x*2Fh8vqThNlIitRqm_LgGtf%E2h$FE~^dObM^H&OFgg}3vTpM(2lZp8qt=4^kE{Ar1KS>`xRis{55dqnUqL#LNZ^zYQ{>0jz6B=31 zjo1g2$F=5eY9=a_RG=#k_f#umLf(k(Xnb!+g3sE6+Ce0HZ%~qHTokep62Vfm)8*(U(Vho;z-PeD;i;?R7MXJFe5^!7X59-tiRc!QWjqJ zBu|p+r7Q|QozmeQu`>|Pdoml!9N1{U9ZW|&?rKT%Ir-AtFPM^>J~$f}OJV7TNs{gi z1o~0)$(ck4QwkE#OFhj5^oKZwRz59xJ3)Vks#^2^a;n*JJIsntz{aw6@$;ph4GCs| z&JbZ%e0Eemzhz3V<(Djba9MKDK}e@@nSIYx<7*%FKTa$h8=CJto3tpT5q$go;?UGz@`!;`M=u(%ar6 z2&7gF*wh!EdmFpqk=d1H2*mH~ITVd!ZoXBuz(0H8nvAP8yqBU0gBBaR}QpdAQ&)1V^qZ$7=jym67ustft_`4N>0sOgA9ZX17YHnWlek@8E*=%YxPtb-jJol16{rjpfT@TGlH<)2`*lQ-n;NS`rm>InG&67ex{f0 zC_5=iUP#KCdP$Z0^{Kw;v0X02mFInR3>M@9WHJm}}g@dVlRH-RDKy8VeSePw9wElY9>0xPQ+@mj7rR;%NL1&#(n3vP8rBfr8)w3kjkdf8#jbGT`c)) zj==ezLhcFX4yxi?%;BcC3|{>I_>BLnDtO6_@l+M7%fx$>@!y+CdM?y(!F}2Gek=qV zzWW{=FG`#d2BwY2JdG1uNoewN0u@?LW3|3oueP^BZO7)uY&nt>fhzxh7W~YD1E|FyaCSE7K z9vcPSt@vV~3_BZ`KyIeF1d5l5y2=MZI&;fG;2IOi)#>-y6`CzIRqjWCmO}ptmWiip5^wwy=(ichdNj7k*SlNmbS zHa!G>&+!2iX?u${P*VO>-2uS+lJ$M$O;OPw$lHMhVsxUK8}}rSX`hm9Zwg2-e>oz7 zFNOt5t0>AQ=&n63*Ffn;6E**=M=S0U^6^dJ&+sv^M2LMhA+;mZ%APymj0oWbGyadEDY0boZ`D;1QIc62&8h3Mvy zvlixnrrgr6fMtiYvxP7nlHQJRFtaLlNX77$6xqkuAW0kjbPNY@JZ$|>d^$Xn@2S1l zfoR_r*`uU6kT>k)3PINp-hIwP`&87nl$*-W5HvH=B)(xzjpV!`6+z_=uv!(X^vEdX z2y>vl3V1M^9b+Rshm1}5io=6{0eYFqStJ4SURol-Wjza)+n|$IiI2kLgnZ_P!0b+Z zj@YNk7(+K1%zuxyUg_WA&ck9HufPqD7ZA3@3Z8GjvIZK?TdvJ-NHg3?jSrGOY%Y7T zNxWRsqL^+P{&cYe%IiZB)GZ_pjr_X}mOZ)%_10O_6~9T(V{koOoNW&6cynmDp?Vgu zE_L$Xk%J`WkmK@PdH}AMnSrq5&I2axW8vcK|7*ek{u!f=0*rj6St_LXzDLU6q(WCW z>v7G+l<)_*qK30%!WdUjA-^te%zH68%Pn1h`|6~PCQ#W5uNqSdfPDA6#% zqE(P4j#+bc>%3WhD8OXZr}eRiPmsk3r11k5(PH5OGDh9~Dq%pLe9I%nW4}|Fahw*# zTP%t5H{Ey|MqLk|g+J;s$U_mFO&qLfw|awp@En;Hk&0QRuXOX%+%BYp#-Ik+(#A=4Oi<+XSokMoX?fl<45^M?ml@ZD6$O@08n_2h%b7=W0G_&Mo^ zQe`VoxYFXpsCtNcTzwx&OWqoKXmnRw=9ggB7{SIg=q+7};`K4=?;(Ix=Em|{>NIPq zY5(zB8>)HjHT@E>^~eiSS9uXxHn^6F-8xM6y*u5kRc|8b&Bqf^6QKcw9I$qti*#I6 zrT~%l-RnLA_pazauKPdMMvOHWVauPKz1jS?)P&;MZ5RozlSZZWc0tWXt|Y(UvuL7} z?8l3^C}<8gc;|$^*n3DJa7Zs8GQCp;yucRrv9own(?!vbk6K<}>4!k-N0bF1n(U~P zyq|~2Kb`rzaH;?r8td!gSy1`277P;NWBelLlGK*u4d9dHJ4cf_$djU{g4xjC3uVJM zY{P4&CXirjZ6^90UagAp#ggh9Mz*m2ZEr|C8>)DKk+^GfbkQ&^{6i=`mUqA*vd?@> zDXs(Bq#;zj{K~I+$CfDN)^{sIk{^okd~42}%b~=t~&N9!2xTgM}YJ z==x%lapSi5*Ga*+9^(+7$ELw`tG|g|1YyX4VK{3rEjoVyu5^o3WXORt)|!;ovdsEh z$w%VPsdr&O#}Q(LH#C~R^dKXqn6equ5KIk5UB1@*4k3Ct>%BVr9y>|L$H*b4!>b(d^{ntLQ%g_G~SP{{=KFxa*5 zj=_-aX>KDvk9h@-KIGNIM*Q3KeF_04JIYu#-PwR+NYP)=w2yS~rPxmd>zD?AF$uU( zGC{Z04|U5NvJs5gb7aXg9_k0B7)`F&X~dJ+#CSDuvG3DV9(+WjyE25J_&ufK^~Zv^ z_oheRCW#f+6$_mB99CoLgEC4dP`PlBniv0DOQv(*#9pb{cwU>BtBD)mFX4je%jZ5O zH`ZU*O)Mcld`?&#ES3htCgi`GeOfDe;MYNL(Nyu~?2oGDk2m%~4Q`LzmAwU_jn@~f zs(n|{0+J(m|5M#hG@$3%iP3l7e>5{A!#vmERb6irT?mE3nQlcXoUSzvY;p*^!3~qaHII}sasg7xb*FRV^Vtj#?;9V&U|Gf$(R^f zM|+6(`iYI+hV-p6J~`%mW<{fNtJqbH$Fqc$@UzuKI%lA@;w`pjqz#}tUeKQ@w@?+K zD<#EY1=^kso_K*6;dTtwLC@CK)DHp8qz=r1+sHn2B9N;%Lhn+S(3R4qK&dDX7jbJr z#eK@w7wpWfYFh%U5Q+9ePR~ltE-3j`3qpQ}NZ!qB$y^g|hAj6wr0jMD*9AOjcb_U~ zHlj?z%;sKM_&(NJ{$V{>)U)v};c`ysZ49g*MUVkkY!f@`XorMcjn#2uon0FF*wcgC z#Hh|qEF!=v7rxog#B@Ccn@9RfSLvmTaoX*noBiC^3 zPqo(Sk2h8J79VGEzx**O2Casb4nlM|OLfQVj;JsBF}Mw-1FI@CprJVfHu4B>aQARR zdebPa;=N0GTXZCi0u%%R?n6Osya-o`h@GqN4qFlnEL_&lN8(oysq@#HgrqKtrloxk2yco-&FC!KG3xN`HhTY?!RHX{`bNRMhcMI?* z32huW`xsXY2!y{n1z2JYes)c&zucFvCw)M(?Lx4p{KH8KjTN%-q?l(XMXHLfhkH$E zN@W6^8B#1CA3kod^CCZ=8InOwfdfW>K+uc%GGpMgaGCm3^ zLMJhGT%0%B)w5Xnmo4cZ%C0}olo~7QX-VyDo>uifumy|SuoR{~UVk#c_02+b+^4|z z$mzkZ4f_pkx1>+C@}AKWA$I4-aT*b(M17<57I5#jsmH`wEVBJbA)WYDU_vtD{cM$$ zpnc~!*mRhZWdz8U^O^MC;ZBHqPMg7DQ@nBors()7nG1r*=|UnpXRt#?y9E@G*%ypL zobc-*wpKct*v?>Dkfl#H0m8b@i{Qa&mDBd2gjRgPL3TCL_4zDpL=Xi90fy^=OB&jO;MbBx4jljJ8R4_>pI%_6jfZI;Rm}( zu%1NXl(~dUL2{%NDD(-4#6#K-FZn5VtI3G;#Gc>AOtZd}eQq}NG_~TJ$ZDsR125;} z4A9LDv83r0epRgU$8GJ8H}>H^@U7M9XRrMK_>!dF6-2Z_l%nP96anjNm^#S;ww6siNRTe1kHcTj*-E}T+w%2>@NaLiXfvUDK5=BWX2$oB-cK3m!mt0bDGq!%J)ae-;HPkH@_oj}|aZLvtfPL*ej zzpTOyAEHO(6J>A;fE&1$NFqv$WNcFc)F))KScnHios!UYq*~l|+L6#>l>wdt^*(0( zUzLA{C%; z6Y-W|oP&+IQo9(Af)`-dP@T?|&ctf6ivX*dq#HBa#G}&13%E;9jQlY0ru8 zoq9i2QFcYkF>zgGTqP(FUP<|#fa-ycM?i{{mc$|3!tl@!4e3oD8;x#>iP`Lht79<7 z=a*CzT+AjEdni#w%44l1E*P?EETDmgLNxmUN=^NhqJToQ*c0@;Gq7aBD4>A+c46Kj zFDxid4)_%H5h7z~6A*%5uv1wakqv2*PfsYnC^oEq4GhnJohkSI2Hl@_p91@MbG+YO zwk^5z{n>&Q>MPhoTu@^r*^6{q!Ay#TPe6uV=1&Vf!s4mhm*wbD$r_ieXu=G zy}4|U*RxjOBhWETd>$|!$SggfeQK}$wcV6gnWHcMuoO@obtWDhe7EjHtM}{Kkr`T8 zctT)h#yO=Gy5N{${L=3zWCdJr-8(3~=j6<+J4tH#zefV0-Q|X0PqYQVy-T4K&-wk) zAad0v+tW>-_(~L+Dk}n;p;_jt)5Vra!&UZScO4EmvhOUNNr>RO?|p16zyVI`}F zvN?pyM(la$>jgeo)FV%u#SN;SWgl9kNdds$HD}$(EKuYz?W@&n9T*M$OGEm{UME;I zp86#V)K{C8bCsj=)f~oyZjzTyvi~D_lVkL)tznk?by$Vd`;2p4FNe;Dq7jxD%=Kz7 z`zb#V=23}wTl?R8u#e##Kh`f;js&|Id~uTS|+=j?)eH%}P*5}KPm!KJQS6wV=c$;p8jht~i9NUCG`@zDJR%p+gPjqG^E<^T+h0#91 zk8R?fBK*s(KQ6 z`hkk4zQN*&@Z-z-ZqiSG0(jwz;)tJ`|1mlHR)*V9O>wEX00wk&zwRcwg}Si*2$eJ8 zUQsGpYAyHPUbbj`=k|=j<>h#~C@0|GH+#Z-(0p`5a9|oY16qcVKLNTaU$r1#aoYTQ4b?#MU3;&@=(!lRuQ2d4%m8skvCd|Lxn zW240!fsV{I46RzZyF3Nnn3am@3_DSS4%$Cy-_2Mvo7XfKUPLjJn$%sto6~kvz^}Yn zNv%eBU2|Hj#7c@GbjJG>&`_TkWmQY-x?-Q2P}o-&i)tSIwV~ zGS);NPxh40)g)Zl9?Q9mcZbd(yz`TP78EI(sT4-Jdv!SPe6*XZk(s+mD{7_J4ab|{ zgo5B%m6@6Srlt&LCVIuWt0z^Rr7Bf92VROXjR)}kwv`#7LS{Pw?wnN?%#|$H_c-tnQsz-`|#d+vf~Rrm!&s- zLL!D?gdP^w59jta8iV$}yuDX1{4W-O_Yb-rCbH=BbU6V3YFEW8J=wsouotk>Y!*74 zJh7gvtgqQ|A8}x=%zhHhaZ_+oV6)a%f8bAi+~aqNmhO(Swze;p4R_r7Br3>zF(7Sy zaUh~Z=c+Pn8CG?K*w!js;GBu6X7v2^s5{Q#(!O_bG9foCY22~jD%XB7-1xQgi`||K zA-k(*Ru@l3Cp+qhrX$`lzOxl!(cM0tN(vA?HYJN;*O0gkycOT~$?Wpr#hMZZqv^>AwrgvB#r37n{7X>+GU83vuL_t5elxvbJ zDTrE+*!W)y_Mhq)-v?hrl|t+8CdZ`BbXSL4CEdIe5*9v!=bKm8_RWlqq6=CM8l`wx zop&$-tL>mBn3X^CskeWB&T zdD5fz3sgow7C7&6P91$$w&co+wX^EYxkqcJP_VBsH2u1`_C`OfZeYr6p3b@t$Ga+; z-)5sgBr``KL!0!ZXC(FhOi-bJNrlisv-ld0saEQVQx&NO&OLJNv9|3ZCnp~F1%^s1 z$S7ss&D&$)t zKkxlf^Vuf#KeUR(GluUymKar;5YjmB7q|Z=og=4ntNH?oDmCRNXlDJ!wOU8d#cVKAN z@cdwQxY$-}zdAY**{OaeQq74-St`I03{#$V;S9oOvuZNj8Vag+%D*GF*odjYi@G|e zdb5<`g=m*lTT!;2ZtTUB%&E{+IY^TrK2F&>Cmk8kwryE0keg?}GKF*efw9Y`DQc4w%}GNbGoT+f?sqo~yVg^t3h=^CR_sr_AYErQqN1CQ z3v@e)PHEW)5qHf>hk2qfLi=3qb8*9K!HL4^sULQa=H~KIaF^pKT7KvEv)H=aql6Y$ zh}Fv!Zk}w^k5@&Q6xs@X=(lyR%p;l_^P7ChVms(T32~c56USLmOgJwUSUE0VtLu_p znR(NAHf8cqfsjz83D{u~OLj;*Gc6 zMMdEXg9<%DIKFf!+3`T3^0{7c=El$R%seK-d*e%A=M%)%TgluJXspK-)8FMNj#sxc zXS@*}#nap6o>nKS6fN3UyMivzpcPEHd=st(#cwh_O7r8_&XO<#X;KVM2E zIhjn7s;{kR*6^OUDkml{u+OAEe*1aCXncf{=nc;YsvykyEW z`fJl?UlLEQ$gO{ujP5KGo77a^X{5D`zGD}?n@He%Pi5F+#CASyicvPAo~E?r@z(?O*wHE``G{Hx@xv^ryx<(Msf`Vro1ES`D=KS|MFDc`;L zfoiaD6LvM0RlV1FX5AoUA(UsKf+6VX|6Ez=da#uDBJw6V^~yPWTNrzvk|N7Io%Q)D z4wYKzy)QF8XFsQZHJ+)RUCdeAQh4dd@a=VNYn4+xN1UavR%y~P*^;7sKH0<6Y(t{E z=f!k*f@zermG4qk0Zm&7aY+F@#23AXuT~f`eQ)NZ{G`C^8N>?HJXE=82WRouR3k}y z;@E756DDe9_JBOcQfJLky%yWh!q&V7^ejD9yPu+r$>3_qhSllF&!ICZ?J=3JCaT|M zfap3axgZe)aXbl+B_u{9RSjT%S#vy_t9}8XpL#^+4sUGdwD5w{Oe7+H4HXI#V|@=! znDKtFVYzia&C%^yd59x&>Su8|l4w#Lfg*PJwQi%f#@9u_pJiT)>+y46yTUz%s*_Bq z#=8BmbSzY10hRLCvohTJnHG0l7k4(X8?}m?{ccm}-kr^yu4F3QoEhogt1;dVyV46I z^xs_ArPQtJ_rJ}idOXRn$F%>b;DS>SU1mB@jE0B)v7qfQ|GCr6<32seJ?DeYUcbg$ zi~Fx*)vkxD`B0Sbjve?Q^_epFK!)Z=Ah{ z_KQ9ClDKIaK0O|cxVhir0`vY$gy%ms0miAKIlEh2tg~bi~3K;1l=t8enQPG#EG4v(X zM#2>YeJcWOk-dUpw?yEqLfy-ArO>+;4Xwg9`HpX?8%4~*D0KZ%YHdFXCDxRAX;*VXK?O1mC>bz`S!o<#fTRB)WXicIG2;(3Q?M|12YY_jq$E?l1i zPGb?OOmg{5x3Rj@cg7ldb`iDJ)++Av z7f{#jZ5A+2m)mR9xyeN+O}M<_Vfh;5e!f^fuBhb_j*Th;`&M0PE#N@xGDccDLwDfi zfa!hKTz877unuZ)KS_2vapn~v8{t-Vsr@lJgX`Lc0yI+8RPfQXce! z`n|W#l+gRrnnbQe%33;*)k2jFM{{FvQmP-}&mJvI`@35&@VJ^R3MmEL`vbr5U)_A7 zzvy0aF`7rliOznKxTwB((nPH)sA+;j@!k!~$n#!^qxSR7iM*&=E_b zl`K>*fluz0R0%SnX9{eMmyfp#4bzK*7&-xv*lFiR-6Lu98RnEPus0RBr zQ`63wb<1uvjzTDfL??{?R8@Us=;Br6#8|gn>Cn^r>iR?jHfeh*GeTC1NjIu3P$Kjs zEec+B>4l8usZgTV!@0}2rm!a1J`bkc?JRH&se{PR)m+Z43F9?###P$Y;v~efA16d0L#=x>PzIT%KZH!D@Y`v-|n-2Spw$S%n(dwbkzVHdMRM>nA7@Uu+=r$97*m z(tcR~LYDe)HqQ$rLOYv)s*#^*P^?w^GavwX}~sjLh_F+~`KU1mS27vI^Srq#)P z^M7#0JhH(~%xcJi?aW{!p%zwI0ewmZFF3!?x&&uv_GQBxJOc2D>7FSM?l8P9qcdUMC+u3HGhKxF$@KCS zv~H`uLsx^symMf^LR_YXM`a#v-Q#M3rzrzJ^YSpOpKeR5kNtv28wS7f6Ivw@chwC& znCwC28V}iinHKMmW1+E8u88f>Kunm57IU&Yz(TXUorT%Ww9*>JrE6uiE>A%)$-X)-8<37KmDEhrbNnB+*|N0 z7;8sM9R6EJPAAoIz$|UR-Mn3?a6Qnz6_C(Co&cM-@a{ zt~aySdy&syQ`!{wS*_u^XTD}V@mq~6Rg&H8KT!+R8Gh17$xLI_B(hLbGycVNV?%qd zj-gPhwZmUdFZU(I%g`XTiv#Zsv`ZO8f7-aJHb+s(7Lg|B;Eb;+eLyQq*!~ObmBt$I zg93Ta)-E0$)xgp1KDn75*KYr|m;$LCfl@Ot8k&*wB#|60s<5<_W(I{rM)L=Av0a#W z;zNtdjNMf!nN9>t<-w6d5MR5DTlYU*5InI=0mP3H0COz_P=QtHxFSK==RfL{@xv#=eX|W zu$KsN{`u~`g^Q=y+(MdwdI;}shh>hqgkhM{1T}-l@ssj!*0E_Pqj}zRo4Z00!X6db zF>ljy33Hqh_1kJa^x79SVrCsj7SFb_$u1oF+RRf~sY%9PH(lD8ds;#}A~3Osdm8I3 zb=rI081|qCVH0PSF!Rmw7!TTiv7UjJp?V;` z4@(f42^ z53g%>c`5d_)-Y3GS#Lvr~QKiaKTA-{+J$UE+`GNQmr(Iet>Cvb*c|msZ>N}kp`5=|SBYmM^mk~^~A_9NsuDOl# ztBEQ5D~j0)GRr+#=W)Uqo}~FOb;XcYau<3Q%@|o#i4Ys~Ud%!xsBwY^{hiuxU*lNu z=(|6tpO=}O&=hN{5?b`bf5N&=848u$OGB>n_$I$SR5dIwq^l$Mz7)3;LJiU6#9@TOEJeq5`&NBf_dV>ITpM(h-ES^9nuJ#7m&2 zu)U9uvXFuGOvP?t3%6KrdQRUw9NHP$dmmlW%flAoRk&k;e?C*8Ol?9EcQQ#3?bPLd z^CJO5H_}a|zSNXtsJ&saLFW0p7$r&~Fd9)P*pKGKqQ4#e6WpPsrDxhFt1DDg>tGJY zL)($3vZeioiF1Qq?SmJ~B2LWGa4os9oTA;;Y5|G5Jj1P$8roDP%r6fZn1sUqe%%@3 z@A894!AsqA!~k?WPo283mu0=Ayg{LSWrQnqJfZuOUC7xTo6hp}sZ9d6*KNyU!6p+m zky94E3D0GwYF0`&U?GpZNk_}iOLL{CqO56(TLXjNv{1S>e=_fpyRToeC?2X#I7Ne<6lXE|=h&!Y0xP;4gql9mQt_U1joV9hL$ppWBf2f)Q zdpA2vCFOAbBH;crE0MN-f!N4C#RK^-JUjy1(K-EtW20)MD{IxuxDB3(>ba~C8@Oh#Z0vMN8% zkJR?s*3}VC^iqIp=t$9O0#2LIAwm7X?lE zyE8F#*VnK;`~}?OEYDNB+CI~1ZcquR0FQnHFlYyWc#s+ot|6)6S!Ox$kw1joJw3x; zTsM3t!+=BFKHns`hYe9GD@*CTd3-jWTP_f7h`w?Wy1ySFGp(sKrW@#E$_koo4^x*D z_7ckT*WONR6jZDwZ42NtDtgVd6R*-=kLB?_xydt?3HhCF%Js1-cledA_KHj*owjYj ztu0wpZciuG%1+bT@>J>yGn2W&?%RpdD$!JO=#o{cF5|-^hV@QdDi<(8CE?rCmEk+u zZ%Wgjn;NjJhLt{lLtX))E7mEjYkc$A;_Abv!b}OR=7X~`yf1V&?hHCAE)K#vt5_bl zlr5iNMBe(lSH@G{=gh}CHNFGg3sJp$Lj`-hqRWc81EC*ZH%aj51PSu;rlU6gX2{qV z(d_SLc+?dyjq*%Uucc=xN{D5-AiZYInwQRf^hxXR7!B6K7QIOSXCLOfMll*nFQ%kY3PVyhNDb~h6-D{Gmn{zNt~1wIWF$njQ-S(#fB&EV zx+;0pB)8-6$L!)jQ%VQ8E-^rA_`=q$vm#FrxHiewOr`(sUN}oS}YhH(Tex#Hzr5a+@9miUD_I0}sE7Xwg|M9cydy3%UAVXZ&wYkMAj}*2g+fuA|5}^KqnC)?-oX<6$6d!q0tG~1 zL}h2wC&vo7Snuuc+kfr9UIt#%r`+5s9v&V~P5Y9LJ9C^pjyH(Zs~FY#wD_3e!tkN= z!JF6l`5A9XaB(T+<5&Qtq(HVkoV0arP8XznQfpRN4vxOdekq2eEyv_WzomWt{4D{i z4ij*EJ+iUMLh#-{uLEKMSHL4{9Utd;Va3N-w%G9cH`gYYYu@3~zUSKSzLGuI1K(ez zrt-xGtr|yO zga*_}^Rqt|Ft$elMIx`Pou@ev-gAIZ^N^4byZAkjGR^_r)vI>i*6{NXaNddpihPR3 zvro1L9eyXvE1lakhtmsBAJFBDw`vmH}ohhtvYyAf2>j8 zPfFk3=Dgl-!eo`z)BDygVFe_3ZvkLr>GIiUyZ+8?JOWp7`?Iw#fkG->B0{Q}lI71- z+5yWU6U3`(j3l$^`+Zph$P!u*h+N?Pf0pQcqz|~* zu^kSc{_8~C!Y~p=N%>&IEg^H=63cEJU1D`8^`P4JDvHHzM>@bUb@~;`6^haCDdNLw z-7!zPxq?#-vY)$EmLkQ96@fQ&e3uR#-z!z*Qoeb(^-d_S*3rqZkc_SgjO{ShCBPYZ ztaajkq;I~MWpXuI`>j3nP4W((E4zY{c1WbYGIkstgUC`vEiiVDh(D8jr3O%8N6x_Q zkN5Cl#!0d#(l@*hl2*EaEy`vEqwx3MQhRrq*{~7dpp-ams_@w_2P-5ze*tJ;nLqv3 zOhI7Qkua=OCW4Gvz-1=}k48=wKJ6D>&L955$a5Sa=D^^rtki%hSM=Xr1v9WE2{UyNqUR!*l9-DD@h z9&R`gbDnN%M*c%dDLBHsK?7J4`BuVQ z4MgoMh20=r1y!dW4Vz@(XuLpM(_IE2!{3toKO0)8b6;%1@SO->0tVU*$0^o;i5*}j z2Gk#EkVvDK_}7Jll#c;x*6n{UgABIY0RcKidcFi~$UO77%~rP|$I+(I<(9yu5=tg< zKMvz)hQOb)-w;;H%cY-jHL4QJ?EIE7nJB{KxTBccE4xOCKbIi7tX=i@OKajc&MZ0# zf@e6)TW$AL{G2N9*=*RyeP?-5Y}sn`JFWF~t)A5{2gAxwS{yyp#mL~G{q^*SE?|Q# ze>n+cjWe-yDMLq_<8f&cL0v%WyaH0@239C;0ZWVu3JS_HNS%o!=CSSoY3xq{LM{%# zJCxQszjRZ*r)seW9ziGmCN==+*BI~u>gWtnUle{;coBRbq!!Zv0aqzO7(n=C4G**- zAE)4mYP>9aaDug9H(g~{0=x?|GF#0hCcU|pJ8%xhy@%t!C#us+gbZroUOyCi^3^J* zap>6zo342*+GSe>zZjjwC+`n7oh-*c*&Z$LXUea)V96>Bl;5+YSuUyRD?RCnDGH@C zTD6EBSRA*O-r%NKH|QIArlA?GE6B1S*Tb4nTeUNlbCva!;kYKI_l<@4nP7Qyxa??YBY>DPYuxz*8t)iP8huwG9`Cwzd3T}t z{M=w$2OaVJFX}7b093r|+$c7j{y$f^PyKGSNI}DzK+wpS;HdD@aH|vlJOSA2$;*k? zaw|-f`1}fXUyH?eK5`VU+3y|fMJE;yY7CG*o>pG-gxd=c9ZjRE!*9rcvsH&(7VV%h zA5#^3g`j1s?A36rr{ZLqMhkN+BO;a+GECDvDaRJ>u+9SAi#&*)B0cUMd=W5Epjlba zfce;fRZxt_W;XAx<}#08eqmvubu-)=dSlX;r0)LVyXE!enUgT|mW0;x4_q$({{H0? zYQQq)pb@&u%RO5=hFX4y^6_?>(pS%^^Ni3!lwX*?HxJjw%!Vw+EU+EinmpI>sWTjA zR5O}Slsos+Z@O~t+~jU<76#6Wiv=3LOSQ6R-pGx)DvNX@(yJp8@Rt0T%V%KG?Ve9~ zh~`XvUUO=CKT^p_hXDGCyaxJG*pgi`mmtSXvt`t8$2P~a(LYy~9hHD4Y z1~K)sV;biA#{E5Uq1lx5%>_b)xew^@!~r|q=;*SdxA`xt`hs8Ge1E!!KT~Sx?^k9} zEnosh<}fOhtc)&cK9(rW=y(?9-s>|0ISro%89MtS;et2(> zwSh+zfp&UV&Ib?Qre%7dJ?9D}%hAR2*rX&0xF})z?baMQ3M~%|ysYcn0Wk(G9%Qb| zxa=wQKM=@Dp%$puxUz;?#_)IE#i;V+y>TScZqbz5Q3DVAPoY8(%lV1cS;Q)-NMk_d>7CAY;`|- z|DNytj^q2seIK)R;=ZqSU2C1|T<6*4@zhqwF;KZKfbEcaIoN)yO5!Z|5B^j7$d2}N zv@_<`z8T@jN zh0Z8a=i9J1#umn~yqEHw%)C>RFJQ@EZv8w+!0S_1R+_caW%5MSGP{*cN;Hd^;%iaf zZZqDR9>QAb6ynU|E1hu7in^++`0Vz6ycQ|Y;-4>7Ga#iarklL;&V&o8=64v#6GcdJ zXE+%sVKD{TFZYZ4TT77H5BghA{VxH$J`Qoi8e1g)9pY`D(P(k0`E12J*Y~fCrSoX( z)tVnYb(MoDYRC0Ir76oej+>m!fMoaL=ve`IAlv;_HL8A6i<85{pL<&1sbp1c0~NEo zKb64KFam~-wyPuDRMcZWEtbG5Bu>av8}ON#E39X{HLDzDBiYsHF=@=rF!w&@_h6EL z${~!mi6@XIqlq4Ebuo}AmL&TL@-vN{FZL!HK33F^%WAtupvoqrHn#qP)RohsCgPNE z8m_Vg`}1WrHGAD}a#k;o^-L8~GqnAI)9IM}8%B;A52>_>Ze_Xk7pM)_H@5#RNUI0)&v}RY$c0=JpOpualfq}mqA#QH&_RP-P66M+^i>x`Xwp_Q> z0z+n(sAw;D^+3g?h4yXvj;df!eJuIFKSPCVdS?k?aqr!1&TK6bMoG)4c_{;z_=mKAtVd)e>DLIhbb6 zL3enhaY||2cSGdv@YyEgmED9WksbSPjuAM1s>dk-OZ2|=v@q2dGVN5j>gImTeVayS zHS$Uk6Z}Bvx8Pc}CrpatDB>J88GH_;gMH3X=zRAVokDcOD4nWP&Wh8WmE(iB2@V2R z&lRYazk`Iaik*l`YR$30en@za;;a4mv8)|z&%25036IJrq#DzW@atdnL@c|XY43a(6q28=r5TvuL%nd6YYaqK4Npgi~jMsVA z&({~L0h{6OL!XIVOJ%=slI*PN#WjmJ0t>=ho%76m*X$U{kneH6&E_FBdWgayPk&4Z zUVTcV&sSXr%vw&D2hvhr_az=zj;8HI|9klvM~I2@gaw$RIYh4RLlawPc<&m38th>e zczP6%2A2rcEP+@)-p=$q&?Wl$kb{S)+}P?(ztP06GSU=&1?kqj6k`In`3clguYmez zKreuflmEVRn&VGhtfIO$4J>+Cry#o#_S6(!IrpYnIHnIW@EWp|JaSOBVx+^Hq`io6pxB%I! zxs(YM#wQux$bl$1Y2%CH1z-0(9nzqLf4^`2cdmXwn67=u7y9)zoIYNG+QqvpRj!B$ z3%|`WOm&gEGGhxy)+q4prZh%disS9AZbm0D{;I|Pej#?XN!Bsw%Y%Y8pRP1Bv|2?+@cfip4S zr+=_t9Z@_zJ2L`D0+68~ndrqPW|abKaYHphR8-UmByT`$L1k;-@jQs(Z;*lbF)0-l z$6RZ08FDB^hP3?)PuAY!6i20`jrYrAK%Zfk2Ol3l3RG>+0DLeH3>M|1C2 z>4Abb1Qfo~U`Je+*m26~uHs{UMXAb)Fh(#1M|fz~Y(UR)xurh&SxrsDOf2M@djc$bjHz8< zA}+<_?06dxz?RKC&QD#mPC=#Px>{}w3^v{MGm97FAD9H40wp8lT=MF3I8-h0@I6Oh z6!G!F21GX&*%ekZ(ajfvbdEV652)dU$Xi0J&i6z4B7SnfO`YMC{Py?yjKVZ%GkWXh zOFOs!x$>9aU1^@&wTT~*`74p|g2mOr!^Rldw|(Zfd)aVc+2Nk9xzaDhfG#)$SV;CI zuKQpS-R0s^IM@Ix_hCRX{Gr-p#Acp{dA>PxBgl^Po8>7CZ}?#xthQ}(8_9gr%q`y3KBt}O7z1f z(ch&0A|>h;WL;cbeEGQK~KR{zC-|&VFx{ zTGhq#?dI*REyAun(5PUp2udb-zGkAUbkc15&!2Dl?n>k1v`?ry?LT7W#kEg0>2##I zI{nK_mSR!|?Np54A;OV0h^GlBMiWlkYaj9!3b2Da_ld>~H4Z&P8-@JYUCmPFu{zR* zzn>aT!ffXo6y0kK)_VH0maA;%79C9aE8+9qp^}d9NyeMf4?l;IHqH-5V3P08Te~C{ z!U`;bg_t_oyL;GX0G6DTOAyood-HBhQ65lF5kEgW&{;cMURX#e;?OKt@E3vu)DD;Y zzF4W{1S_yZ4FMQZd$wAUJ++S#sDtw+bxPhue54jocU&K5VU!O43>Sc@pXg4l17(l} zpbh&OIH^7Z!HovbpyIdBH;%n;^r55|w!azzT&Z-~ zNyWQ9k);BOqDzuB010Fmh>&r9CTH_R$WMx<%CoD?czMBB`O88( zw?TO-kU*GmzA|IE>%mPq@8-RZ&V^}0= zd0l?6JRtc%*b4-vnH(7fnrlEuITOL0?mZ#kD2uoN!PJVw@af2PbNXrgjjJ~ssB+bS z*2f;-$7W||(ezds9CRF#+u`r(Uz`EOm3%)otHJaT>fjk{i0$B32U}j7j^|m5Xf!G7 zODV455EEd5t=7CPz#eyar>)D=`E6sM9mt@)p+H!;Pg6RAH3j7`#H(Vb4`gHS z+y7MXDzhs!AKg{uf&fL9v{pav%DGG&z9WA7#kcl{Gy6M&R$#XG)4{ z@3}XX08DB!enCvCv#jw3Rpd%F|-09}HhwCOT014KC=A!CV5HrfV=O^7l zA(7P3xh@_a;^~^=n>WvaJkT8;QEwtFOtA5UK1T~uR|Ur^m&0>r7R@Lrkt6)6VGQSG3h*aa~YWvyG~R4 zJE8asT<;eGhyU`$9b2?taY`O1{BQ76Ub7vEs1OA?U2867#4`$JshYZL{pp4bqqc5Y zng~LclibxvQYN7+EahGu(Lj!*SGc5BjJb|xo2i^mEGwb6J6}DIq*iAy!9hj}@H%$3 zF*ZVP!N!VBlZNws@~_&ECQ^1e@2^~E*1^1rdlfrz;yOO`tKHM`w+MAET9Mt1EnQOP@l#aIpOQS;B_6Nmpp3C-ppGLqe*>y1m z%}!8`v;&!oygZggOx$Z*AVp2@O@O*6B|Ok+3-&np*(Vz1Uyj#D0l}>Ux{nM51${Ss zzJn7|CiQvQMZ3=ty5jB}81b~bs^{LPLfObQ%mooVnkUj~1$xn(K%2A3?^bVuP%S)A z7r&2km>5;7&tc|s-r~UTaavBYXok#Yab~Hjww}(l%3VY&HwcN~*=FTIGpZ#%O`Se;?b+YJGN)?2<&m|Rl@c)Z$QuCQr{sxARpK@It*$*q+` zEuU*_7o<5h^{EYVX~p=o?@O&eAn29p(t}=DUrdu<3dqZ!pPyI_R11%TLMfNz9e3@f zd2uZyJElVPAF1PKPAsmEc9qrUs}KK5Tj~&Ey5?0h$x{X-1$i%fZr%UZcza>bc^g=uL!DB5sy||@Z962k)BX?)mpP!ZPUALku|wv zw_YaqWb<5GTc+(eLC1rL2PSeX+STPGt}_{a?ryZSl@FZRfOvxnt$71&sKsQp8rYP9 zr=j#$PE4g#DoP4`x&H&p1xDY)p}TqX5hFiW%>C)eDUiB$2r$(MhVB*sG~)-VXP!p` z)8sXdLqmRWFyR;38o79Wtm-E-5jZw@iE`2-K#sMf?&i#`YXd+fm@rRW#hsTN@Lz1a z%IKh2g!&X_EEMm`aksGdbTxtG0{1h^GJT z7}1e0jtaF~w{(ENoUX+*@GHcEdUM~vtR;y51k8^#U_41yLAZ8k@VaOmXSI5tBnjN^T_Qc&;~6|(7qWC&1_X}j@6Ts z6EhEjBfZVXei)o$)|$NYFwiVZ0iVVXGMvx#LveL(m$(TDGK3WVvs|9bBbr&$L4sKa zy^yrOH^uU%wo|BR@{4%V6uG`1{O-?Topvm<+sn>JO9!g-S;v;dLK|23?6T>AHl1EKx1pz%iL)kr!CQi)KO{Nx%lU_lyEkUQ@$JN?;1f^X(k^!3e2ZMF!4fk*x z@(~KCZ0Z06m-PdS#zqQ#Xv34SJ52(xcXik^q}~+OZ?^QW<4&Tuh_D4pkUrv4mx9x# zc`q=>$#{?PWUz88=}C3YV@}a&R7>Ady293s-9oCvvFAQ6z4C=dqAED-!zhD^?6XKS z=(JaA{|g!2SE_XTc{D~{{CMUkdHRFN1;|)_hqLjtnvcR-ezM^8twglyfUW8}nQPHu z0VHova7;`6NHGX<-b(y)SK!7P4|*2K-P)=c@~hOT6A-QAmHSyjWqpZ%&g6zw?LNBe zmNsd8(3#P2eX8Q%d|K^w)6KDSpmVS4f8p!I+@$R@U&8Ct=ZlespgUz36Y}A5FnVCp zGYC&czz&d!wAIT{o}x%rG-3vVACclxIzG65dQ-1-oB5or8dlN8=NvxpsLn+AE9A9& z$E%hs$j#+;=1fbjpWFSfB8d^2RUpR>eOr*rjQi~kA)M@n*L@Zd!WK)KZIZpGy1vt< z)iyImuQ#}FuJ3yjcYIOk!<@Ql)j7yz*G<>+YE^ECrjMxDlKQWqGfHA?#=RzB3DktZU`2}t0UScoZ zW5VcBTlbgt&CD!m=J4KxZ%$N7y?h?Nf86}k%iMQXdG0590IuMp-b%8)(7-jDYV1(! zx3E!1_*{&kRp}DJEctM~a~AU+IVQx#Ib&|p{s8wO>fzWyVC%0S`E^15t}gz>7hB}bhRjQbCnC5k|OpXLi= z8Zxl|@Mn{t#!LW>&;hd`k%v69WMU%DzWQe}&N(8F#UHDWl~^lM>*FF{i?_Wt|Mc^t zpz!wxiSSWgcTqJTDPMF946_Z{FqMRtI+(sVA*2SEP7!#^=KcA;ul>2|2=*Ep8*`|? zVwPdQBlxtVYBqqR_Bhmw5%R3jHXt_JXgC=~LYJ#7JWkx5BjKPskoUx_$;vt7&UKSr zESg;i%Z2hd-KIIRhLa^&@k6F=uefseM^-{D7S3qYpzBBjL$=UK7Kd+0~$>Tzp z>j=)tR|{_B<lFSip9bybdAl0Ns7jq>KAAA(l;!Gt8_JNT&S zS-!Wyh#5-c>&aTrT9Prso>5g>xR>+t6^prdCpbIU5XF%Xy{S}||7>C0=LFd;`N0oc z!`}-g;7UUj{{t1%k`KdSz3LwaMOs${9xUh&y0{dchHbCUCEV-PmWgMXnK4_jMM?%l z5io%!FcX1?td!>*+GkHpOD_L znlT+Bjb1~{51UdF&T3qjI+;8g0~lk4r6)`iVH{1}SFhjriMq~-L&6Ryv?kohmS&8T z{To{aAR}6WQU(XsS7g{$-ePq_v#v%oicBOQ4%L+t}Z?y>1-~`^*$@m(LLs8D2BB7 zMG>v)a9I!X7H_o2)#2PCn5Mfk)}!pC*?kyi@vf{l{5?fWXUBJ9ei_wyEcesyA7kF7 zRZF-{BhVPTPUMToeAM+>Bug*JCEIel3ZNrds3|+gGHsR9w4Q;X7u@UQi?c_sk5AP) zXDWkI{G)!>tYH^rpPBWCRk{NSlsy@?a>*&!hJfivzMl8JD+Jp*Zd>ArSDf0JKLJCf zdNK4E7qmpvzPe}cd?dZgm6uOlx`x9Rty6m7QSImQKS0Dw9FY#wIfJH?-KKGU(|Y>o zEzR!lL)`RT9e6J-M<35Jt=VUAaxc&ft9~d(PrFWo$_^zLye4NSkc*tiIWZ#U zXtMOYcnf^M*MZ6_2%aRr6cTh?jrN#9gRiJ+j89IF1|I1)!NP*_ zr$CxbQeAfy-%0bu3AEk)#;g$;^g1CPpnI&Qn*Jk9X&Y|=;6CHr;?<+}xyIV9u26|| zw2)R!a_v>$nW>e4@%gSUC7quB7pT+tGv&J_67{ZsM!M#$&8_VaC6|u!F1s`$I#;^5 zO@LXHkeVp@Xz!Bi-$U!HGHNGOiOWf;fE`$_ee?IcyE;}T4=V*kNqTXA%!0m%>FC5y zqc3lGQF@cJ^FczQc`Pk5YYeJ3yIPo87N7BT;h5Mluvjpqc1_HG@F3|_?l*NRfN#V< zI%S|XTL(zd)Up2+NfF4gVT1h>6b+I?Qp!m~Cc}ib`aG;Kx70$X{GT+#&mNa}ckOVH zz;C?U%U<@%WIV+qa{}iY#PLej@*DQeMX}eqCe#(}RJhPxT782v*#Hkf%&D=2RP3et zZUbm%(2N}5p$XanrZe8-3xa5eMv*tfDWoQ|k2Y-ODS3k(9EWo>(i$_Fn~SKeJPmHX zmW+facm=gI+Wp!Re_c5F17L27YTZ2Izai^>%By}hHL$38IUn$!B!kXhfAxP}e6~d+ zYvB`=**Nrv{-$XCy)E!hs-YA<+P(c$|Ewm#e?gFcC0c)8yrlv=x}&hO)IV~xe@5zm zy|V0eiM3isj7a{@rvLdx{(L2$LAYQT4-@~-MfN+l_zO7s{R*uBL0fpVTu;mc$;F=d ztiav%35|{@NJ8{%*Pq-(Eqhx5rlPDdX+~6l8KFnmpX!qX6gNXXM__m z!`W0GDIUed{LxbQ#}^W=o8#&MM9sVZ%wzw0KG z2;dBZ&kLHYc||`?_y6`N|N9Pl4KhqCtooAXS+p6ObdP*SiP3t^UpA99z1ja6X#e43oGzC|)s=YJ-?#VAW%kc&VOy-nw_yeA z>rL;8Qh-XC5+`GhN9@gNMG#QI8P~ zE+ueV>WOEoudkCL6oQCM{+9~1E z{YhfJa%~_=vh6#0>-^wC?AF`)-W9fu5*AhqnhuJMQ-ou_6GSJpYB-m#_0P5a=S}`> ze{G?|r3TlcU*j1KWBdmMfy+#MEE&F5_J&!5AoCEzV|QHx$gkkKC0ZBqJ#d@AGsH13 z5TUCG2(sqC57d7y8cs|f-+22F*9}I#+b~fPk=@FwXZ24*cv%Kj{r)>D?mxN;3)UX% ztCTM4ZvBr`pZ{81fYnU-0kKsyF1RvLRpxMW3YaSaY5D_4j`3ej@jv&5f4!FCylN%S zR(GemRJ)2vvnXvLCis7g&%Zqg+67FotSrQ}Ol0{w-toZxMkoL0umAIv4EZ`!vL&F3 zh>cu#Fv3Kcn+2iij@hsOC;sg3X3Jk|F&X!2nm#1ciFvY8Mm~nKkmUa&;JltZ(**$J zeSqqx$<_eLN2LON@w8=@h{K5js;ZT@%&kH;rrk+l~Y`~AV=y7+JafenwaWjkq~n^TceKy z-@bh-1P*!2KT5KkVNW!#-Zj7tPx{x#5g>U*WCJ)(uD$$@Aezm}nf_Qa&ayS6H9zM3 zRq5wfom!9CjSUE(ElYp9W2JJv-VM->%H*B%G5mYyw;zcJd;z8}ud82KEFt5gWw!40 z>SEl6Js3e>u>SSnpv;@K8@!!xq9_M1$v6oo%cZqMzvFS!-1$^*h{90 z82u!%+|~V%EPn)}Q{(HcZaX&Z2q63)va{4{ruax(1%v~lXCgn=>ZwHo`Be_4C z_nI&gZQpbyyu2DC!Im--D*?f+694POv^K$nYUHKkt|*KeJCNo?jlV@J=3gq&b3ZSZ#x5nG9%@ zY9d^f)l1DYc7WQtBrrSAKES{wqXn*-k-)E2=KU?kWN;exSJ;_sZ5R6FXamOlrJYP6 zV9%cg2LLU@XMmMsuXd6L*dHs^pm`kzrfo)<|DBye`;L%HKi*9gTc=xgZ1pT?R^pV&>RLHpP zy7q0U_9(Q_vEf~HxW1Ju!giJSFQ{7h3#0YEQg|_Sr3g*~4u%kWjUYS+ef71lzr^vg zEn{x=pEoZWRUB__hnY!=6Ik<`OE9?AI_CYDGj86j`XZ`B0!< zijBB|TlVL6_21Xl!31wHQnwna9Z->bjJCK_nECDEdXU{j0e}szik@~$d0VxIGA@9u zdU)>Sa4vYy5%}i1#vr0 z)s`4iVB_KGMcReQv9%A+7W@vEK1CRHU%$IwDYd^knv($J7^MNFg6Oj=aAQ~ij zI&B>sM1c~i6EuQKMo@EeNjnTL3Hr@1W{fom8`jY{0;GUixhOUn2JEt#W@% zNG5dIhb*m5{3$OzI!|){Hp7Wfa_(wn0 zQ{V>Peo2!6ypdr&IA|4R`Rb0{0IkwQe7ajNR!bQFl`dQEgh>f?R;YbmV>2hgqFJ5~ ziZ15n8~6o7L^DH}H3Zspb(KV>S`Gw8GZDB}dd@eWWZ|quTQ{{^45)ctYTvT6j4lyw z|4d>%)(FA!c^=H<0@`$EcEHx^JAF{26NC`zHErU|M)Q+P{#tB=SGyu9Z*Pb z`PZ-0FWdCA`?)%`+HUUdvU|tf%GuxgtDLN;TGGHg3G(6kKTk^V$~XnuB`ILEz`*~K zr5Fj(;RGgh(gj|kuN|05fAEGzSMM8H1S{er=^Ysle_$ipP*M*|at0=9wwN3vtpFnu zD|Ff@cHEKZWu}}PSZl8Cy6i{dGPP*c16ISMcE8VFsaQ12@sAPHd(6A7uU^VZTiLz( z6!GBVSSs?wAbGcl`+k+`=dK>&yGubW_&2q3)Ok2ofVI(!Ty4Y~Vh_S=m_RU@1tGW* z&22F2a2<8Zsa`CVt5wCMI!0=C)Ifk7MG_oW05ZE0@C8KpzW(j5-8HmHcTUMY=%{(aS-XGlb!Fe(&9!=Q#at|ghy*2Uf_mC@Q2Ue!V#v2 zv-xJfp5t;@op0N{<&&L96GyFYNN(2x`H<13kC}t9ird4YUP`k(G#Te}JQ-Xg8i^53 zd7j@?7vf=6sh52ef4TcsE|(*Xz%hams0@cia%gk_@&B|3qS@kP#?IFoX@3CC*eZ3} z7VdrmBp(+Sf&W(5*q8wvzBqGU*`LabEp}3 zVB(#pKfIKjUWU(`-r{&lY|zxnDJTOb#vFc9o%qqN1X1!S37#JstmVXlR)Mn`L68gJ zf&p(%Q;sWRwIQH`KHzK>yVwySgUr>U$iw@0(x)Nxn#2`{hxbpWNGTVdSuJ%r(;YT^ zJg@){(kT}7kYkd8ZNnw`7z+o7NX)J?gu(wd7)bWJk&qMV-Wo7;8pru_tBvh zE#a@!C$mDS_vjWCOs4Sg+#PMv08wcqL9(dPFlvcijf-196us`*#q_qU5W zqH8^9Pr_xNarf6iGStf4H6Hs-2W}2=6%Apq>_&`i(R%84I{z2~uD0zUM}T(2lij9G z#dhy1*NIpH^v5AEfc)1R-e>jt`u6#)aGwPuTf;2`Oe(x)H8zlbW1{Mc`M*{B>m1hz zplQ!zY=>{S^ZGOl61d!~tKy~}C_O6$#p!;Ua~$3!z~oQ*NsH-MBKf~uc+C^E#qwI& zekd}(LpL2Saa?ctue;#Cz6gd`nkU8T^k4nn!+zcMf4z`myTpAW*y=t0e82L$`Cv%2 z!sD#66Z-t_?=K4j@l|l^)-8HK3g3SQ{oC*HvX_6WZ&g|AL-|)x1Kv>;(|oj097JOt z?)&QoFn|ItEOy75>brr7qc8xUY4^YdVp-3yZr+Rl6sM<~bN13BKnfPnav8BQe^l6| zbtMQ1)gR5rl@}Wo#lg*D{CRufAz_|9pHa+>aVjpT$npDg#S_77-v$Pc9-87E2YP5v zmJW5$@*GnU6VfqzCUCZToIbS_p9mO6>u>OgP3>Xc(5`m~#2rmS1tG{eBx}e`&ct+T zDD%2+3+2y!;M0v~V3|~H$@~rv(=(}xC_h;~{ZWE6b9Ef;ufl7n*+>-)T0EIpJ=y!7 zWV_1juo1Hqxtu=nC}mxILhg>s4ZmJEOO(@EF@qB8bpc&39swsiz{a4LOn?w}w=~hK?w`7CKQAhU!Z9M-fNT)5QNz*7AGjCuAsXoKt;z({n=^6=;O zYVz2tQ?)w49~G`oEd?e}%D^P{lcxg&UchVM46USE*fs!6@a~Ny$ zK3ys#XHp^Pg6~C7t`4M3e-@YstQ~TXPMUme(yZ;~V)?Xjaja2M?vTZ~lH!w!N^4jM ztqh_QZI$jyk$INUKX<=WlkSJE+;q{dmuul(GbuXm{m^)9I76%F5zDBkcUx`p8;kc9 zj>^NtTfOvbPZ|lv26nt1T%XSqyK4oX^J~$QK;)E8GKQQ6$W3+x7>_+(??%~rEMO0W zw~n>F!!R1gXcSM>f(2Aol0r7dIq7ITf0oKtcNJt?y7L{MiIuqsbYF3V-Kb_8mZv-@ z4&a%p-y&L=U6e)6g^zEAXB9mX_4^cVFp#=3U&Qw6XRSbi&A~M}R~QUT9BUbZrVGnH zeMC{sCiJh8;J?CR-1ORQ*v3PfQpvxHK;die?^^5S1b6ik-Hok6Z+kEFpxyiAerKwq z7uFjuGGG}NjkfgSL=4nw;9QsF)GYtZD!Vp`(4dvROfQ;I`2S}%HrW}moYDZC%+GVX z$Qze3vAf;%TSQpSD#g2$1#6gCx~kYs=WNVptF0_V z-D4TwN56a`1h6?j%6O=TC0uC9Gz2$n1B?g|iuD|V-+jL;`Z2-Z_Pr}0kB!6lmHgcO z?1wcTuo>z}jT3%nE-ATvQ7VIP;Y%@;l;ztFVqx2nW@6`(v3FUD3h2~crYcDl*$p{@ zK(3+CX+s-z1iafc&JP>68FsK2&iX4I%r0xL!E{AYYWH2m%Ym{UoDJpVQNIgB3iPzn zagFswntHW~f)ki_vG*7Z^r${9Y?dKn?mK5raHGGRc-dpfd;v-GB_GMM)olj)BjK@4VlK_erY>NB^lk?6SO9muEZfua<}gz*;9EIC zC+hce^)*ofptOCNwLA&>I1n+71VacJpg(y{eq3!u@)!*NY63Jok1tn6JGyj{bK()i zc}{&qTXM|)#ZEd7CBTka&n$YueE@20?b2P_c2nB_ASOY^$WOr`jg$V(OU)j z?b)zc&0w$T*lTe2Av}~^Z zLfXGy%`2S9z%ls^iEHT?+BxxZBY>^{6#I7g>X4b#vd4yWO)IL?gQ6zN$vo;4qmq+N*PRhU3G>Bl|el&d<50q+*7vo~33-nk}yLd2P?w885@gdxbUg za44)U*6PQ#r%Dl)#;DkhpQUAp;}0cjDm$Zw{Iy0|HImly=ANi-=UzgmgT$JF_&4Nd0YQBE42&f# zdkBy7Ai*g1S<(s!9`?(!Tt6%4>jxgwKALc^dmjYrM9Jy%3*1*Qc0n&VRun}da=)n; zj8w`?fB}3!ehvYsth}iS$%aCN?t=`-<`FPKGn{V=sf-6m zSIpfg4NW4=9jn^2S2%t9B6hym#Kn(qa+A>|t=$}^88-U){hRvk8Gvz8K!f8}R;18r zjxLFRg6{$P@*8Xx?r$9?G^n$#<`2(G3PkTpf!X>%sIzd;MUAj@^ZT~_<(?oX_swr& zI<>5O?=HfR?JmOI@Oes&#a|EJw5XEFt;CmVY%WTEJ@%;ZDB`01(~Goh7t3@9A5qVn zkzqR|QhdZB$6g7wL>JXA4*VjO>*P>%>8$)d*+-Bl>30~ zzS>4+F@#9YP6ggWMG4iuyfFUwP_0&M9aa&$O35YGrJm8J!rAHDO=s&d=BZ>bt`eH$ zN_>=e0DGmEKmDM0xzcVZgXxGSH}9Y?(&=q;(eRj%aAtZGg|z(|{nGG8q=qA%vD5w< z2O*n^Op3fv$Pb~QsI1nY&X7C~2=BPk!B`}(t4!{Sa!Swqvs*dZy8+l$9B8pNPsW&{ zsxpSjRl+PwD~2?i$mqgF#&;<)Q7Bgdh^ zvT%%G_%zHOPHzEMUf@L8V-{BCmUkEs^J@CPCYGi|xoiI@)%j3r#fDB}!H!XLstqI6 z`2iE|VJ{lo_>k4qw6ZPfc_P|@{}W;aLvYfGwn!VseaGE+so5wZ4`p|W9W2!zSS@*j zniT#xHMl3sLz>yy7#o>Uar_P~;vV7fz!*h#Z}mo&Rk;7D%qzuB$INu=dYbm!8uwj{ z8JIvp96mB^z%wt1wD6!!Ug%kAq0vwr4uO;VfuY=2njWx^)KuMhS5*_Wy5(maU-4Xz zrEg!2&1p)r!oCE)uO;xUGh#0YR#Q~0rqQb`&%sIlk{~f z*t0ba%0JIyqv%ueU0s@@tCpIYb2JRcD6S_9545ZtJ(j(nRV#o}cj|+1ehtbrkdPT7 zOhB!$RuAixX`Ro>?{e2@_AHfa2DB(ti^4Lt_M|HE4az)Xu0C5|7k6?G%+j`?v~xIc zamZ5#T=gu{uFbMm=Jg}zD$BkLop%=--~QGR6qca>%))Tf&)|>t$480jE`FN5 zq)plP{3u45$|IQp9m{)qtgj+@@D(PNBm2NOrq*UV7`P4f#}6I@Lk-UWvSsw6*tAfu z@rG%C@_P~?9p@QzzI5b0cIC`);axK@95B>LGFfU&H33R`wcaEKK<4@kSnp=|O3qkw zFM#u=m!uh(`!0z#=f~Yp4anlc{9Jm2?nlUo+4k2BDU9pR?4rr+FH%2tsZht`3!UZ< z?T}425F{6rT5~gsByJo<9Ns88xZ^nNG~fu^yDosHH3AsCB_lYNBmSyE`}uMJz(h9! zn+HE7PlHsY%rJ2#B%2wi{;=CE^*FTG&9{=!L&`<(w%}YjoUR;I|9q>Z?w(a#n`x|Z z)detgZ@24^A-Ta#r5K6AEzc+`J<8vRQek9$7nQp;xYr#oiCygRDxvIxKe5RZQ&VE3 zP~OSWjqOnjy`k%SG!h7%IDbUGvB1`teS7!=slo_0NXQ#%wE1pk4_{ksC`MrI@fRK% zM^^Xq3bYn1lhj(K>tZ#}M+>EcDjm8roHb0`50vc`Jmba#4-i8#maGLwq1$o$TL%N~ zh(vAUwlnJ&W@nfYl&YDai z{bS65I2Y=+Zp+fQK)w(`l_A8Jr94EPk}~M-B{_yeM0nIg!hLV+d0p@(U z2XL>0z4fdFtc6LAuNy|rtfYW@Jh9(hetZp^B!Jh^X=l_21M)pFth|59<*W6Q?j;>Hg;^hvGYkQyL?fd30)~!6=6y8T zIjF9W1aZvr)bZ%lwOj#}m@*c)@W%WX9i2q_d2DvW2hU^JFDNWl#I#0q5UwS;Bo@Mt z{lDWh26vJ{ssr`b`yH6}(DC)ipwZor_pnHHROiTEJ?iPiJt5c*(+AU&&lT8ZGDgsr ztQ(lWQO|JE$kF?(7ns_mtY;_8Eo|*?y>~LltFE@hc}W~mcWZl;i#YVahGRZ*eNy=Y zVmq_vj?2TBZbj(r6*nnl6wfU$nG=QWXX`l^noOb!5Sw(1E zL7Fs(c-@&Qg(A6?1T}!%vl0%NC2-Zd>Kk}BvwhxH(%}8f9f$f^2VBBciMTSi#rq_A z(t{C2Zx|~ax9>WZ&<0xIF{&~R(H6U&9Lc2CUD5J&V$xaIqhMSwcRe{?K7G2faNXeQ z(Y91^B_5m%ZRpLMV3>PSZKQi+ZA5m2bmq|r^JM2$_1iC(Q*>Z`-$?m?u#6YjdBf_$ z_5&SO?om9cr880v_a7xi78ZqR@Gdq+n~MB&TWAZr1gXrClv2k;FBGW~P_!d{V&-t?B_-xdDjL!Kp_E zYJ*cTY%me@Z~YUKfzi$mxj zdkZ-%s)J9(ZbJ_al~GRmsJa zrp5jahTOp?pa2h|D%?54H=Ig`Bm+1Cxe69?_q5^CV0G>M>W7@|O(by#4u68S*}wY{STF9dkNE}XXK2Q1V9-6cbREM)U3~Gzb($H?RWcH4Fs4c}{k9|Bd zlVtD38iQi(&%s^R%Ow(O2~-BRfr?5n-?x>(5-cq%IRhLkL$_Ri>WFtbpF(mF$Ik}3 zQQ>KMc|9c5Q0AxUI%C`L)syrnH%r!FD_G~Ag2BR3jJ#Yrk7@Apb5YitN_x#}-Or{3 zVu+loHndk#;zy!=JS7E|pKuMv3cSd}M;t9o_P-W)L7f&!&i@=aU8_hr->)MYK~7gx zZgdtXtqkSLp}UQpmY$riSoJO|qq5WTOCLg~&Stg!M7*P-7EO^sI}^AN#q++w9FBnIA8_36x?ig}kZg=={6ziWCHG_9BZOvHRuh zHx}##^AYaJEMT=y4K}P0l8_dnU?>ViZ#OS7-l#~jY6b#KHke$VamLOT^XqzOiRef<1ZJZ^JlBVV&CHmHBXlg$nyU?ul1 zC(IUDd(x!3i(^qo@x!^nd&;F`lF3KiI)SU6o`gR6ewS8dPjZd*nd|OPI^DM%$bJVC zFtOeDE!Oj7jh8Q@n&RRZ1rhKVvsEP`H)uw##ukh%7EK+z%KK2N?$SMzHnTm35*dps z67a)dMs{&b?8uG{xYD518svHowe*LemDLtmaT>L6Cv(J+88UFXp$}x4d4%c~&NypL-W=q$?A0)x~z1pfWWst0orO zQ+ke$y_`&u*5Gh~-*--!-G*XWdGQA8czjClvrCI4HKU^wLXhaJdw0{D}S465>ce)I@}uZt$K^oTs04w zaHbHrXf){}Sr)JL{vuzY#Mtp=lPNL!45XrZCx8AG8{&?$L3SM62l1HbB3hDRPtT_1D z#BFRb=f5|(tViUMeQTCHzNW@$AA9bvU$4n{gFk&(8%ENh4LB@qleE(!gyO5)PK++5Fp6Dxdz3=m5huJqGRWy7ZTumYY;5s&rd_8==J`oj zi@*UaY%t2Rhp$$8yN9Y};(SJ2+r8|n2!o9JKqEfSfEJ_(J`h$Ym|zEk{BV%rElX`K z>z}0YHupEJnh zm=4BYP%sCx8Zx}Sat)LAzy2T~xP}J1KKrQe85)7`m8)n3_j0f8t;^tWU{>x}Q1gA*)Q@M{G?=xJ;yX)lmD2_%&^(R+ z9bEZ1$sZExNfWu{^kz9SC!@J0LT2%5MaJ*pjd*xw3WuQmK%zzS>Rl6mjtyYmXi8L9 z9yD6^9;r!A?}F)AApRq|)X0n;uXR5szG}Tv=0~Xoo2VZy_W&QNZ43w)$^;C&tnYhu zMSSN{rw~w?qSE9(V(z7GxC&AoWezBrQK;+OT*_mXJtb2^GE{* z=FuxW8dIc(!$aDq%%75rc0vJIC?2H)d0>{nQTuRB7k%P>@_QcNoOoMQl~&7uoeP_; zj@k4=;dE%0>_u?4nu+&ke*U4mC!;T}@7v%nJo@DM%ou6G8JI_AF5P1}I<#uW^-L1) zk*7^NM=T>wC|=|G$4>XOq2W&CF`E*7Jy* z{%dDTF&+*8N7vI10;D@Pw(aD`R^;H)A~uAi1jr!o)7%G+mLnCb0J_Wu?`0ER6}vWew0 zYqeCv@m1snv^c;_bg2HcSk9 zMGM>3#`SDAtF3AypOAx6X`4?9kBDf?G^HpIJseAL@|YT$T3XI!g1hETfFt-R_l3=3 z-A*VpP9yLo9kPmbO4RLJe*(fu2{B}b3GKU=;MG0r*n^;N($g1wI{p-qtJDY4&$H5l z?0X$RZXb~(YPM*1luLhk@yi(+CF2cL!tIWcTw8Q_AYU$V%ss! z)#o#BAB%7+O~yi`01q3p&P_$LF=ynwn2@t`Tagp@!IxQ(K|RUJcZm|UtDd;kG9Bx+ zMJ9_8(Q7g1k42}avQrzu^IS+%`s{sW&pU#rR^pCK){st&n>-%D^mcMCc--bT*pds7 zg<0tMd8rW>!p`);UY#D4fWLn^NL&$lZEKd1A>X`Bjv36T<=j$eY{&pHMwKd zAbDH4_jL6Jc_yex^i+S$_rnwbd*FNR6fLaG$uHbX}aIV}`bLI&SxwC~Q!+$om5 zJjZt5I08}QYV3uH5nP({va^-bf!&`whfPv5T3=Wd_&Q%>Ug&(0?yae6L|*+A04Rf6 zP_M&Zp#_VCZZH{!2o%EULbm#(tnBL5C>Ej%$mZ7-JQor$@V1;q=rQLI2%UQnBz=vDbfXZQ-5+N7j<8+#>Nec5S%Sz5sr;6sP2gH}-IeBns?|t^Yxt{IY^a)zY~8~c z_40SZ?bOA(u-mS3`OE01!wa?$RP#ZnvFeICvU5x4q`0Ia zG{i5bq)s@K9M9kRwUUk;DBM>+DpljPLig8D_8$mJ_D8Rn=scd+6n2>#-&D!VbqCmQ z{FJJp04r)O^5eHFr#ibTWFRbeP8--sCGzDyV6pl(2ixCg0e#`m8t}DQlWcAE5)iqI zq%h)_t^}VZua0kh(JV*bq~G(0etAa!{3P}I*r|Cg4Lg-5fBT95xKV$KeO{=T)XphlZvrlKXW+Y#II}n#}0tPGc(XY!m2q~X!ha~Kxj^zxb&wV zV3Ppe+l!9sX~iEZe*DBK;iRyuJRKe?SKH`t%YC_H%c!bahw!YO_t_|)?BrPXrO}qp zg)rDN)m65HA}Phf#>$EJEO4*U9g*$b%3*M@0iGVl?X<)BxzNfs zRCQHF=JVSSxyIp(uaR%qlZSNB%(v((Mc?5(Jv8Md&NEY1(Qpq5O>HLwD%|`?0880| z#u|C4MF@?$ryeC+97N;O#7_Q6_~$4ChHDKu=nomN>`kh}r{kLi95B95gZ&b9fKd?K8P)_%d(nG3hC`u1dnFr=NW zk>zy8EaIwi8o;1g8ueOCjBNOA%6>S{U_!rKz9TReM@6Zhe`fD{VHPrDo}Jr+Vpw?^ zaZPojjrG}x!A+m{*UHDIPO1@`vb(>;ZY<3R5fvX;zhN)EY8U0=`}%R!3jT$;-*(AC zKKCWaQa6u@iZv^ed{)bb_(p}LZDNg9sWDD7zel)sS*`4)?{%+9e6q82x;8c(kzqH> zV6}t}j-z-pnM8YK>W(*NW}Fh>w9t~G&m~?NRFPDrh-j&ZiHu|p3nSiowahJg;Tlgh z)xu0e&B#FN3tZc=s2W^UB;V?r$*MILv%#3paP{f+2%&dV_4-U^OCKC&2QwmP167g} zwDpSkJ|x(G$jmJ)ssF&sa!K>)*lBj#xG1Lt5;-g_pM)On<;k&Ej=$CP2w8}S(ull$ z>+Sf=(Rsm7R`$szDC+74mBz#`h%IizipDCBgyv2;3BQRo{YK~;@; zt4to>a)ssF9e9c+{==t(EzxZQwG3iT@}>cRx6!gl^G4~A=ul!_ z9eztStx<$JqeXn;o!RW>orY(hOIaT)^vK4ly$hlq6TPDOg!yoj>lV{w(CDnwDwh^( zuEuLMEf*c6PR)G#k@ZE0)zT(}z3?9VnStaL1HpudtC~3$uL?=TLrtQ!j~gHB2j-^W z9G{mU8VHRG$?TEuB`qMgg|Omovd?ByX)HO^p*e;l`7{yii`DVhqH=<+6}LO$dp@hB z!k-wpd>MqzvCr85L@gD^qKt472B*WORm<&UdX~a_^6&G{Vi%#n8 zfE-|t4Jf!9a%}aP$SrCQ2O(DrC=_AlK=Xwi;#V%gL5QguhK4Q)T2b&d#9Hd~NxnOx-@b1Z+-biC+cts&ke3O_^{{nXPm z_{*j*K|`VAcFZ6ZzB!&;GP;IqBi=IxQawCRN!U(N5-lh6X2o#3&gs8co8~thfg0wQ zi)gmjUQ@QK5L7mAZ$9RUfhBPJ#F;m;ENpPahqq_Jms(d$qttU#pSrU)~B@u^Lugkl48_X9V0jwJeunlN}e^NDdaK8F@H9 z9&2{kh>%K)v0xYiWNUrnIfpzUiC9kMxsb+4)v=>7*xI8}F?q8|BkHdpifit1_?0fb zj3h}0BuU-;3iSEm?pMNrNoHVvsTz~C3CKoXl-pTJH(<{bOhTa zIW%);6Cmzeg%G-;<+y6?6NO|07?YvtQx}OEWy7Xjt8iN?GiBG*oBX2B zlxExDFze;BwyKS?8`0+ky(r4y@YtkeWM`nlHK#XQQ|Zc;LacI0%Z%k1U@TEwUz$pn zetL54e$o&3;BWG4XAGFPA8k*0@kMRVPYK|cNii95ZcV@TT>pxYN{!mW#u7Y6sj{#Q zZ4LmHFD|WBX`8MM&MqzSbTs+Zlxx3xsJgSbF$ljMF`!lFx>|T@;GXjR%_eHrXh(4tN zSIKZI0iP&dk;IcyQ9JGJs61)H0?I%aA+1a2i(qfUYYi zORp`rtGUwcQ2eR~s|4X(J$i|#!{lq>Lv6ls&XCK{8q<7C7?~bBhwxL>ZJC8bd6biA zoeT~IAB%DIvmlE12XPy_bjk4fZF|dk5Q711sA0P}(SZ2g+z$wl&!+!6$Q2nN)-T=q zoA~}Cvm@TX)eG>WEGPTn0tg2lyz*YLiF|}!P&>0nz%75IY0A1%Xy$T<(Z(B*wLVGX z=Hs;zoIOID%)vPVPVCfYT)s$>9;(n^#;bKUB(9C*t-24TDdBRiq}wLTTVcOU-YurH zwrKn)og0=DBiH=4m+_*&N?D6;eg!XQo#^C}!1H3+W0gp}oj7-sR#=hOEk&0XY-i>B z_(pgXA{0W>nlrWErE8JY)cTq$iD5_hn=y9MJXj-1ti*InQ9i9$x1PoKB-VH7p7Cl+ zX%MKkI&v9sgLuq#uMWy*jl7M_-4@b;YPFw=7y2e2I=tj?v zPBMz}4gATY6=pi{5Oi`mF9|t4cCZ31bdjKnit1gg@WCHu#hvN8>s+tF3#!RQ_=>VF zzTFdAztBV{ZbtQD0zhiUSob|}s>Vn+ebcJr*UgS@DmTb5RkV3`Qb9opjhPKTO>ri< zx!4)@OrZL@TC}g*Z5qKA`dC=9iM>InE~k(H%s<~p2U>cK#cdRBkY5hbJULGhZH+D@ ze*=F_g=rjHz{)i8 zxGSD^(}4bdVXNNE`!zg{05n`JU6D#|3-s`blsFI0031k>gwRk%00!Ee^-Kfm+G^~n;#Onp zCGu)dZo^W+(sQTT)kbJO;}gzM9&XZ(ct1&pcl4Ne>3z@>ElyXiUyUH%ctFu9Xp0lM zFxq}h-cMAtsXT|iR7oFC>7_{Hf(ffXs0=FIVg!im$$7H78QQS0^Vg34e4StWD5ZZl zs46Kc2=O3NG1Ru*Ep z-L`Qb9+Zs==+*JDD48|K*kARsVd-urA~oLJI@HykBl_=W9we%;j0pwNpli72Y<3hE<(|WH@t=-JsAzSbcfDrlqG>m-#`=|@9 z#xPy}&F9(kOaBq09EjVEZYxbzsQ)02JsJE*w-kV&eu?i-yX5cJ$nN;xv)?x=Q1SBT zGsihX6B4e0^q-q}W0G9J%aV9Wo zmC*#D8)`l3DlO<;=wwinNa$YsZ06TAYBd&b(m**MQ--PypDfdf;agMb{zOr@oEl=< zK{+%>Fs#Iwy6uEdtS@W z*9T$@V38^iDp;Vdme~^uncBUcZHlFp+y)l*)y$49zfICp)9E<|F^rO;VtDDOr_v!x z`gyghYn54r*0;3w30slix*|4dm(a3B6-e8~Di-ICN`tlbZJCp(jUjPS#x>%$nEC^x zitCCiVcEp2!6G?si`*F(2kQ`yo@}H%c~TIN9InO9uo<8EgUF=rfu&2)N|HtW64@C` z&D6ZsGUxnA1RKt|)*?5S8wU3}`A#L4Pv*^=H)&uiNR;ML!F^Zr1iavnieone1imje zdZSN2i`vDb9?E0<4tg)h-jY@>{$Y94ojD@2|F`R^WF0CDEHF1zG*i=4v#F%obd~Z! zzR%f0=^m>X1c!aAkM8zx#U0@OaYj(rvL0nW>SSDc-)ZM@hysW-4K@_r;?sA(S6 z>uqTZL!9_eFT=#0&&xZQmi5kfW%c@fy$=)Zp0v=zk8sZosm_gg#DDN5R?iz(0T$H! zs}R&2&(W4>rRAe%4J%TocB(cwtOPdiKF`&Z!@S%?WksC#)8KHiLW!X?q1qkwSHct#5HGA)H?D5>brI`+U8pQ>(^(rN^(>4 zg=lFQ7yI9)UIPfxo@?{7%2dt&*t&oH`5h<@Bgx9kE{FFf;wx)FnRKOTd-Q;lG`@bx zq2un-ylO~G0lQmc0%^y^MIAe_czmMPwD^*!uf|4g zpZr3XkV_B&Eqvf+qZDEwL*?|QkYsWt!OjE zcb->4RUzzPH7asL`lKNb-rY|#AyrmsA z*`Et}iWT2UNMmJHA`!1DY=*NGt_^6XZA4Lvw)eSze}BfPq-p@wUtQ@+h+<~G!YNN+ zGqvpn;HrRPgy8+mc2AG*sf*q0+P*W4{j^d-FYGK_s`-p=C4y<^`)lb!w@+kNYr}r&@5*uU6JUuqm9VQ91y-{vYFDn@%p!HZ zH%GbTV8R{eUk#`fKkSUt8r~^zN!mbP(tCUJOKw$)ngj)G|-)La|I`u^C`Leg4gv5()INZYY`u@ax9$A-gV+Oq>KZ;i7Z z&vG`7e0aN9b^7dCCVul)2Z;}Yf6{pSjslIB5-+3uQ{%lkm8o5me{^N8JYKf+x}i?t z$G8pRd<)%3)|rbwYAM*T*aq1bGz024wV$!^e8a4?k7g7UT+dQV5Bl)&qlf;-9H%T2 zG1e(P-t_RR76B^`{jtO=u@^6%nG`T{nR4!Mwo*rWk@Mht5kr?4tO26Hm!1wSJtOVSJKvQ(FFD1Z7 zzPZ$Hp?}TH1mrQ)D&e-q*>}$G?cv`HYF)?yZ~gw(5U7gwURbpMz%jXd>YL_G#oI`$ z49zJ?dN=J866!J683iN%xoHJJw-@AIL}a;!dZY_{(|(GjPNw3BaTd@ zDQhGZkRrIFm$(Cr#1-TRB2d+8G3`!YQMJtQPdwdD!h51=;dwMaWnAA&u zm^FXDBfzwWkfeBi4yTq%zD+(_Xc`eyCM{~#J$zKH@-|(TQGT@cmU74KQZh8V)3B$d zqDaz%wH|){s%+HbLF@u!5vDa%St~_^7t(dwy@Ynx``p!)-90@7>4Ux}bMD-2cC+CS ztx33|5HHm9*-UGE4Vlc6(coc(Hq5sb9oU*MV88o1_ZAV)`gW`}ytm(-eMk^S@wqV$ z!pz04fM5{2H#oL59dCrrd3;k*Mj?ux&WvP{tTeYEJRnN&QZOhQ!EeCpr7;?PpxP}8 zbiofkb(j5ZyYEfrk?SBP+Yf&~!@VbtJN;x(ndO$+;FvX@L@wvj)+knPA58l38}&EN z*oZqRrq+1r%pvk0VAlIKOH63vw|Js&m#h`e9XM!)?Sl`?ZH#)h-vJRsm@{ih#3z&N zmV9??-L(IH@>|#i@jh6GU@4|qSy`CbxzhORangxMHH*H;4NCE}nEPrO>a_%uHRo8Z zyjF9U5c)IcjBZ*RjBvuLjbP1I%6<72vWgn{Y(@HT2tf{#X^*JfHc=2(rY0n4+xK4| zsJ)lfYUNgwOT#;OAcG}lTNsNj1&zX2c1it?bC=ageBj?3<8Q8~l>gBKgn=Sfm`csf zg)&6Vj76{$;}72yEQi~PPzV-r?ymsKZ*p)Agip?Gl0EzYTaKqM`{Va1vr*DISK(qK z{kmx*Wg(%?JWKaA;?Y5iJRZC%-8LZQxMlo)%$G!9U({+&64$B|G?>pyRGm7iMkGXa zW_G{9Z*Ii#<%PAfV;V=#bn#;1YA;J@V()f>d@3lfhy=%u8yYe1{@d%5x_j)yr%wv0 zvJ}fBk3k=*eAHRVMyxDo_l<$+Lvs=`9z8msTWlHDPzX@0zrAvU@^3E2SM+$Gi+?4F zLwN#1Q|fuK?^q$iqcm+-L@xbh}zqLwJqv04y zQ6hT+FF(U)reb8@fA<;1A56*qT586AJhAlI$zP^~wX@2TQ$hH=u?yYTrB<4%U0&vx zboLB&i;0i;KdsAu<_e_7i;oVE5^8!df)u~ijv*Ge4xZ(05BEC2Pa%w?&9 zQnp?Z);}zWzu$aE3ha-A2hyPbe&@N}^QD`K&OZlzBtb1;a?(_{)_05Dwn$hg8k`ye ziaYl90|ui1{d50ST_nX=aB%GsB0>nK9NN1Q+2eFMf>kvdv|ItxQL`3@)dtcooh=c{H<__ zRq8-UvXJ&#$M(ncrtja~X91MJeE-|`Q^J1?Ft$N}fvsIQT>UfbZkG>GCJ8n66w$|| z^`=S~k4=n8GV^QijK4KfQwy(J#pv%+;KuO19aE0dWGW=9QvgZu=KD-jGlOUZU;L=M z{>d$L?^{0(p>70A%iLWa`K1y9vT(q=6ivj%o)LA;&hitxT(9$CXB{~(wlPX7bo`RT zZHvv-RR@)X_YDmB`hUL5PhEZB)K^Y0UB0fOGbi>0>|lPZIJ-;Bd{{(VQ7&?Mo$Nn> z#tz}_DBvt*=HO^Pz$$+J%6p4|8-a(O4=#*W58)~_ezS2u7bn}LfX;U!6c)dpd*oJt z1u`TYU8&s?Hu=8LsbfqHXmuS0(wy4dLFx zx^VArQS4{?(xL_Ex1VWL6g&lmO`pQ3TFh3@rlyb5l`83V-yM*&PZCJJ-qj8-CgS?E z6;eNk1epfexLYM$+7N3&ZI|)^oLUdJIj@z%c-mSz^ zD62fMy4vvuhJzwg@fi6{h8@KwsD;9(Zxd?xC-`)H$e4UX$7sw-szTvB;&uB+mabzn z`M%z{X{>Y+n6pOV(H$XS_i{SoFu#QsES#rmh%u6vT;DfG@t>a(b-CU!H|$g-Z~=}2 z?{EZ7_5ppZG?=!CtbNV#kD2pLIsdTgZD*dDqkB7YJcjIZ{pqFKfHmRFxho+nm)OZA zX&4VjoRohYPGz`3nqFm42;>MhKs-KhP^o8TIz05;qSLU4MJCmen#^zM)1CbyJdIVs zwF(ih&V%IKUxCH&v9DSnurMS?$9W7N3;NX1u*+i0j^9*`6||LMhL&jwKNHuP@naLK4#4R}{STdX0klXbT+hp(0jo7fJ_VZJfW+7GZbI`YUfw>>B%X7Q^r_-IUs{3WvHgG<# zwf8Kb|JadUPGjr=Pqeghs{Siqu7SeNbU)22sjA0TD$X^UA!UN+*domDPnNKKK9NXX`zB8dBCEp(7%v7G{7p?~@I62I!DJTou z1bchg+-g?1AioMI{w3J?xyp;n2Gwq=F;XacG3$fR zATkikdr7OwOL_64B|ZcF0ITHU2wsV<`Ioi*m4P^y1oDROil%Sx2|`Nhm`%`_4cBED z*A-ZL8%gLD&b+Tk78yr}Pv&E3?vIy{#BWrUr-!E8QBdwk+8(I30p$y(02F`WY4xD$ zaBT0SuGsMGR#0gK00xsxLL&cM~?Mo zG&-!hL*`}9d^poF08 zJG)uW;GFyl)U;2>VQL!Sz zCM^FjPfo?E*NswXq)W3ES{pY8w=pT&`54}WG15w8M6Q~n>4;%9rAho6YTas+!_Nu1 zck}zqv^U|l&0)Lb=$M5>UXv#iuS#jB^aP_N^DAVg;kCEB!^6|2Q*LBfTh4ZO&kvj~ zoc5cT!X3&p8)-qwqqnKcb7S-uUq4NgO-@?E+EO-ktm-v^LgcW>5ibK(pLgjPIyc2A zrS`Deg{qbV4lTvjX3a9hnaejmv!s3Mt1iUg(HUu&-tn6IXKEirwOV(|o+mO~)8v76 zTyooP$Q{9(;{F*w={36*XXLEi!IUR0uT3^Zm;oR{^Eeyagw3Bk@9!Z}oghppoSx*I zl$f1);frBAw3h__{ zet-{a7x-?SP&&p8g+d$FM4tWK@869p&OMgOg`Cyj4Y7RYK!{bgS>^-K?8-~hi<+$z z<>f;`S+2*mp&utn?{{jlL=&}@hW0kxKC<;QM~)tgu85k$<=EZVpfL?@Y+Ix}<6>%i z3vDd|=NG;vfR#dN9pgTIISiy?b!t-%i?6oC!y$FFcdMTW$*6LuH&i6CC`HR#!5;Dd z=`5wr53Jse#p>4WgJ1&44RXl_ENbMoHG$-ZYP!I`PHBb*Gz1UCD&?u&0l{ka` zaP%U>z8ev(iU|F*(m1kGx5|zVFL~lG@9MWtY&B%{ABo-NPJ27)r=R?ri}u$wMhEbc zhZd?+aeFWM-=hAH-n#R0cQa1JcclMmo&C~njQ&#Ny9WxxUUD6K>9s#Nh<`mj-!-5P)ECm05fMrM`gGp&pc3(=y+q94OC*1P!yjV|-~>lA4vTc+ z|N8U?*T4*181v@dE9LwjuetvKnKn4X8Xpz){MV;{)B$($)~p-+YxsH~ADD}}YPXe3 z|4Wd2^BeRQ-1)$Jx!mR7X8wo)2*v`u7H35E7S(Uj`rp>TITLW_$M4h5v;Ve4PL=KU z;1EZ>RpsQ+z2GlsqZyTrhu@lTHWZvz59y{QR=Q9F8Zi;2-a***O>7og8mlfBs*m=G66F6*8^W^}klx&-s>O)Cc;rgPME2i*O4R`= zQoPcak^47gvoF8~2xH*9&2KXknpy`k^poTDf!rP!FJ9d34^mK7b^CLD=svQ?ARAX& zkS0R;W2oN~>ps5+NK@Ne(_yi(v#vV=Gssm?7GIt3%yA4PwDd=Ph=vbeAjx{H0rt{% zs8vtK)PUZD2X72~*QW|W%}_)$xMRc!?WiX6$1-t7hhw`v6J)pIek|R9x^u4phgWNY zNRJrkJ*uubQrQmpdFNdd)$clh+|AOL7d(oJip^P;?RY_f{&hg1+l-(StpYD}96>9j z0_Z}#IN}9t+W^8IfOwq{8`_#L>PQ1nx4R|IF@=B(VmE#0K|o_~$*qU~T-LuY7PehE zKIUp)|M6`d>cCuI4lC%Qs=g&F>p6mgp{lh2nJmF|U0t&PV1w=gPi^#K)g2dldo|Vp z2{7~J%bzs_aL%+e8e;|b13N&O{lTr%bzY0rtfj-CJ*sVWam={HevG@wyseoxIq<)y z9qiZy74RBJ^_Y?$6VJ$a0I(^0&mp6UXoM=O17{T=>6`@|3nwmImwG}$JIx2Mf5Mo$ zOYMzyOYIU0ikz%A;82!F&PUL=j|fs$s#UU zwWEAG*&~+~0r)>3ZU_`WRxP)Sn(KGe0^Ct`O(dG0&-59<)BTj7?{l6@Z?bznO(EiC z{F*Xp#!3JVV$GTLg|dm0chyC$LUlp;+*nPuPvevb2ZtgQKF(r(g!X?1G{(A3^+@vq@eVp6_U>chNLI;Er<{qeuR-YcmZ*CKxw$H&iq+`H@a>C?Lhvt41i z>&k(LDEOGX(A$6JmJL6&iqW700GmbfzCzpeTM%ieDF{@q#7<$IOvM(*b zkdBuG&ct5bLm?1-DJUV>1tb!=i&UQu;GwvC_+VafxCEDDX2xXI?XKds_CqCYw`udR zDKNGnCIsPbJJd;6?qJ9T-ggJa06EwLSy{ZimFTQzCEB_#96(pcKlLimUIzRIX88?0 zmZoiPl(&Wm%fITxesf;yX3w$pCp<(JS^PRpJCNyrh|aYb29$-0hks9E+O6kjrbDaS za}js?tl~!^5O8#a9X_%J-VO0^<*-Z)*G)#1Pt%CDhz;G)vmFw#`l>EBoaBnpHK(yi zY##8PftIvq0XX0GN(tK>q>45n$;f9_^Q`6eRE%+J=6yrsei5r3mmmuY8^3QKsjpra zw;LA5U3n8=oam&#h^;1$!ZIr1+?^>m$ct!cY4P6uX8LjhuziWYb-?~|IHd0HMoBh~ z?N5JPv#9b{naIWW&P}+m()DHIyHw%@m0c9S0D=UZdo#ek!`!wP169DffE~g_(Sj?t zUXN}y8@~cDQk&*Qh2HEpr2Cw$E{+#**)kS2^q7j;)l0>IBk@Z$C$o3gwzK;M>= z>g}w!ySrQCAT0^0fRP(<1@wT0>{4nh)R63~02^-;7Htn1w`HZss3119FHh@|BE!S4 zP#e000aYstDtAq?t&%x4GRJki%77K-ZUbv^*L+?4I1Ka~Kh?5Hi1@UcEGa{QydKaL zhVDW(*K95+$z7xq5eK~P9so<@tNu4N;K`h#ni|Cob6#U`*HGtVVpt+F@y zA7$!93TCSeJMq}8N3P8Fw{AUBQjan6%$kWSgTB+mcC@imFi02xY-sZgX-r*)S2r4l zCLHIf>J)F8hxubg8e!DkHDc?N5$1*lgyciUBS;&}s{iqDm&roJZMbSzSC{#@xB0ZW zFv1+$NEnkGG|1{y1pr4a1Y|kv+W86W6OF}!E?maAwEdp}#RimtZ>ZwQH(+&4LHmr; zs0U7D2@<*kjC35H7=wdjTjpj#*jP#-ltK6v_l}jziQ>?4sFU&5T#ScbD{qy%Uz)D}E%WVRaB(X7$+F zBFe~ZJ$d|i;G>Tx=5zs6v#?Zw90}-VYa6~mJsBc9?wGp=M@3^5Z($y3*? zti*4>yvUz+JC=_gA0F^)fBpIdR7c}FZF7%@?BGwOxOq!T?tMmjdhDoWLHCf$M=I@C zZ;(#Cc6KK@)yP{!Zml+#B?|IE3j?bmSLR{!i$tmDQ zFY%m8*vanQo1EX>x4#ELr`R}v8zsZi;qmYOGuiH(@1xiJ_dDN!k~V#frG$SkA7=9h z;TqZaU8;W#GyZ?Q=%e!~+`ql2N|=<5%`m(5bMrrXv{ET$yF=r4i^K0ptd`2MH}mz_ zS}@|baN##YqI-VuKjir9&;OI?|E)8G5&CJ%UX_B%M`Fa?G`^|h%x#3+pp-Kfn-H(d^uH44<4jx=4h#bHkL~KKm5M^pw3H((e7;G z4}BW&g#$!=42_^~K+MP&yvbCAszeynbSSYQZ(4j<4JbQ4 z-6ph0uC(%Zyz-w+4Uw4B1_9Z1UqGVv!~=jspP{1CG@U$t{@V5Z+rvMC&K3$czG-tzGOWXKyXbcp%OkD?Lfj1wu5m0pjdpeEaxI6#-^^`1iLnO6Z;eU#2^_%aoz zRwVE~Dqmh$`Qma;X*X_z6Nt)-s zDB)eU{$LB_0or*pV`F2jK+)bvPX%Hq^!;{l*Q+>K!I3$mI^M32CzQ}L> z>RPp3;-_*D%_80axDNPj<((0?v8F_~(b-1lF<~knNBf>39|DQizUnhuK0jwJVDeh1 z-SGv1iQgoMiq32emrq6OUouM&Dy|ap6H36JGp%IHJJxMK_j`g?>hqDwtvs#ID}hD7 zvdrHZQ*Ov5(rX}FdehX^p{#T_;9$AMW;p?>Wt$p@WT`Ga4pmp^@;pp;DAc#{QWYV_)6fF43Nn=yi~T zyX~MU_h10mi^C;(wI5I?2(IWNw|ldsTvw7tgedGlCZf0eRZEG=x9lx%d|qj*Bi(u~ zX`=wCgxMFARO`31{YG#%VsGK89bVC_z2l}b`dv82ZPcj7yZmB#HjW;El{sr6hCHi5 z2?rl{#rdK&RdRXg-0g^b{T$m_OP$1K5INVb(g*+_f6T4jX<%*Dvsl&Be%>n+_e#v8 zP$_e902Ca|J zEItivq9vdnlTC{)I_Nzjyu6aaVb==}JCbUG(2# z+J{>k_!&t7B^U68tFCeMi_cEE=MN)z>L*r04s3^W$p2CCJ%tfQie;(wU&B>ZlNth5_tTHLAT6MH*7{yqrKxQ)bPXn^fv7A+g zWY0PFp2L+CTVISB=0rel^K4lu$_%uSA?hNdjZ17|Q|(l^^{z@{m&bOtu~YPeDubiS zLL+Ia8GeJP-K>@m&QH*8*fN8r)Yt^RgMK+U<7;Y~)&gr4S0$orqWiFwyUQ3+*F!RA zrI?$;n(7mmkEHN@HNiqKzMBd7i_yKIk{z_v&fy=$sO-hx#@QTkY zC>CK9*iG}SZ`(RlJ+L0$4eKt9%dgq6ev3RJcsSQj(00&NkiZ}bSSb6e7Uw78vWNx@ zpi#Yg{lqonnyLvja-HUumZUOGYtIUxPK!%P^1W)kuObCLnBnU?>p>%ElUYe0QJOqP zPus{O9hg^S(IJSUps=2EX|>Kl$=F960;l#yteWYAUpAJAW-kzYgEBGcry5zSTgPp{OhAMvj z8A!tm>_0##p=0hotqo7n#@_9fdVDBRpe|cM?&4r-;mUFAEk1p6n!)Jkp7A%nx_QPA zhHji!=VjQjOwpF(wPsY4%NLzQx1) z0MZ9yCJ^d5*?3yc!%Lc!kL&BwiWBgEO`^HbEA<)?Sx8PeX+z2Q5e2gA1Vs$f`)4mAu}=hMn(kK^;?0jY23T- zwyoh6$ln4kPyRkgsNG@Zl^WPR&EVCBlWX6G+f(;5h<1X~PapdkOEn)`>pMN-R-p|i zSLVdKyfT?|#=I8Am!Z9N$p{^+3rIo5TKG^7HhH9FN25Dl*UMKL5En)WU#0;r0!&tp zI7cZu4>3jOBVKZ6rxA51c5yhDldyM7;#Rg^Gu?(bm>oX6nk39xtqSiB8?N2iuH-Ea zvFcPCDy4XOlFpi}Zo>$RjoR*)3Y^=xh&;K5duV8!3@ON{H}p}R-ReqP6_~}Pjw;!! zDJlk)NeUFqDJ@Mc55>R}Tjs|0ZB?l=`LxY!b?&R=N%i5WUQv^a7>wTD>I!Sw7)U2V zO4Eor*Co`Q@X|&e8@e!7ici+v9bzQ=@cgM9qM~sKymJj8U{fWU)vY%iYcIMpGX@o% z?Zv_icj)oBH*^Tx3< z%T8r-vvwx0`MbXGJ94$c^!(;8kG5N-NF#5eYae0m`o;;cWhuz<4crh*gsazfV(LRp`5NkZ(IfPj6Rv<;s;-U%=cofEBBjpwrD_^6ryG>6Q}G>aCR5zjaBy zJ(MpXpB7+6$CB&w^{UlZ74k!C=juI^33vV1&eg*2+z#|;S63Oj!H2^>B8eL{O@ay?1mU@ZyGb+~skyDenRFDjsAWz7T_pk$*QwTclp296exhU7#GJM2i+oaCuLAix&3F&?@l}dvbCf(Q;yUF{ zX3p12{Z?vbtDXE3z!3$)D^A}>w4=tzUZR!&t&zZI8f*=vNG^2Eb`WampvMeQy2w6k zU53i~0{iUP8H&-NdO3<$IZ@kfoneE3qz7f+NqM$$$xw<_ozqOlmjzsT7XQtE4hR}meQ7%qZv@I_a;lt3{{@iaDs&<+D zL!7MV2`lcCoTR0I_oOJN})BNu$ZS}Xg7A2boem5h6d)Z z6YQ)+^QtdxQ3!X}&-3=;5I9iCh)0;#HWXJKs!fQXJJN09X$*15y}@6N)KewdhmiF$ zSXufT2EL-1e>?WxlVR9cf1{#=e0H$p5If{ZIySn8&7DjAg2LD${CcZI^5|jF3we^J zU3Iy71F4~XI)S=tAre)(ssr1toz64Q6iG?5HR5eWc6dh|!#&|zBLU-x?eTZMi57HC zJGC_jro7iT+q2|0qz?_XiW?h?wg)&-D9*vO88(<`4c)YM=x4+`yK10(qbHLpOVdd- zRoc#8%;gB-(lLt-6B-Kdxf$!NyN{iZJ?c1z!@k(Qpoq;bH8x(WXkUZOdbMw#csrK7 zLDY~-9Aa7_bba4g&utK>%PG>F2}yM6&A&>LV}Ms~%-i+l=?n4a_)l9U&)=(Di1-;N z11p>2zI!Y-N=(lAQN9-rnY&^S9# zHEu4hOmTkBQrN4)j4mpt{sNtA>{wd;ql9p0DaGZ$hYt;cz{ApvlZK2_h8pH^n=iF` z>s}!Adb)Hx(l@`)RKJZt=&XF~@DPX{c{1vzJea>`7`Q<*j+wQEym}L$;-zRokv}v= z*l6!qFQ+G3$+(_NuVo=hh#W31M#1+fmbylkdj%m7a7mZ$ZBTR2`Zi%9>`)cqUY_Ay za*;SE4wM-hfq01_J^*BP%5~auJ>bZF@pTQusvI?qKzCZ|x7F>T`qTv3{d0Q73r${~+l%k?6L#9+@ ztL%nQ)~tnOi3!PWvWzu5Std)7HQACalzl1N5N0ab8b%BTgTY{o8RK_(?&b45pZk8g ze}8;m-|s*A!|R%^bFS+=kMlT>^L-rWG4Ep`9<_Mgr@oX_=~Rk*9P6N~fgW7lo*&hH zvql&?xU@9~j*sL#{nTBy&`@@tO)d)yZI*@Q!dVve+-KJJ-?V+))0GvXDz0=oT|!Ju zEMEIWUZ0G|f!wyXkJL@N1$ohH09LkE+?ZuYgjtrN%Q7p;O@qx2{-wSpqr+n!W6gsN z(<>7kO-o_PV`JS?NGu|*krl;DxI`SQ#NZX|Qg|#NMhU!eczi<{II<_gse+&!2SwY} zFDY(!`NF4N=u@4K#$Egg_b^73LIHnHywW!M*5c6HjioiYQnF(}@4;go%U@Jj&1ti` zfoBwT#-K>5GsGl*aCYOELGXy5_d`cay3-a(j^B!S=m{=~9NNmNHpNis?jOkARJ|Zd zQefD-|Ijf$r6+qB_BJwxHa1d;!|cC(ys^JxZF=aDG=4$*LXy&jY?~@|6kHVc0P|)4 z0Sc6agshj_$=4SfeFmi>Pa$i?$S+A|wjM_GJsEebn6fKcgaSh$W;}l0M?V*f)QzcC z;dsr%-1>}eK@{q+wW*paL!xvrx?nut{Gjl*|7ehtZ~Fb7XPt_ zqRiU5u(d^O4xB42SXXo&LpYUG_O6>8$>?uo@?}37FnuMfc4HVFP-Ra3ehP()E8Fzz zSU^!W(LioWT+wJ7tp;~}QR3NPo3y*gKm;hj8%UHMJt&8+xle92c@81yTL@CB6+1jX zRuYg89D@{;UxzBq+F67^wbu66LHqAlWuq?rG|LnaqB(DJ3XRwyV9&`xKp=4yV2p zZ;4|OW4NlRn3HZQ6?%(6p0p;*|tJ0oV>EUtuj(hV%OZX4n|I2_Y)Xs(8+J zzMWmJmoD-l$9c&=`L}uAe`w^!PZ_V;k@i2QKAH(UwACgEDZlglWEMhUeaI$Epi}cp z{K;l&V6AXa=b_hh8eY*7{J0DBlARMTA2&>=Ty}wXdj<}NOQ;3MxN(-*+@6;iIXL@7DC0)CIuC1Svo6wSBKtu z24+CVK+Z2y^=#7XJ(^EGQl8^xVyigp?r5B*H~-x&0}q>{ci2E}uqGa|9hIM5o8NlI zAj&HF@af$h9;mI}_WKX*Vb*>B&yKh~y=>%A@^@Sg9;%UWULk5QCo_^XN4oOQzo!_s&$i#yxpQ<~}#b|F7k~|B&$c zXLsHRJo)U|i%U-*-n^!!+*=m7MZmmBpvYkKY=yq7@xmwFeewG3-U31CJJDpJ3Y>ui;!-B7N#Xk`l|0IJ%1gz$i*fAM?Ll?Vq<6+$V zCX0gEJ<<{OMy$Zh3fN&boH^)t#X&frJ0L0qlRm9pld&WNyO@vL#}rY`m2I^*LEHuD zp<^yt3d<;Ky*Tb*^ekYU*|KJ11P`fpvVkb~8j|bBtk?E|Ha-FYG0W}E!K(fxHOLAe zP+8@AvuDpqOg~_g9vkDVf!BG}nKmVAiM;$74EMSvswShb_gX2<7 z%dyc}^wk0ktapwq>OZTMEVQV$?M^KoK#N;cL$%gvahxz&aw61`+OYLg@lW#sT-1aE zx^?EEv+~YP0(|zaVq7_hJFYj47{Hm;-yNl7k{j%VM1wT;=Fq;%1Zo3zHB=p%h7L87 zsW2r2M;{|bwbQIWaELylA6~Ayn}pmd*SuI1UvXHhs1Cav2UJiPYYSvM5c}jZgPzgC zlpDLrbm!r>hC+AMOr86pq(&g#`c3_bs`3dkbB&dB_*}v#=k5lO?bCb@=x*oqx!y zrrh6?mlC&-hqS>FOoCoM5&!h99=lfR_H{~gaM&i8qYi&MhOp_FYEyOWi;GU?UST6}h5VL|>7&2kjvlqC=Hafc5)`Ogl z25g;UprJQ$cwb8Dg00$8(lH^#bJkOnAqQo?VU~vm{Xv}zq|MKs^WxnjJPcItu`}rL zUY7vUh9@<(DfaRll95W{v%^f(#GPD%A1%B>^eFQkC5Tu6I_^RYqWn~iYalP6Iph~f zOAga&SdQk6c2SG)j)=k9sI0OTy8kG0n_#(JcY48i*_L_3KQf@2s^gtCc5W7Gbiht9 zyg?D~u(ywUmg*6xHi9N68KTd@z}pp1Wk5gF^)dut7lPJIveV0T12_9H#CX3feAZYA z#r4yMk`fA*#i|o40CNe9c_$6UT}viHC99&)=%fFGJN6$kQ&Sr9<+)b_8yK3ZTJM}}F^`Ua%j)A~{a!AMxMfx%VbE`7;0jf)`C2jvjrrZ_ev>BjV$_X^u zhcf>K>(ItpH`J)y-Z5G_K`@1rU$f&V!tevg!iy#!oUxyEjJ{qix^i%}_}yoi)}9oB zakJnGw^29D)?nzPtj+HQBGq;}A79)FV><4x)3yJkOvSe=f$z{;Q>-)@wp_gQpdDz) zNs2}etdNzpny5jO8;SnOMPnTvD;_RMCx5_u+$O6iGL1u)qRE2NoaBr$;AejrP=ErN z5`zaU3sg~19ZloBXJzNyCIxdSem^oMIZi&baHYx89zI{8-zRLM@c!kR03z{u$KCSN zcybfD5p$^{^cl2hc-SBBiR*N72T}1!9|d7{Jq2!~e*S%F*!NCc3Px0$qXot zxiU^cSY%lqleZMi7`^8y`Qe7ivE7vtZk|nZ`-gY-4%mlooGfu)p%kKA>NC}-Y7cKw zIC5XZ%ni{Wo_w5&h?BWEykGCB=Guj!RJl?dt%WH#aHADuG1(|G09WK9iP!IsVeO2e zBvCPeSy`}{cj5@~>9uXkm7Z;EuUFv0vNW_QE{iUrOsH8&Qd{;S@Ov_WJLJXa~#& zHGbTtQ4vDh^&p^=&xxv+FfzS4b}d#-x2CN zefPt#?8az# za#AUt)sQKlLPdUu;G`|Lxr(hd)D|5HXnMOPeI?l`4=asdu5mI7u>0T=T;ymNfI7a- zP2EHKzOYi)#jP^~D=sK-4&+LnqKyvQ$`aX`NIH#{=_JTAR%sf5+{Pg9mVu+A0N<_Y zk;d6IX+gp`>9d3UCs<2oD0KKz0O%EuMPRy}blE~H<;5VKU01$bVW8Ik$JqAHQdv;I zea8{Y0@=s1sC0jJg#V(yEn&JRjYjX(9O_wawMh51tR#$*$!{940jfAHhZN1uk>9<^h~*U)P+4Wk|b zC@2qR;L3PEF>sse)IV=>SF&oV=`(cnJE``meM=h@M*41J49~>XQL<{v4=V_@vL^|? z>z!+L)4^{^e#Utsn=G!$_YiE0-Q_*A(CZuB7V8z!G&y_?imIgC2PFw?SJLXoww#$G z*jJ}FHjs(|5;MIXag9~5yB1Wkb%5|Ql>gX(+ai_h)PKb{-Oro9=sv4)3#!yNqZ|se z^|T~Y{h{leP+@?2{NJFN(*2VRR>VsgdI$dm>Ou8~{D70?=FS(RxK22@G6Cgg%7-tf zuayP*FGpL_tT7gH_Z*3c{nONvJaRIHKFt_F#f(8(6^lyj?rDwro#PGqAK~vj_B;Er z^#Q`mr+;F?)WZ*aN7X?7k1-RJBQD;p?Es8Blx zI$7UOaQ}B(e~j;4!jpGidCqUb{_NdHrl9=~DfpU9rK(3w0303kJyHGiBL=%(RmsnG zm}p#_<*#B4{vEacAvKZA);|>Gf8F`Rq1`{nbjX*mru{#D&c7p;-@IJlf8JFLa{u|^ zZ!eT?FbMeydEV;%+wcGJmG>OS)~mAi?)mfee|Pcb0^fYe(6wv7JK2B#z%Ld`LzsZW zd;WOxA4C3+2flj!G3J;+NfiEyT<&`vE5y7|Dj!Yuy_A1JYd+n zi=%b{Cjjhs;lVCE*zx{$;lVCE*m1Xa;lVCE*oo5Y!h>CSunQ0V6&Kx&9_+${ooMrJ z^k5er?3@+tMh|x30dU^5d$_P0J=i%3-Gv9c@L(4n{C2Xx8$H;C2fNXOf91*S!h>CS z@P8T}jNUrn{0A4n|J_`vk;;$zSmj+;Av)%st2e9tg5Hr|d{W^4y9BL2bIBYI$pb_o zil&`~-Er8N7ox}3u@W^A0{<&j>pzmo?jH#%VuAK;$Z|awiRJwnjZ$6`t^Oqxk@^J!~(RQv} z`Clfs9qQR1+IZz8^iLuG?S+Y%(ons3NRryG^X*;&36S~LF@G!4Kc5@Te8`u><=M)A z%+p_F2E4x(sx2UhmruoiEeBx{zIpS8>!1HYsM<`qp_#3|7^%j43xpzYmzv1iqyP=5Rapi6(;8;YP=`jQzgYv@bfui?;(`FJUu zai6u)b;jEtp-OE}-aJXUp+Dd!#-Ab+2isBV!L>W$}wWtB+Id>KuHU%ifwYFrMJ7ebR-E54^sZx%cpeS5JQRM~_Wx z_@04V>aW`T>j`4+42g{b+rc9Wj3@MzfS3IDKd->|E2?uRuKudczn*Y^a_@OxH9BLh z?>}|(m6FEC2QC`z_W#)MV<)5bdK`_}o&1X){?`AX1VBs)Py7D4VLWrnJP(Cqa`ED? zr3u_U2Z-t1$%21wD8L~$EVm-sIcoT8X##gO0WoP^()s6xH%0DOcuiV9t1tNFr0)zO zX}|XRi`(h>|JX3b?BfGsXEUFd?k^{OXAl>dQsyrgO5gg&hPw*7tDyg6ads7S*Mk1q z+V5J>|A$-ryX*F^|J*r)>OKP0y=GaH4*zY>54v&UogMM)M7{lgYw_o%@v02imYdu~ z{&^1*ehsv|(dO0nYdrz(Dp>;Cvj3Es&Yy4jZ=0@(D$p`Il9+$;zXP1#z8loESIY<} zZBs{V`Tx<^3-6@&uFsoudsV;v$1S%908n1%Asj~kt&@Lt?UWj@IZxEo-}y(W-aj97 z#MX3EqGpMa!%Pq0zHTnsp(i)q2Iv^y?$7{-Tr36(CH9ANIh!~&EO+!@D|L&Vdu!Y{ zcc0*S<#bBC3-h=h_W!x?PZH3KB}=1xG(yDj6N3BP#bF}_b)aDbIM zgSUFvHf;_A&~1*u*qftT0s$l!`xEHadKK6lsCu$iptw~y zHPYH!)pb7Y_C1%zHCG|CpV!*Mu4na%+^xv{0hDB<3d@ot+?3tNm9rVcImEA2*6sa! z(Q;<*S=OVU{nRZ20qCMjdFe|L1&y)Z_Fyq}Grj36@Oa6eioh5?o>Tw1!{Yn{(|T0S z(fR8Bl?VFO(|r*%6N2K_tghnY&XYR#26s4s=PI+d9D!;qc6YXcc`53Z#1x+`RD)wl znZ{c{f+)9qE#)QJFMo4JP0@7WeVkTg1S!9Tv(h0Mke zmn6rYZeEUc$qm`KH@0cizb0B&jjYR804T(_3jwC~vRs^fPON0AB(;r#Q zhv30X!x=XP%P?Hh!j2-91*{cW=2$-R8~{6MmLE^#_X5Mkdeq&-E7-2`bEL^dE6{4Vg0?tQ%;v$!}Z+ z1SDJns~+`CD>BZ%d<8_rd!T?Roqg2)^kg9_<$cKxJ$!!{hlR=4nj8z1JgH|=dRE%I z*TKS2U-y2pOLY^krO;miy#IcbG%jY&C6Id01L(PbMtf~P50Sloo?Q-nex)M0vAB*p zN^C;aQMLK7s`4xBSsSW#Cqd;egv@6({f(~uuvszr@=T5btd&dcN*YU$)oj+1qp~xv zYE>CA{O3QD+kB;(DZPnTqAYtN&SWO(Uu!015O1fsG*Nm$u_jlM9Lp^m(7e2jd-l*u zZY?M4$J_-YA}gtb4Yu~A#(aP$nOGDjJrTK@@vJ%pF(ihk9djG80J(OAsaG{_ZxPFY z3WLd~2jAC3qrF%f7TibxoAc$VRZpN4=^VhPOKW)bel+>6q|PFup$8ZwCaG-XehA&G z`COsasG;tRftrDffs`Kl$XZ%*;3C~OqbzTV#0t^AqFd&Hj@ITZ910 z`lr->gP@~u){&tSOWFiS|IRJ%;n{fK)oB$eQrO7MO(|0c9`J_^Ttu*_`%KF^$0l#n z?H|4iAB;CIjL2<-iTYWs0qa5V+;lkIPo)^az6>a}t;b?5-sm8=tI~(D0P32l)BC!} zu$&K!WpLgy#r1MTyKo2uf|o&S5OCCKWM&0WbErTw5tduHl%cRzs9RzIbfrKu3T|(* z{47E~*S`EZ7B^zv=L&Wr8dcZbxv`HaE)5qaC&E*o7wh+PA4omw@fqEB#F2ezxkTZx zH9vsxJ_W8&Ilr%a+5;6Qw`j7^L|p`Qz#BQYIoKTF)=xWRM)Vmh;suOl8Hs)+b2;XL zFTfSsS8a=f={JHvG`f$`=O_q{c5K>{$?V^?DzHJ}BrzsufNZxymQPR8>@AnKGxFjNa7wqO#TH;E-mV-y5i_z8 zsU~kd_ts9~MJk|LiIXakAI26vtBbH39a1`@S&~hD%w(g9JR+RdUAb}F|zubt-b$=_^}bP8;O{&Yfct-CcT)X!@r z4(F2J=z=JnYWbMLga`qo8uqm=?p)P%dgOsCZbzpvHl`4803CY_%^^J89i{5dQ6s2G z>9QZIEthLu&GcU%KFJghB**8e_~l<8JkBI#q-pRrbEa3S!=3$t6E4aBD)d}p_Q9mW z^myo0`QsJ|v~bwx(AYQZbeOBXu%Bd;A}Yg9fp9mu12JP~@V|HM zAg%H}3eeeMtqL!EhF2yvH|D1LYcH0tqpEG+4}Ix_Jq=%;ALEbUHc*(ls9UHbx9}cL zfJ=s>6+u*`iRYopbYmqRrXLq5oF+;sY`22w&)l!xBDMLN<7y_KQ~|5Y#P2%jw2dSv)`MU6#@g1C zHUEYB%rLXKc$cVUO9SSpM*#whzGKvUZWjDGnImWJf61wFuumL&^{ET^y@I?bC2zVk z*j({eD~&~8^K4RPx&A{d#ytxD7gz5?-!L3b0jgg-_yCl4&K*y%XK49y06ZZb`7pa> z^MYWytMX=f_SE4Lt3`n7x~D|2SC1OeL)$4Vaa9!474fjNH`Rx`3(Ai(om5Ggy!>He zOA-Ng5g68Dt4`|dEx?ni?*?YUIXCa470AWX`J8P0m$*dUyk3H8Fw3HjgzII0c@;eF z^7`4H{FaG^aqog_h9|JT^H5M4HfNl%YwALemR8+VcX9zN4EBuwtY%ntG6J$WlQc8s zf9AZ{ekQzs;TIBlu$k`Rfx%!U8*icSLk=Q2H5@Ww;#~nS9X$TxTARKaK+jclPW<+KKTdy-%HxR8)KfctB_%ysxA-_08>$vXV-Y~-CnG3`s@X4V`)1#fg-^8nVbzvtEPqf9i&?3kqP>UnlD` z5u&gL+}AA8dgxYegh6p>6OT0lyrqHlJbml^Lb<9Af7~5wk4cFij&?xlU>)(B&W%#H z$K}WCv-J;TM6N&)PRF{ZIeTF&kG02X@>5BM1};u7Y%)2&CU`$?i|sn{Q@0CO)8PD+ zABOtU{jx>@FPC>Q|Ff4COR9%^sVla^a`il34nN}aqc5|{Xm2w5jEum<5X(Ltc$bz% zg1hSDn-eg7as!Tss165X@%2#E;0won@$&hPV~cd;Ae51|oM8?KbJmq6#J8rZ;RuA^ zq(EWmCq{vZN&TU$4f>qhcD{eq6fPe31V`1f+%(Wm^#u!l@;xyp>h9|&Nw-*jYMe`y zH6soAPhQ=i`Qg9UdoRfh**W*W50PP4##m#mU3J z&9W>%*WG!eeU+bgkjwg<(}3IaBC~T_$Cldlm)+jEEx8dfoXMHkqWH+y-B$%ciun(| z+56uJPCgt7QVe)l`0N6h`X=d(w*9S}qPlF&6owPjPJa`brPHNBi7!Fl1tH5(%j>d; z22ftFzb>YSoj_PgA*v-$ZT58gCl;!bK-`N5HM0%o?>&@JQI=RFeDP9Pn{xK7Ow04| zbSh6Oi?Cu_QRNlnx-7iXOxyp!;>Am1i%iS0ezO$4;ibEUZ8@%1>bj=+YQbbiQSa+- zldO5C@eMiWxbQqGLWXZTrPpiiM6x14XM8em+I*JTsH$$foFT+A{s|#erWH0GRqJ{_ zs$1-HdSH4txJWAuWbPx5tVPS4Jz0j^f9-&wd`&G@)JH!a>rcXrOiRuxD~e=Fqs8mi zN?bCqh`@~ib)`zt)zA3X!J$F|hn6CES~v7IzGl1I)vzx*To;{7>3DVx97Gv!NpaFo z9Z0tAq>uBS_B%jtmCzc;GWa3PYL>E?*26&c2D7Gv&j-f;CeXcy!Mb7xAQ|?YSJjc7 z%{@`pON{DXst21z?PCjh9r7^;v^Dd6INaiPo6oG9mS6Jh@dE1%FrWJVKKPTG%ggr$ z)_~Fj5T}HNugpbP9cOmRSzOgsu+k7!wY~@*%5QIp6lYh1NG)COx^vRJg zr@P+y(P{8BXcY80F%3EmlvROgDkXIDg-e%+J%x_(TML#@OT8K$o|4=}*tALz$29Ol z+AF9FAllD^hUsfMUUh~iRX$V-SHFgWZ=v8X1e|*)D%>w|PuSedGNsy|$&hQz`MWw= zSryZVxu9>o6CBT5+~Q8vmzUv)vac+aY8XFqa!eO&PmxcT+9^dNB)cW1*d-fxvI(>p z{n@P>|HHAe_uIpy<5}lAv{k}q%zHZ-8`KhNbNEStnT=gH{6%TPRuF`O!;CK~@r`gKt#|(p}8d(ajO6#9pMk4Gk?$afR z4yZ8d?u~rF76rW|*1f&|b&J4hjnI#RAwTm<{y~4g9A=vrr;@_h6u*q9y#&c$GC*;O?6<$BqSemXlB$M^xNy6}?7M4| z8`ibK@L{Y98|78V(YMY%eQEj8t^s@O7fwt8ewC$Z97yqueiEuMB^ikbq~T^G(xMI6 z8Y%4o5D=16@!hSk=s*$4ji4#z%)`&3BSt+jmw6(%)lL*Ix*>dTb2-_9M<)1sRKaW_04;*GE7@b zYQv#o4dGw6G*UqZe`)7pD*B1 zYi6{~yuZF8EZaRv8px`ac_DpN`ryoi7I77p#JEdfJV(y6t2tJT_%k`GE3%^L9Nb>V z$y4%Rd!5Yv0x$C32@6OjY&c=u_}3ZxOQWn;kd(kJA>SbmzbEJuCjDZ_Kq z8uDDm6OC<14ekWXSFkfgHI;o0g&}WXDJVVM;}w2S9lvz+O+i(NaU(X86Sg`a*r9C) zf_NF4J*OSY$eSR>TtTI1aY7AcAsQF@?#6Eh`y_cwG3ifW9#3CfaOi`yc`3CrSPvH` zTFvJedm5TcIaykduDYHN&Vb4*_Cfz*zu z-mWgtNs(4&)uO{XojzT@T?sv(+11h?f1L}SlKwgf1bAxiECIDg7xCvy;qE`$OI}Zi zV{-^(phi8fD4w@aPo!?<^=hhU%G1KaRhdJn102MusNxh!OGJE%^@XS2yd_`Wj8K!_ zT$T9=*O_v>$m}}uO*st27x4Y4!s?e=kjfcFy>UG&lDfmygqmCDhzi|gj#zLF-?)62 zuc`|W$8uPUL=dyqNI8lnYg`=pv}J*IIEr?Aeoa}c!d}H0uh2a!qn({laL^5$N73D< z*hUFEPx=}e%EocYElABh>4i?x!|@1@q=vXCd$SXIZ!DMY*H!@wL)D>i$X1)Uxe)@Q z@f~pjb?dX|z z($pQ8b-?b~9PtX4bDS{Ph%vbF-kM;Z5$ zydU#6TQ{x^nY8((Br97-B_Q2rqqNIZ!^5UXsuKLY-ph)$0?P%Q`R^BpPcVNT9K_3L zoRMw(l9*_vuCbc%1(BT=)%=w`K{-9f;$sAlr%BeW-uK!Ujf^e%#q|WMJ_>p&ArJ-wNolV$6Kdpodjl7t9|*(h7!0L}OvCZV_uRVZKZj+vcrJz3`6zAAyps&uZp zZkLl^SGv%oN}L@+4+ZoIbZuSuzVN*(W>V~OK|DxtSspP=+ER@ocezdW;kHB&-`VE{ z;2m$+Ctj2#1~{Y+=w?V;MvdezboesAm_MhXVh}#C2V6cQgA8L3JpOXyu2$x(o%w@p zdj=y>78Rx7qNFEp(cKO@o9gQoS!gl`KT1+N?E$8KF^PQ=^3SB{mvH-xPt&em%qS zygUmo5Se1+ll$7XXRBc;WHfIt!p7dFGfe36>kBlcau;pU3y z!D}h8lAxL|xhGF>IEY&ohm0%Cl=sNfHafy&W3?X6UUitQYho6u;d83U@cnUn&z>V| z8miZCW8an%0$l0s?U3$o^>Tq3Wn~WL$)W_#{QI3gLg2}L}YnFefs;L;_NjAw~MQhHV^7AI7WwpGuXmo!rC`6^53)) zh6K6{eDbZ7h@+BuTEcI7H}iW{5|V2K8+~W5emdkO?nel42!EXK*u1qqJKzPrHYZ+t zwe=8vyymrZnP6T z>ArXJ9_N`_i%pg+uI0H>@jF$wi1rCSEXlVTKPi;pni~A_c}BCw=oZ4z*K2=&EnwzOyb@7gS=rHO#q~bO(ARBkQI5 zJ)Fp4*!O_nzrgN*koEm$_h9qwoucV8$6SxPL)zd{mkYkW$8ugug}<(x0$1m+eILWg z6)ef%eEGfFAFqh{AI>2ENHP$4+v|RdL<=+?%C9}(jj*T#n{uXQEPe&+Y0xFil(sC= z{3Vf}{J!g}^?uOK?)I5gOMJmYyZ^N}bNS<}dCwAOVz_bdl@0D!EK(AMOBMk^6824y zKGN1qrQe{7dDXP_*qinX|l@;uO`Jje-Qa1ma7;D-_DIukdLTHtSWw(UgrzcxB zeEIAI33qcQQ8;yiqff@+Xqh!%A4O6NRQHADuemiJZ?X)RpWYA@?{4?O4Cqqo?6X_H zu&+wz_a9NLVafR7;w6CBtTeb>HPG>1r8^n#iY`=>tk9!q>*tWK6NSQ4aKM z_i5gvW(}^~FnzU^mmc3LqR`!5S|N#$Ji_M0LHhwxO0>fnO)rW$RV~$FL&(7o_-?+9N zf-4>-vdi(?wo<`vk)v($tFF<1lL4(B~G2f zIFUHJ#A!pK)<#ZR0E)nP=%k7IkgYisoYW6ar@oLI_S#f*oW^-npN$WSPbse{dPf=Ul+X9DK7n!-j zzA{F}!z^6}bm>vYAK<*lVPB(Sha4{+?Zr=e3y|YkH4wLwxeeygxoYcK9&BmnBUBsG zeB!WWUpw=f%6b%?KsA;wlf^a0B23olsdIIb)E4|JRKB1*Pw!0nsWnbAIAL+&+^-#<~yxe z6f1@cWJv~M$Q;FXqKYiHOKIxqkH2ZasL0+V8P|*Omi#0c^cWsaPmN}7VrlX=75nl? z(C73t!S>DGqfY_S5V1S=&F;H4M?Acf;0dNih|kTQK}nprba1h#u03@=01PQqf5V3t z`r5rdj0h8siPvdwT@pUg5r8aov65z79M*D=lUNNJ@oKiNRi88n()l5kkM!*Ub+^f$ znu`Z9T)KMZ)+AM_T?-8lxVj-87wQv`IZ5rj!E`yh#*Q&Coq6QmSrN@S{QWJDCK1%N zxWG$vb%;R_On2U392u;vB_uMb6*fhYFGh^}?3@YI!ZQ8&yR@ddw+uDI4rif}wWyW% z-zGTYK+bF}YiW(i1BO|rGW!NNNagY#>?J;RRhlI(Y1?}GgY(VX^x4ZJ6i1r6TGWDP z3Bei;`s6o;|=WetY7BxG7f`XZS(ejoa4x zVB53mI;`(<5D?ke?;X?=2{Mm0v!Bgmt4%N*0=1WZ&t}5rLU8X8t zKGk7&q20tpDvT_+b}9zF-7GUmms(WNc1)`9qpWN?a{r z9qb9%r&wat(8V%R%4AI1JqNcY!8oqC%7$l4X!8}aX5VaBrvx`Uci9OxJh$%&pW*~s zH|gMPZ1na^PvP<`aH9!`M>~>o+|VZX(}ldPr{sz9UU_ksF{B??x^bJza<5wSq&z=e zSNKIdTdm7ACcTNC`)iZY5!r)-y?YuP=N z|J-r5_u|}Fd9ph(-0q$Hvo-v-C^B!xYtHc0NijFuaQ!ll_2;tD4JR|czh*gYd?fJ9 zYuARh1#R&HNQ}YQV7`B3+zqgYfF#O`9Z7y)=lrATo$kC;>kU}Fs=F#TGxyjC86JJD zw%p*L@B}TL^~9WWQ}u;J_cM9Kbeon~ykj&q+feEq?-?UigcX)}Z|DG0AWP_G=uhgW zCI|l0x1z?1@~3j>xpA@hTcm1#m#)-AE03dto}cbc;~q3DwJ;kNg>80W>>9aU!RO%* zdLAEb@yRo=6jh|H&QGBquj?7~Wl8Q~m3WQ{;9$*4a1k;c)|NZCVzZGM z@lfw0yq$6j7iEtTwCY;c14)FwFs40(jW~YjH=3Pjz&?HEk5`Lwx%aJ0zAfuxt@PyjP|5+WwnFxn+-vL}(j2H=L}}-eR5R#G6xXmHEJ- zlAH86it&TFT^i-=Coa;>2`LN2Ry;K5=%uzxq#?mBp^3-f_*Bh*#YJxJ>9fwhg}M8N zEG??;$5ttNL`sYI3D`7KG}}%TKPQ~Usz$GAv`dl{?@#eE^-Cnn6bN*2buV}NIyxAr z#QI@3N%oi$^w{(@adW!nnv%U`!NR_=(^YwE<%+jF%=?4-MiJ- zBm7GTM;(hU7Kq#Xif1e&P=)-F;%H$+%=zm)U(aJL#&o_Xn$M}7NfJwfCELtldM!!* zjvqA+e^)-m7U7>5(YFr!#;EEq1My39>wwCzm9lJ~QSgIq`a!QueRl=)=`L9y zq3OMQWUjLUlO-iEIRu^#O(Q&X1VAFl}k61NYOyseRru^3$QSfz~f z%=u0?qD3vxY&meR%Dw55S>!#&c87j?Ets_fHb4-C0A}_iJD0 zlETlvkK}r*jj?UYDD^OJDn|c}fr9L96POdv@Kfm9yPmjrv5&3MF$7*?XJdH!(y4*> zv75e$3F2!M&9eHoV9joF#;HcXsN|~I@&PY7(g$wjULN5UWsy)!_pQ1uGl)wX)c#v} zYMC7idUhNIQV`LKvRE%qI<}CY@^V$iZBkU!JZ{#0`4%?ocEOUxENqG@F-v*BpkrEU zFUv%fpUptu`!Q%w6m z>SjBGy;R4=eKxAhIl5V%!%Qn|4^8y1?l+qWb-}P=Y$1zdN-ILnuSaZOHj%!t1&_n* zzbX?y@{da2w_jFb)aoSaipQWa6X#-tpB?xof#U^* zE+N6dL32Zd(-gA)=FEI2vq5a{q_R3>p@D>IEnZ@)cu|Rl$W2}0)Gyby%hu{dK9=Ip zc;X!}3t5~;T3R15BA>^XOaULWZ63|Q~_3}0uFILqIuAOUNg3yaIIR_q7j+ObvMhE z=2n<|iODjtDVZMPlnoirH9F!{g+9heU95LWyPKi*>s+9q8yt}%oGNlxg6E&*bR@{- ze9;O79E2hrnW+;Ajd$iy^^U_8b1Bfc63-F0h}vAy(^j_A(_N?j>c;%#JTQ!%&r&P9 zQUs2 int: + return a + b + + @workflow + def wf(a: int, b: int) -> int: + out = t(a=a, b=b) + return out + + res = wf(a=7, b=8) + assert res == 15 + + +def test_jupyter_run_task_first_fail(jupyter_patches, mock_remote_execution): + (mock_process, mock_write_example_notebook, mock_exit_handler) = jupyter_patches + + @task + @jupyter(run_task_first=True) + def t(a: int, b: int): + dummy = a // b # noqa: F841 + return + + @workflow + def wf(a: int, b: int): + t(a=a, b=b) + + wf(a=10, b=0) + mock_process.assert_called_once() + mock_write_example_notebook.assert_called_once() + mock_exit_handler.assert_called_once() + + +def test_jupyter_extra_config(mock_remote_execution): + @jupyter( + max_idle_seconds=100, + port=8888, + notebook_dir="/root", + enable=True, + pre_execute=None, + post_execute=None, + ) + def t(): + return + + assert t.get_extra_config()["link_type"] == "jupyter" + assert t.get_extra_config()["port"] == "8888" + + +def test_serialize_vscode(mock_remote_execution): + @task + @jupyter( + max_idle_seconds=100, + port=8889, + notebook_dir="/root", + enable=True, + pre_execute=None, + post_execute=None, + ) + def t(): + return + + default_image = Image(name="default", fqn="docker.io/xyz", tag="some-git-hash") + default_image_config = ImageConfig(default_image=default_image) + default_serialization_settings = SerializationSettings( + project="p", domain="d", version="v", image_config=default_image_config + ) + + serialized_task = get_serializable_task(OrderedDict(), default_serialization_settings, t) + assert serialized_task.template.config == {"link_type": "jupyter", "port": "8889"} diff --git a/plugins/flytekit-flyteinteractive/tests/test_flyin_plugin.py b/plugins/flytekit-flyteinteractive/tests/test_flyteinteractive_vscode.py similarity index 96% rename from plugins/flytekit-flyteinteractive/tests/test_flyin_plugin.py rename to plugins/flytekit-flyteinteractive/tests/test_flyteinteractive_vscode.py index 45e6e5e42e..0031d10868 100644 --- a/plugins/flytekit-flyteinteractive/tests/test_flyin_plugin.py +++ b/plugins/flytekit-flyteinteractive/tests/test_flyteinteractive_vscode.py @@ -13,10 +13,11 @@ VIM_CONFIG, VIM_EXTENSION, VscodeConfig, - jupyter, vscode, ) -from flytekitplugins.flyteinteractive.vscode_lib.constants import EXIT_CODE_SUCCESS +from flytekitplugins.flyteinteractive.constants import ( + EXIT_CODE_SUCCESS, +) from flytekitplugins.flyteinteractive.vscode_lib.decorator import ( get_code_server_info, get_installed_extensions, @@ -187,7 +188,7 @@ def test_vscode_run_task_first_fail(vscode_patches, mock_remote_execution): ) = vscode_patches @task - @vscode + @vscode(run_task_first=True) def t(a: int, b: int): dummy = a // b # noqa: F841 return @@ -206,23 +207,6 @@ def wf(a: int, b: int): mock_prepare_launch_json.assert_called_once() -@mock.patch("flytekitplugins.flyteinteractive.jupyter_lib.decorator.subprocess.Popen") -@mock.patch("flytekitplugins.flyteinteractive.jupyter_lib.decorator.sys.exit") -def test_jupyter(mock_exit, mock_popen): - @task - @jupyter - def t(): - return - - @workflow - def wf(): - t() - - wf() - mock_popen.assert_called_once() - mock_exit.assert_called_once() - - def test_is_extension_installed(): installed_extensions = [ "ms-python.python", From 600a1a90aacffc0d6b89e2b438de315accebe2e3 Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Thu, 21 Mar 2024 11:28:56 -0500 Subject: [PATCH 07/50] Setting interruptible on ArrayNode sub node metadata (#2288) * setting interruptible on ArrayNode sub node metadata Signed-off-by: Daniel Rammer * lint Signed-off-by: Kevin Su * stuff (#2291) Signed-off-by: Yee Hing Tong --------- Signed-off-by: Daniel Rammer Signed-off-by: Kevin Su Signed-off-by: Yee Hing Tong Co-authored-by: Kevin Su Co-authored-by: Yee Hing Tong --- flytekit/core/array_node_map_task.py | 6 +-- .../unit/core/test_array_node_map_task.py | 42 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/flytekit/core/array_node_map_task.py b/flytekit/core/array_node_map_task.py index 7e7dbaff39..fc35dfa62f 100644 --- a/flytekit/core/array_node_map_task.py +++ b/flytekit/core/array_node_map_task.py @@ -113,9 +113,9 @@ def python_interface(self): def construct_node_metadata(self) -> NodeMetadata: # TODO: add support for other Flyte entities - return NodeMetadata( - name=self.name, - ) + nm = super().construct_node_metadata() + nm._name = self.name + return nm @property def min_success_ratio(self) -> Optional[float]: diff --git a/tests/flytekit/unit/core/test_array_node_map_task.py b/tests/flytekit/unit/core/test_array_node_map_task.py index 0816f0c24f..8b078cdf2a 100644 --- a/tests/flytekit/unit/core/test_array_node_map_task.py +++ b/tests/flytekit/unit/core/test_array_node_map_task.py @@ -303,3 +303,45 @@ def wf(x: typing.List[int]): map_task(my_mappable_task)(a=x).with_overrides(container_image="random:image") assert wf.nodes[0]._container_image == "random:image" + + +def test_serialization_metadata(serialization_settings): + @task(interruptible=True) + def t1(a: int) -> int: + return a + 1 + + arraynode_maptask = map_task(t1, metadata=TaskMetadata(retries=2)) + # since we manually override task metadata, the underlying task metadata will not be copied. + assert not arraynode_maptask.metadata.interruptible + + @workflow + def wf(x: typing.List[int]): + return arraynode_maptask(a=x) + + od = OrderedDict() + wf_spec = get_serializable(od, serialization_settings, wf) + + assert not arraynode_maptask.construct_node_metadata().interruptible + assert not wf_spec.template.nodes[0].metadata.interruptible + + +def test_serialization_metadata2(serialization_settings): + @task + def t1(a: int) -> int: + return a + 1 + + arraynode_maptask = map_task(t1, metadata=TaskMetadata(retries=2, interruptible=True)) + assert arraynode_maptask.metadata.interruptible + + @workflow + def wf(x: typing.List[int]): + return arraynode_maptask(a=x) + + od = OrderedDict() + wf_spec = get_serializable(od, serialization_settings, wf) + + assert arraynode_maptask.construct_node_metadata().interruptible + assert wf_spec.template.nodes[0].metadata.interruptible + task_spec = od[arraynode_maptask] + assert task_spec.template.metadata.retries.retries == 2 + assert task_spec.template.metadata.interruptible From 9e270241a12e78bfbfda5c2ae5413a13f000e850 Mon Sep 17 00:00:00 2001 From: Cornelis Boon Date: Fri, 22 Mar 2024 19:59:48 +0100 Subject: [PATCH 08/50] Add extra-index-url to ImageSpec (#2269) * start work on extra-index-url Signed-off-by: Cornelis Boon * refactor image_builder to keep old test working Signed-off-by: Cornelis Boon * implement solution for multiple extra-index-url Signed-off-by: Cornelis Boon * run make lint Signed-off-by: Cornelis Boon * remove utility function Signed-off-by: Cornelis Boon * update image tag in failing test due to additional data in image spec Signed-off-by: Cornelis Boon * Update plugins/flytekit-envd/flytekitplugins/envd/image_builder.py and test Signed-off-by: Cornelis Boon --------- Signed-off-by: Cornelis Boon --- flytekit/image_spec/image_spec.py | 2 ++ .../flytekitplugins/envd/image_builder.py | 7 ++++- .../flytekit-envd/tests/test_image_spec.py | 31 +++++++++++++++++++ tests/flytekit/unit/cli/pyflyte/test_run.py | 4 +-- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/flytekit/image_spec/image_spec.py b/flytekit/image_spec/image_spec.py index 326e8a0e80..e740e72449 100644 --- a/flytekit/image_spec/image_spec.py +++ b/flytekit/image_spec/image_spec.py @@ -43,6 +43,7 @@ class ImageSpec: base_image: base image of the image. platform: Specify the target platforms for the build output (for example, windows/amd64 or linux/amd64,darwin/arm64 pip_index: Specify the custom pip index url + pip_extra_index_url: Specify one or more pip index urls as a list registry_config: Specify the path to a JSON registry config file commands: Command to run during the building process """ @@ -63,6 +64,7 @@ class ImageSpec: base_image: Optional[str] = None platform: str = "linux/amd64" pip_index: Optional[str] = None + pip_extra_index_url: Optional[List[str]] = None registry_config: Optional[str] = None commands: Optional[List[str]] = None diff --git a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py index 6c11bd6838..3d424d584e 100644 --- a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py +++ b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py @@ -82,8 +82,13 @@ def build(): install.python_packages(name=[{python_packages}]) install.apt_packages(name=[{apt_packages}]) runtime.environ(env={env}, extra_path=['/root']) - config.pip_index(url="{pip_index}") """ + if image_spec.pip_extra_index_url is None: + envd_config += f' config.pip_index(url="{pip_index}")\n' + else: + pip_extra_index_url = " ".join(image_spec.pip_extra_index_url) + envd_config += f' config.pip_index(url="{pip_index}", extra_url="{pip_extra_index_url}")\n' + ctx = context_manager.FlyteContextManager.current_context() cfg_path = ctx.file_access.get_random_local_path("build.envd") pathlib.Path(cfg_path).parent.mkdir(parents=True, exist_ok=True) diff --git a/plugins/flytekit-envd/tests/test_image_spec.py b/plugins/flytekit-envd/tests/test_image_spec.py index d77b2ca89b..8e5d8b1631 100644 --- a/plugins/flytekit-envd/tests/test_image_spec.py +++ b/plugins/flytekit-envd/tests/test_image_spec.py @@ -84,3 +84,34 @@ def build(): ) assert contents == expected_contents + + +def test_image_spec_extra_index_url(): + image_spec = ImageSpec( + packages=["-U --pre pandas", "torch", "torchvision"], + base_image="cr.flyte.org/flyteorg/flytekit:py3.9-latest", + pip_extra_index_url=[ + "https://download.pytorch.org/whl/cpu", + "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple", + ], + ) + EnvdImageSpecBuilder().build_image(image_spec) + config_path = create_envd_config(image_spec) + assert image_spec.platform == "linux/amd64" + image_name = image_spec.image_name() + contents = Path(config_path).read_text() + expected_contents = dedent( + f"""\ + # syntax=v1 + + def build(): + base(image="cr.flyte.org/flyteorg/flytekit:py3.9-latest", dev=False) + run(commands=[]) + install.python_packages(name=["-U --pre pandas", "torch", "torchvision"]) + install.apt_packages(name=[]) + runtime.environ(env={{'PYTHONPATH': '/root', '_F_IMG_ID': '{image_name}'}}, extra_path=['/root']) + config.pip_index(url="https://pypi.org/simple", extra_url="https://download.pytorch.org/whl/cpu https://pypi.anaconda.org/scientific-python-nightly-wheels/simple") + """ + ) + + assert contents == expected_contents diff --git a/tests/flytekit/unit/cli/pyflyte/test_run.py b/tests/flytekit/unit/cli/pyflyte/test_run.py index 9dac0c8e8b..b8c64cea5d 100644 --- a/tests/flytekit/unit/cli/pyflyte/test_run.py +++ b/tests/flytekit/unit/cli/pyflyte/test_run.py @@ -297,9 +297,9 @@ def test_list_default_arguments(wf_path): ) ic_result_4 = ImageConfig( - default_image=Image(name="default", fqn="flytekit", tag="DgQMqIi61py4I4P5iOeS0Q"), + default_image=Image(name="default", fqn="flytekit", tag="urw7fglw5pBrIQ9JTW1vQA"), images=[ - Image(name="default", fqn="flytekit", tag="DgQMqIi61py4I4P5iOeS0Q"), + Image(name="default", fqn="flytekit", tag="urw7fglw5pBrIQ9JTW1vQA"), Image(name="xyz", fqn="docker.io/xyz", tag="latest"), Image(name="abc", fqn="docker.io/abc", tag=None), ], From 80fc5ebfb2749dd6f0d465b7fdd094c8e31451a9 Mon Sep 17 00:00:00 2001 From: Austin Liu Date: Sat, 23 Mar 2024 03:01:18 +0800 Subject: [PATCH 09/50] pre-commit: upgrade shellcheck-py (#2285) Signed-off-by: Austin Liu --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1e91d21bd1..5446621f9b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.8.0.4 + rev: v0.10.0.1 hooks: - id: shellcheck - repo: https://github.com/conorfalvey/check_pdb_hook From 4c32934d7818848a7ae562527dc65d8bf64f1a06 Mon Sep 17 00:00:00 2001 From: novahow <58504997+novahow@users.noreply.github.com> Date: Sat, 23 Mar 2024 22:59:25 +0800 Subject: [PATCH 10/50] Core/cli recover (#2294) --- flytekit/clis/sdk_in_container/executions.py | 83 +++++++++++++++++++ flytekit/clis/sdk_in_container/pyflyte.py | 2 + .../unit/cli/pyflyte/test_executions.py | 33 ++++++++ 3 files changed, 118 insertions(+) create mode 100644 flytekit/clis/sdk_in_container/executions.py create mode 100644 tests/flytekit/unit/cli/pyflyte/test_executions.py diff --git a/flytekit/clis/sdk_in_container/executions.py b/flytekit/clis/sdk_in_container/executions.py new file mode 100644 index 0000000000..a436522b7e --- /dev/null +++ b/flytekit/clis/sdk_in_container/executions.py @@ -0,0 +1,83 @@ +import rich_click as click + +from flytekit.clis.sdk_in_container.constants import CTX_DOMAIN, CTX_PROJECT +from flytekit.clis.sdk_in_container.helpers import FLYTE_REMOTE_INSTANCE_KEY, get_and_save_remote_with_click_context +from flytekit.clis.sdk_in_container.utils import ( + domain_option_dec, + project_option_dec, +) +from flytekit.interfaces import cli_identifiers +from flytekit.remote import FlyteRemote + +EXECUTION_ID = "execution_id" + + +@click.command("relaunch", help="Relaunch a failed execution") +@click.pass_context +def relaunch(ctx: click.Context): + """ + Relaunches an execution. + """ + project = ctx.obj[CTX_PROJECT] + domain = ctx.obj[CTX_DOMAIN] + remote: FlyteRemote = ctx.obj[FLYTE_REMOTE_INSTANCE_KEY] + execution_identifier = cli_identifiers.WorkflowExecutionIdentifier( + project=project, domain=domain, name=ctx.obj[EXECUTION_ID] + ) + execution_identifier_resp = remote.client.relaunch_execution(id=execution_identifier) + execution_identifier = cli_identifiers.WorkflowExecutionIdentifier.promote_from_model(execution_identifier_resp) + click.secho("Launched execution: {}".format(execution_identifier), fg="blue") + click.echo("") + + +@click.command("recover", help="Recover a failed execution") +@click.pass_context +def recover(ctx: click.Context): + """ + Recovers an execution. + """ + project = ctx.obj[CTX_PROJECT] + domain = ctx.obj[CTX_DOMAIN] + remote: FlyteRemote = ctx.obj[FLYTE_REMOTE_INSTANCE_KEY] + execution_identifier = cli_identifiers.WorkflowExecutionIdentifier( + project=project, domain=domain, name=ctx.obj[EXECUTION_ID] + ) + execution_identifier_resp = remote.client.recover_execution(id=execution_identifier) + execution_identifier = cli_identifiers.WorkflowExecutionIdentifier.promote_from_model(execution_identifier_resp) + click.secho("Launched execution: {}".format(execution_identifier), fg="blue") + click.echo("") + + +execution_help = """ +The execution command allows you to interact with Flyte's execution system, +such as recovering/relaunching a failed execution. +""" + + +@click.group("execution", help=execution_help) +@project_option_dec +@domain_option_dec +@click.option( + EXECUTION_ID, + "--execution-id", + required=True, + type=str, + help="The execution id", +) +@click.pass_context +def execute( + ctx: click.Context, + project: str, + domain: str, + execution_id: str, +): + # save remote instance in ctx.obj + get_and_save_remote_with_click_context(ctx, project, domain) + if ctx.obj is None: + ctx.obj = {} + + ctx.obj.update(ctx.params) + + +execute.add_command(recover) +execute.add_command(relaunch) diff --git a/flytekit/clis/sdk_in_container/pyflyte.py b/flytekit/clis/sdk_in_container/pyflyte.py index 2a52c6adf7..a492e1cba8 100644 --- a/flytekit/clis/sdk_in_container/pyflyte.py +++ b/flytekit/clis/sdk_in_container/pyflyte.py @@ -7,6 +7,7 @@ from flytekit.clis.sdk_in_container.backfill import backfill from flytekit.clis.sdk_in_container.build import build from flytekit.clis.sdk_in_container.constants import CTX_CONFIG_FILE, CTX_PACKAGES, CTX_VERBOSE +from flytekit.clis.sdk_in_container.executions import execute from flytekit.clis.sdk_in_container.fetch import fetch from flytekit.clis.sdk_in_container.get import get from flytekit.clis.sdk_in_container.init import init @@ -94,6 +95,7 @@ def main(ctx, pkgs: typing.List[str], config: str, verbose: int): main.add_command(fetch) main.add_command(info) main.add_command(get) +main.add_command(execute) main.epilog get_plugin().configure_pyflyte_cli(main) diff --git a/tests/flytekit/unit/cli/pyflyte/test_executions.py b/tests/flytekit/unit/cli/pyflyte/test_executions.py new file mode 100644 index 0000000000..254eb2409b --- /dev/null +++ b/tests/flytekit/unit/cli/pyflyte/test_executions.py @@ -0,0 +1,33 @@ +import mock +from click.testing import CliRunner + +from flytekit.clis.sdk_in_container import pyflyte +from flytekit.interfaces import cli_identifiers + + +@mock.patch("flytekit.configuration.plugin.FlyteRemote") +def test_execution_recover(remote): + runner = CliRunner() + id = cli_identifiers.WorkflowExecutionIdentifier(project="p1", domain="d1", name="test-execution-id") + + result = runner.invoke( + pyflyte.main, + ["execution", "--execution-id", "test-execution-id", "--project", "p1", "--domain", "d1", "recover"], + ) + assert "Launched execution" in result.stdout + assert result.exit_code == 0 + remote.return_value.client.recover_execution.assert_called_once_with(id=id) + + +@mock.patch("flytekit.configuration.plugin.FlyteRemote") +def test_execution_relaunch(remote): + runner = CliRunner() + id = cli_identifiers.WorkflowExecutionIdentifier(project="p1", domain="d1", name="test-execution-id") + + result = runner.invoke( + pyflyte.main, + ["execution", "--execution-id", "test-execution-id", "--project", "p1", "--domain", "d1", "relaunch"], + ) + assert "Launched execution" in result.stdout + assert result.exit_code == 0 + remote.return_value.client.relaunch_execution.assert_called_once_with(id=id) From 224b1c9a2e1f628db3059ec194c2bc0afb835f0e Mon Sep 17 00:00:00 2001 From: Ketan Umare <16888709+kumare3@users.noreply.github.com> Date: Sun, 24 Mar 2024 05:19:46 -0700 Subject: [PATCH 11/50] Sagemaker inference agent (#2027) * [wip] Sagemaker serving agent Signed-off-by: Ketan Umare * added sync agent example Signed-off-by: Ketan Umare * initial version Signed-off-by: Samhita Alla * add deployment workflow Signed-off-by: Samhita Alla * add a workflow to delete sagemaker deployment Signed-off-by: Samhita Alla * clean up Signed-off-by: Samhita Alla * modify dict update logic, create sagemaker deployment tasks Signed-off-by: Samhita Alla * nit Signed-off-by: Samhita Alla * dockerfile and setup changes Signed-off-by: Samhita Alla * update Signed-off-by: Samhita Alla * update Signed-off-by: Samhita Alla * pin aioboto3 version Signed-off-by: Samhita Alla * remove boto3 directory Signed-off-by: Samhita Alla * update imports Signed-off-by: Samhita Alla * boto3 update code and add tests Signed-off-by: Samhita Alla * remove output type Signed-off-by: Samhita Alla * add await Signed-off-by: Samhita Alla * remove sync Signed-off-by: Samhita Alla * modify imports Signed-off-by: Samhita Alla * modify container logic Signed-off-by: Samhita Alla * modify output key Signed-off-by: Samhita Alla * add default container image Signed-off-by: Samhita Alla * remove struct Signed-off-by: Samhita Alla * add region Signed-off-by: Samhita Alla * add output to gettaskresponse Signed-off-by: Samhita Alla * convert to dict to str Signed-off-by: Samhita Alla * revert Signed-off-by: Samhita Alla * remove timeout and add creds to boto3 calls Signed-off-by: Samhita Alla * add to_flyte_idl Signed-off-by: Samhita Alla * subclass fix Signed-off-by: Samhita Alla * invoke endpoint async Signed-off-by: Samhita Alla * remove output type Signed-off-by: Samhita Alla * modify create sagemaker deployment code Signed-off-by: Samhita Alla * dict loop Signed-off-by: Samhita Alla * add wf output Signed-off-by: Samhita Alla * set lhs to an empty string for pythoninstancetask & modify param name in create deployment task Signed-off-by: Samhita Alla * update tracker and delete deployment workflow Signed-off-by: Samhita Alla * instance to instancetask Signed-off-by: Samhita Alla * add tests Signed-off-by: Samhita Alla * ruff isort Signed-off-by: Samhita Alla * ruff isort Signed-off-by: Samhita Alla * isort Signed-off-by: Samhita Alla * add test Signed-off-by: Samhita Alla * pin greatexpectations version Signed-off-by: Samhita Alla * update secret name Signed-off-by: Samhita Alla * add typing dict and update tracker Signed-off-by: Samhita Alla * modify tracker test Signed-off-by: Samhita Alla * add sync agent Signed-off-by: Samhita Alla * add name Signed-off-by: Samhita Alla * add syncagentexecutormixin Signed-off-by: Samhita Alla * modify sync output Signed-off-by: Samhita Alla * metadata to resource_meta Signed-off-by: Samhita Alla * remote conversion to flyte idl Signed-off-by: Samhita Alla * add output type Signed-off-by: Samhita Alla * floats to ints Signed-off-by: Samhita Alla * in place modification Signed-off-by: Samhita Alla * chain tasks Signed-off-by: Samhita Alla * great expectations revert Signed-off-by: Samhita Alla * optimize float to int code Signed-off-by: Samhita Alla * snake case Signed-off-by: Samhita Alla * modify plugin name Signed-off-by: Samhita Alla * modify plugin name Signed-off-by: Samhita Alla * modify tracker tests Signed-off-by: Samhita Alla * fix tests, revert tracker changes and remove pythoninstancetask Signed-off-by: Samhita Alla * tracker changes revert Signed-off-by: Samhita Alla * dict to Dict Signed-off-by: Samhita Alla * make images optional Signed-off-by: Samhita Alla * add image_name Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * debug Signed-off-by: Samhita Alla * dict to Dict Signed-off-by: Samhita Alla * state to phase Signed-off-by: Samhita Alla * add encode mode to secrets.get Signed-off-by: Samhita Alla * revert: add encode mode to secrets.get Signed-off-by: Samhita Alla * add decode Signed-off-by: Samhita Alla * revert decode Signed-off-by: Samhita Alla * ergonomic improvements; change plugin name Signed-off-by: Samhita Alla * change plugin name Signed-off-by: Samhita Alla * nit Signed-off-by: Samhita Alla * image check Signed-off-by: Samhita Alla * add api docs Signed-off-by: Samhita Alla * add api docs Signed-off-by: Samhita Alla * incorporate Kevin's suggestions Signed-off-by: Samhita Alla * handle scenario when the same input is present in the wf already Signed-off-by: Samhita Alla * add support for region to be a user-provided input at run-time Signed-off-by: Samhita Alla * modify workflow code to accommodate providing regions at runtime Signed-off-by: Samhita Alla * code optimization and add region support to workflows Signed-off-by: Samhita Alla * nit Signed-off-by: Samhita Alla * fixed an input_types bug Signed-off-by: Samhita Alla * fix images bug Signed-off-by: Samhita Alla * replace endpoint_name with config Signed-off-by: Samhita Alla * input_region default Signed-off-by: Samhita Alla * add inputs Signed-off-by: Samhita Alla * replace bug Signed-off-by: Samhita Alla * add return type Signed-off-by: Samhita Alla * kwtypes fix bug Signed-off-by: Samhita Alla * add cache_ignore_input_vars to task template in tests Signed-off-by: Samhita Alla * fix test Signed-off-by: Samhita Alla * fix test Signed-off-by: Samhita Alla * add test case for boto3 call method Signed-off-by: Samhita Alla * lint Signed-off-by: Samhita Alla * lint Signed-off-by: Samhita Alla * lint Signed-off-by: Samhita Alla * lint Signed-off-by: Samhita Alla * lint Signed-off-by: Samhita Alla * ruff version Signed-off-by: Samhita Alla * ruff version Signed-off-by: Samhita Alla * nit Signed-off-by: Samhita Alla * docstring update Signed-off-by: Samhita Alla --------- Signed-off-by: Ketan Umare Signed-off-by: Samhita Alla Co-authored-by: Samhita Alla --- .github/workflows/pythonbuild.yml | 10 +- .pre-commit-config.yaml | 4 +- Dockerfile.agent | 1 + docs/source/plugins/awssagemaker.rst | 17 - .../source/plugins/awssagemaker_inference.rst | 12 + docs/source/plugins/index.rst | 2 + flytekit/core/base_task.py | 36 +- flytekit/core/condition.py | 2 +- flytekit/core/task.py | 41 ++- plugins/README.md | 73 ++-- .../flytekitplugins/airflow/task.py | 2 +- plugins/flytekit-aws-sagemaker/README.md | 67 +++- .../dev-requirements.txt | 1 + .../flytekitplugins/awssagemaker/__init__.py | 85 ----- .../awssagemaker/distributed_training.py | 82 ----- .../flytekitplugins/awssagemaker/hpo.py | 168 --------- .../awssagemaker/models/__init__.py | 0 .../awssagemaker/models/hpo_job.py | 181 ---------- .../awssagemaker/models/parameter_ranges.py | 315 ----------------- .../awssagemaker/models/training_job.py | 326 ------------------ .../flytekitplugins/awssagemaker/training.py | 192 ----------- .../awssagemaker_inference/__init__.py | 36 ++ .../awssagemaker_inference/agent.py | 99 ++++++ .../awssagemaker_inference/boto3_agent.py | 68 ++++ .../awssagemaker_inference/boto3_mixin.py | 185 ++++++++++ .../awssagemaker_inference/boto3_task.py | 56 +++ .../awssagemaker_inference/task.py | 236 +++++++++++++ .../awssagemaker_inference/workflow.py | 182 ++++++++++ .../scripts/flytekit_sagemaker_runner.py | 92 ----- plugins/flytekit-aws-sagemaker/setup.py | 16 +- .../tests/test_boto3_agent.py | 100 ++++++ .../tests/test_boto3_mixin.py | 119 +++++++ .../tests/test_boto3_task.py | 52 +++ .../tests/test_flytekit_sagemaker_running.py | 37 -- .../flytekit-aws-sagemaker/tests/test_hpo.py | 124 ------- .../tests/test_hpo_job.py | 79 ----- .../tests/test_inference_agent.py | 122 +++++++ .../tests/test_inference_task.py | 189 ++++++++++ .../tests/test_inference_workflow.py | 93 +++++ .../tests/test_parameter_ranges.py | 93 ----- .../tests/test_training.py | 134 ------- .../tests/test_training_job.py | 87 ----- .../great_expectations/schema.py | 28 +- plugins/flytekit-papermill/tests/test_task.py | 28 +- pyproject.toml | 6 +- 45 files changed, 1786 insertions(+), 2092 deletions(-) delete mode 100644 docs/source/plugins/awssagemaker.rst create mode 100644 docs/source/plugins/awssagemaker_inference.rst create mode 100644 plugins/flytekit-aws-sagemaker/dev-requirements.txt delete mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/__init__.py delete mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/distributed_training.py delete mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/hpo.py delete mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/__init__.py delete mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/hpo_job.py delete mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/parameter_ranges.py delete mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/training_job.py delete mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/training.py create mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py create mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py create mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_agent.py create mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py create mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_task.py create mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/task.py create mode 100644 plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/workflow.py delete mode 100644 plugins/flytekit-aws-sagemaker/scripts/flytekit_sagemaker_runner.py create mode 100644 plugins/flytekit-aws-sagemaker/tests/test_boto3_agent.py create mode 100644 plugins/flytekit-aws-sagemaker/tests/test_boto3_mixin.py create mode 100644 plugins/flytekit-aws-sagemaker/tests/test_boto3_task.py delete mode 100644 plugins/flytekit-aws-sagemaker/tests/test_flytekit_sagemaker_running.py delete mode 100644 plugins/flytekit-aws-sagemaker/tests/test_hpo.py delete mode 100644 plugins/flytekit-aws-sagemaker/tests/test_hpo_job.py create mode 100644 plugins/flytekit-aws-sagemaker/tests/test_inference_agent.py create mode 100644 plugins/flytekit-aws-sagemaker/tests/test_inference_task.py create mode 100644 plugins/flytekit-aws-sagemaker/tests/test_inference_workflow.py delete mode 100644 plugins/flytekit-aws-sagemaker/tests/test_parameter_ranges.py delete mode 100644 plugins/flytekit-aws-sagemaker/tests/test_training.py delete mode 100644 plugins/flytekit-aws-sagemaker/tests/test_training_job.py diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index b9f1c2b516..2b444df511 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -2,14 +2,13 @@ name: Build # Schedule runs to run twice a day - on: push: branches: - master pull_request: schedule: - - cron: '0 13 * * *' # This schedule runs at 1pm UTC every day + - cron: "0 13 * * *" # This schedule runs at 1pm UTC every day env: FLYTE_SDK_LOGGING_LEVEL: 10 # debug @@ -116,9 +115,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ ubuntu-latest ] + os: [ubuntu-latest] python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}} - pandas: [ "pandas<2.0.0", "pandas>=2.0.0" ] + pandas: ["pandas<2.0.0", "pandas>=2.0.0"] steps: - uses: insightsengineering/disk-space-reclaimer@v1 - uses: actions/checkout@v4 @@ -257,8 +256,7 @@ jobs: - flytekit-async-fsspec - flytekit-aws-athena - flytekit-aws-batch - # TODO: uncomment this when the sagemaker agent is implemented: https://github.com/flyteorg/flyte/issues/4079 - # - flytekit-aws-sagemaker + - flytekit-aws-sagemaker - flytekit-bigquery - flytekit-dask - flytekit-data-fsspec diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5446621f9b..703bcda938 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,11 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.1.6 + rev: v0.2.2 hooks: # Run the linter. - id: ruff - args: [--fix, --show-fixes, --show-source] + args: [--fix, --show-fixes, --output-format=full] # Run the formatter. - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/Dockerfile.agent b/Dockerfile.agent index fbd8c4c4a8..886e4af613 100644 --- a/Dockerfile.agent +++ b/Dockerfile.agent @@ -13,6 +13,7 @@ RUN pip install --no-cache-dir -U flytekit==$VERSION \ flytekitplugins-bigquery==$VERSION \ flytekitplugins-chatgpt==$VERSION \ flytekitplugins-snowflake==$VERSION \ + flytekitplugins-awssagemaker==$VERSION \ && apt-get clean autoclean \ && apt-get autoremove --yes \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ \ diff --git a/docs/source/plugins/awssagemaker.rst b/docs/source/plugins/awssagemaker.rst deleted file mode 100644 index b8ded38bee..0000000000 --- a/docs/source/plugins/awssagemaker.rst +++ /dev/null @@ -1,17 +0,0 @@ -:orphan: - -.. TODO: Will need to add this document back to the plugins/index.rst file -.. when sagemaker agent work is done: https://github.com/flyteorg/flyte/issues/4079 - -.. _awssagemaker: - -################################################### -AWS Sagemaker API reference -################################################### - -.. tags:: Integration, MachineLearning, AWS - -.. automodule:: flytekitplugins.awssagemaker - :no-members: - :no-inherited-members: - :no-special-members: diff --git a/docs/source/plugins/awssagemaker_inference.rst b/docs/source/plugins/awssagemaker_inference.rst new file mode 100644 index 0000000000..dc16dc1dbb --- /dev/null +++ b/docs/source/plugins/awssagemaker_inference.rst @@ -0,0 +1,12 @@ +.. _awssagemaker_inference: + +##################################### +AWS Sagemaker Inference API reference +##################################### + +.. tags:: Integration, MachineLearning, AWS + +.. automodule:: flytekitplugins.awssagemaker_inference + :no-members: + :no-inherited-members: + :no-special-members: diff --git a/docs/source/plugins/index.rst b/docs/source/plugins/index.rst index 06e4d7cd58..c2f6599e03 100644 --- a/docs/source/plugins/index.rst +++ b/docs/source/plugins/index.rst @@ -30,6 +30,7 @@ Plugin API reference * :ref:`Vaex ` - Vaex API reference * :ref:`MLflow ` - MLflow API reference * :ref:`DuckDB ` - DuckDB API reference +* :ref:`SageMaker Inference ` - SageMaker Inference API reference .. toctree:: :maxdepth: 2 @@ -61,3 +62,4 @@ Plugin API reference Vaex MLflow DuckDB + SageMaker Inference diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index 7901618241..e286e1312b 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -24,7 +24,20 @@ from abc import abstractmethod from base64 import b64encode from dataclasses import dataclass -from typing import Any, Coroutine, Dict, Generic, List, Optional, OrderedDict, Tuple, Type, TypeVar, Union, cast +from typing import ( + Any, + Coroutine, + Dict, + Generic, + List, + Optional, + OrderedDict, + Tuple, + Type, + TypeVar, + Union, + cast, +) from flyteidl.core import artifact_id_pb2 as art_id from flyteidl.core import tasks_pb2 @@ -165,9 +178,6 @@ class IgnoreOutputs(Exception): """ This exception should be used to indicate that the outputs generated by this can be safely ignored. This is useful in case of distributed training or peer-to-peer parallel algorithms. - - For example look at Sagemaker training, e.g. - :py:class:`plugins.awssagemaker.flytekitplugins.awssagemaker.training.SagemakerBuiltinAlgorithmsTask`. """ pass @@ -479,7 +489,10 @@ def __init__( self._task_config = task_config if disable_deck is not None: - warnings.warn("disable_deck was deprecated in 1.10.0, please use enable_deck instead", FutureWarning) + warnings.warn( + "disable_deck was deprecated in 1.10.0, please use enable_deck instead", + FutureWarning, + ) # Confirm that disable_deck and enable_deck do not contradict each other if disable_deck is not None and enable_deck is not None: @@ -703,7 +716,13 @@ def dispatch_execute( # If executed inside of a workflow being executed locally, then run the coroutine to get the # actual results. return asyncio.run( - self._async_execute(native_inputs, native_outputs, ctx, exec_ctx, new_user_params) + self._async_execute( + native_inputs, + native_outputs, + ctx, + exec_ctx, + new_user_params, + ) ) return self._async_execute(native_inputs, native_outputs, ctx, exec_ctx, new_user_params) @@ -716,7 +735,10 @@ def dispatch_execute( # Short circuit the translation to literal map because what's returned may be a dj spec (or an # already-constructed LiteralMap if the dynamic task was a no-op), not python native values # dynamic_execute returns a literal map in local execute so this also gets triggered. - if isinstance(native_outputs, (_literal_models.LiteralMap, _dynamic_job.DynamicJobSpec)): + if isinstance( + native_outputs, + (_literal_models.LiteralMap, _dynamic_job.DynamicJobSpec), + ): return native_outputs literals_map, native_outputs_as_map = self._output_to_literal_map(native_outputs, exec_ctx) diff --git a/flytekit/core/condition.py b/flytekit/core/condition.py index bc7b4df865..50403574c1 100644 --- a/flytekit/core/condition.py +++ b/flytekit/core/condition.py @@ -428,7 +428,7 @@ def transform_to_comp_expr(expr: ComparisonExpression) -> Tuple[_core_cond.Compa def transform_to_boolexpr( - expr: Union[ComparisonExpression, ConjunctionExpression] + expr: Union[ComparisonExpression, ConjunctionExpression], ) -> Tuple[_core_cond.BooleanExpression, typing.List[Promise]]: if isinstance(expr, ConjunctionExpression): cexpr, promises = transform_to_conj_expr(expr) diff --git a/flytekit/core/task.py b/flytekit/core/task.py index f528d451f0..f1417feb13 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -31,8 +31,7 @@ class TaskPlugins(object): # Plugin_object_type is a derivative of ``PythonFunctionTask`` Examples of available task plugins include different query-based plugins such as - :py:class:`flytekitplugins.athena.task.AthenaTask` and :py:class:`flytekitplugins.hive.task.HiveTask`, ML tools like - :py:class:`plugins.awssagemaker.flytekitplugins.awssagemaker.training.SagemakerBuiltinAlgorithmsTask`, kubeflow + :py:class:`flytekitplugins.athena.task.AthenaTask` and :py:class:`flytekitplugins.hive.task.HiveTask`, kubeflow operators like :py:class:`plugins.kfpytorch.flytekitplugins.kfpytorch.task.PyTorchFunctionTask` and :py:class:`plugins.kftensorflow.flytekitplugins.kftensorflow.task.TensorflowFunctionTask`, and generic plugins like :py:class:`flytekitplugins.pod.task.PodFunctionTask` which doesn't integrate with third party tools or services. @@ -103,7 +102,13 @@ def task( secret_requests: Optional[List[Secret]] = ..., execution_mode: PythonFunctionTask.ExecutionBehavior = ..., node_dependency_hints: Optional[ - Iterable[Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase]] + Iterable[ + Union[ + PythonFunctionTask, + _annotated_launchplan.LaunchPlan, + _annotated_workflow.WorkflowBase, + ] + ] ] = ..., task_resolver: Optional[TaskResolverMixin] = ..., docs: Optional[Documentation] = ..., @@ -135,7 +140,13 @@ def task( secret_requests: Optional[List[Secret]] = ..., execution_mode: PythonFunctionTask.ExecutionBehavior = ..., node_dependency_hints: Optional[ - Iterable[Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase]] + Iterable[ + Union[ + PythonFunctionTask, + _annotated_launchplan.LaunchPlan, + _annotated_workflow.WorkflowBase, + ] + ] ] = ..., task_resolver: Optional[TaskResolverMixin] = ..., docs: Optional[Documentation] = ..., @@ -166,7 +177,13 @@ def task( secret_requests: Optional[List[Secret]] = None, execution_mode: PythonFunctionTask.ExecutionBehavior = PythonFunctionTask.ExecutionBehavior.DEFAULT, node_dependency_hints: Optional[ - Iterable[Union[PythonFunctionTask, _annotated_launchplan.LaunchPlan, _annotated_workflow.WorkflowBase]] + Iterable[ + Union[ + PythonFunctionTask, + _annotated_launchplan.LaunchPlan, + _annotated_workflow.WorkflowBase, + ] + ] ] = None, task_resolver: Optional[TaskResolverMixin] = None, docs: Optional[Documentation] = None, @@ -175,7 +192,11 @@ def task( pod_template: Optional["PodTemplate"] = None, pod_template_name: Optional[str] = None, accelerator: Optional[BaseAccelerator] = None, -) -> Union[Callable[[Callable[..., FuncOut]], PythonFunctionTask[T]], PythonFunctionTask[T], Callable[..., FuncOut]]: +) -> Union[ + Callable[[Callable[..., FuncOut]], PythonFunctionTask[T]], + PythonFunctionTask[T], + Callable[..., FuncOut], +]: """ This is the core decorator to use for any task type in flytekit. @@ -342,7 +363,13 @@ class ReferenceTask(ReferenceEntity, PythonFunctionTask): # type: ignore """ def __init__( - self, project: str, domain: str, name: str, version: str, inputs: Dict[str, type], outputs: Dict[str, Type] + self, + project: str, + domain: str, + name: str, + version: str, + inputs: Dict[str, type], + outputs: Dict[str, Type], ): super().__init__(TaskReference(project, domain, name, version), inputs, outputs) diff --git a/plugins/README.md b/plugins/README.md index d738c5b5a4..81d3ad9530 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -4,31 +4,33 @@ All the Flytekit plugins maintained by the core team are added here. It is not n ## Currently Available Plugins 🔌 -| Plugin | Installation | Description | Version | Type | -|------------------------------|-----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------| -| AWS Sagemaker Training | ```bash pip install flytekitplugins-awssagemaker ``` | Installs SDK to author Sagemaker built-in and custom training jobs in python | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-awssagemaker.svg)](https://pypi.python.org/pypi/flytekitplugins-awssagemaker/) | Backend | -| dask | ```bash pip install flytekitplugins-dask ``` | Installs SDK to author dask jobs that can be executed natively on Kubernetes using the Flyte backend plugin | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-awssagemaker.svg)](https://pypi.python.org/pypi/flytekitplugins-dask/) | Backend | -| Hive Queries | ```bash pip install flytekitplugins-hive ``` | Installs SDK to author Hive Queries that can be executed on a configured hive backend using Flyte backend plugin | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-hive.svg)](https://pypi.python.org/pypi/flytekitplugins-hive/) | Backend | -| K8s distributed PyTorch Jobs | ```bash pip install flytekitplugins-kfpytorch ``` | Installs SDK to author Distributed pyTorch Jobs in python using Kubeflow PyTorch Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-kfpytorch.svg)](https://pypi.python.org/pypi/flytekitplugins-kfpytorch/) | Backend | -| K8s native tensorflow Jobs | ```bash pip install flytekitplugins-kftensorflow ``` | Installs SDK to author Distributed tensorflow Jobs in python using Kubeflow Tensorflow Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-kftensorflow.svg)](https://pypi.python.org/pypi/flytekitplugins-kftensorflow/) | Backend | -| K8s native MPI Jobs | ```bash pip install flytekitplugins-kfmpi ``` | Installs SDK to author Distributed MPI Jobs in python using Kubeflow MPI Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-kfmpi.svg)](https://pypi.python.org/pypi/flytekitplugins-kfmpi/) | Backend | -| Papermill based Tasks | ```bash pip install flytekitplugins-papermill ``` | Execute entire notebooks as Flyte Tasks and pass inputs and outputs between them and python tasks | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-papermill.svg)](https://pypi.python.org/pypi/flytekitplugins-papermill/) | Flytekit-only | -| Pod Tasks | ```bash pip install flytekitplugins-pod ``` | Installs SDK to author Pods in python. These pods can have multiple containers, use volumes and have non exiting side-cars | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-pod.svg)](https://pypi.python.org/pypi/flytekitplugins-pod/) | Flytekit-only | -| spark | ```bash pip install flytekitplugins-spark ``` | Installs SDK to author Spark jobs that can be executed natively on Kubernetes with a supported backend Flyte plugin | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-spark/) | Backend | -| AWS Athena Queries | ```bash pip install flytekitplugins-athena ``` | Installs SDK to author queries executed on AWS Athena | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-athena.svg)](https://pypi.python.org/pypi/flytekitplugins-athena/) | Backend | -| DOLT | ```bash pip install flytekitplugins-dolt ``` | Read & write dolt data sets and use dolt tables as native types | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-dolt.svg)](https://pypi.python.org/pypi/flytekitplugins-dolt/) | Flytekit-only | -| Pandera | ```bash pip install flytekitplugins-pandera ``` | Use Pandera schemas as native Flyte types, which enable data quality checks. | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-pandera.svg)](https://pypi.python.org/pypi/flytekitplugins-pandera/) | Flytekit-only | -| SQLAlchemy | ```bash pip install flytekitplugins-sqlalchemy ``` | Write queries for any database that supports SQLAlchemy | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-sqlalchemy.svg)](https://pypi.python.org/pypi/flytekitplugins-sqlalchemy/) | Flytekit-only | -| Great Expectations | ```bash pip install flytekitplugins-great-expectations``` | Enforce data quality for various data types within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-great-expectations.svg)](https://pypi.python.org/pypi/flytekitplugins-great-expectations/) | Flytekit-only | -| Snowflake | ```bash pip install flytekitplugins-snowflake``` | Use Snowflake as a 'data warehouse-as-a-service' within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-snowflake.svg)](https://pypi.python.org/pypi/flytekitplugins-snowflake/) | Backend | -| dbt | ```bash pip install flytekitplugins-dbt``` | Run dbt within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-dbt.svg)](https://pypi.python.org/pypi/flytekitplugins-dbt/) | Flytekit-only | -| Huggingface | ```bash pip install flytekitplugins-huggingface``` | Read & write Hugginface Datasets as Flyte StructuredDatasets | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-huggingface.svg)](https://pypi.python.org/pypi/flytekitplugins-huggingface/) | Flytekit-only | -| DuckDB | ```bash pip install flytekitplugins-duckdb``` | Run analytical workloads with ease using DuckDB | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-duckdb.svg)](https://pypi.python.org/pypi/flytekitplugins-duckdb/) | Flytekit-only | +| Plugin | Installation | Description | Version | Type | +| ---------------------------- | ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------- | +| AWS SageMaker | `bash pip install flytekitplugins-awssagemaker` | Deploy SageMaker models and manage inference endpoints with ease. | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-awssagemaker.svg)](https://pypi.python.org/pypi/flytekitplugins-awssagemaker/) | Python | +| dask | `bash pip install flytekitplugins-dask ` | Installs SDK to author dask jobs that can be executed natively on Kubernetes using the Flyte backend plugin | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-dask.svg)](https://pypi.python.org/pypi/flytekitplugins-dask/) | Backend | +| Hive Queries | `bash pip install flytekitplugins-hive ` | Installs SDK to author Hive Queries that can be executed on a configured hive backend using Flyte backend plugin | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-hive.svg)](https://pypi.python.org/pypi/flytekitplugins-hive/) | Backend | +| K8s distributed PyTorch Jobs | `bash pip install flytekitplugins-kfpytorch ` | Installs SDK to author Distributed pyTorch Jobs in python using Kubeflow PyTorch Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-kfpytorch.svg)](https://pypi.python.org/pypi/flytekitplugins-kfpytorch/) | Backend | +| K8s native tensorflow Jobs | `bash pip install flytekitplugins-kftensorflow ` | Installs SDK to author Distributed tensorflow Jobs in python using Kubeflow Tensorflow Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-kftensorflow.svg)](https://pypi.python.org/pypi/flytekitplugins-kftensorflow/) | Backend | +| K8s native MPI Jobs | `bash pip install flytekitplugins-kfmpi ` | Installs SDK to author Distributed MPI Jobs in python using Kubeflow MPI Operator | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-kfmpi.svg)](https://pypi.python.org/pypi/flytekitplugins-kfmpi/) | Backend | +| Papermill based Tasks | `bash pip install flytekitplugins-papermill ` | Execute entire notebooks as Flyte Tasks and pass inputs and outputs between them and python tasks | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-papermill.svg)](https://pypi.python.org/pypi/flytekitplugins-papermill/) | Flytekit-only | +| Pod Tasks | `bash pip install flytekitplugins-pod ` | Installs SDK to author Pods in python. These pods can have multiple containers, use volumes and have non exiting side-cars | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-pod.svg)](https://pypi.python.org/pypi/flytekitplugins-pod/) | Flytekit-only | +| spark | `bash pip install flytekitplugins-spark ` | Installs SDK to author Spark jobs that can be executed natively on Kubernetes with a supported backend Flyte plugin | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-spark.svg)](https://pypi.python.org/pypi/flytekitplugins-spark/) | Backend | +| AWS Athena Queries | `bash pip install flytekitplugins-athena ` | Installs SDK to author queries executed on AWS Athena | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-athena.svg)](https://pypi.python.org/pypi/flytekitplugins-athena/) | Backend | +| DOLT | `bash pip install flytekitplugins-dolt ` | Read & write dolt data sets and use dolt tables as native types | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-dolt.svg)](https://pypi.python.org/pypi/flytekitplugins-dolt/) | Flytekit-only | +| Pandera | `bash pip install flytekitplugins-pandera ` | Use Pandera schemas as native Flyte types, which enable data quality checks. | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-pandera.svg)](https://pypi.python.org/pypi/flytekitplugins-pandera/) | Flytekit-only | +| SQLAlchemy | `bash pip install flytekitplugins-sqlalchemy ` | Write queries for any database that supports SQLAlchemy | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-sqlalchemy.svg)](https://pypi.python.org/pypi/flytekitplugins-sqlalchemy/) | Flytekit-only | +| Great Expectations | `bash pip install flytekitplugins-great-expectations` | Enforce data quality for various data types within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-great-expectations.svg)](https://pypi.python.org/pypi/flytekitplugins-great-expectations/) | Flytekit-only | +| Snowflake | `bash pip install flytekitplugins-snowflake` | Use Snowflake as a 'data warehouse-as-a-service' within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-snowflake.svg)](https://pypi.python.org/pypi/flytekitplugins-snowflake/) | Backend | +| dbt | `bash pip install flytekitplugins-dbt` | Run dbt within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-dbt.svg)](https://pypi.python.org/pypi/flytekitplugins-dbt/) | Flytekit-only | +| Huggingface | `bash pip install flytekitplugins-huggingface` | Read & write Hugginface Datasets as Flyte StructuredDatasets | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-huggingface.svg)](https://pypi.python.org/pypi/flytekitplugins-huggingface/) | Flytekit-only | +| DuckDB | `bash pip install flytekitplugins-duckdb` | Run analytical workloads with ease using DuckDB | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-duckdb.svg)](https://pypi.python.org/pypi/flytekitplugins-duckdb/) | Flytekit-only | ## Have a Plugin Idea? 💡 + Please [file an issue](https://github.com/flyteorg/flyte/issues/new?assignees=&labels=untriaged%2Cplugins&template=backend-plugin-request.md&title=%5BPlugin%5D). ## Development 💻 + Flytekit plugins are structured as micro-libs and can be authored in an independent repository. > Refer to the [Python microlibs](https://medium.com/@jherreras/python-microlibs-5be9461ad979) blog to understand the idea of microlibs. @@ -36,15 +38,18 @@ Flytekit plugins are structured as micro-libs and can be authored in an independ The plugins maintained by the core team can be found in this repository and provide a simple way of discovery. ## Unit tests 🧪 + Plugins should have their own unit tests. ## Guidelines 📜 + Some guidelines to help you write the Flytekit plugins better. 1. The folder name has to be `flytekit-*`, e.g., `flytekit-hive`. In case you want to group for a specific service, then use `flytekit-aws-athena`. 2. Flytekit plugins use a concept called [Namespace packages](https://packaging.python.org/guides/creating-and-discovering-plugins/#using-namespace-packages), and thus, the package structure is essential. Please use the following Python package structure: + ``` flytekit-myplugin/ - README.md @@ -55,7 +60,8 @@ Some guidelines to help you write the Flytekit plugins better. - tests - __init__.py ``` - *NOTE:* the inner package `flytekitplugins` DOES NOT have an `__init__.py` file. + + _NOTE:_ the inner package `flytekitplugins` DOES NOT have an `__init__.py` file. 3. The published packages have to be named `flytekitplugins-{package-name}`, where `{package-name}` is a unique identifier for the plugin. @@ -113,23 +119,25 @@ setup( # entry_points={"flytekit.plugins": [f"{PLUGIN_NAME}=flytekitplugins.{PLUGIN_NAME}"]}, ) ``` -5. Each plugin should have a README.md, which describes how to install it with a simple example. For example, refer to flytekit-greatexpectations' [README](./flytekit-greatexpectations/README.md). -6. Each plugin should have its own tests' package. *NOTE:* `tests` folder should have an `__init__.py` file. +5. Each plugin should have a README.md, which describes how to install it with a simple example. For example, refer to flytekit-greatexpectations' [README](./flytekit-greatexpectations/README.md). -7. There may be some cases where you might want to auto-load some of your modules when the plugin is installed. This is especially true for `data-plugins` and `type-plugins`. -In such a case, you can add a special directive in the `setup.py` which will instruct Flytekit to automatically load the prescribed modules. +6. Each plugin should have its own tests' package. _NOTE:_ `tests` folder should have an `__init__.py` file. - Following shows an excerpt from the `flytekit-data-fsspec` plugin's setup.py file. +7. There may be some cases where you might want to auto-load some of your modules when the plugin is installed. This is especially true for `data-plugins` and `type-plugins`. + In such a case, you can add a special directive in the `setup.py` which will instruct Flytekit to automatically load the prescribed modules. - ```python - setup( - entry_points={"flytekit.plugins": [f"{PLUGIN_NAME}=flytekitplugins.{PLUGIN_NAME}"]}, - ) + Following shows an excerpt from the `flytekit-data-fsspec` plugin's setup.py file. - ``` +```python +setup( + entry_points={"flytekit.plugins": [f"{PLUGIN_NAME}=flytekitplugins.{PLUGIN_NAME}"]}, +) + +``` ### Flytekit Version Pinning + Currently we advocate pinning to minor releases of flytekit. To bump the pins across the board, `cd plugins/` and then update the command below with the appropriate range and run @@ -140,11 +148,12 @@ for f in $(ls **/setup.py); do sed -i "s/flytekit>.*,<1.1/flytekit>=1.1.0b0,<1.2 Try using `gsed` instead of `sed` if you are on a Mac. Also this only works of course for setup files that start with the version in your sed command. There may be plugins that have different pins to start out with. ## References 📚 + - Example of a simple Python task that allows adding only Python side functionality: [flytekit-greatexpectations](./flytekit-greatexpectations/) - Example of a TypeTransformer or a Type Plugin: [flytekit-pandera](./flytekit-pandera/). These plugins add new types to Flyte and tell Flyte how to transform them and add additional features through types. Flyte is a multi-lang system, and type transformers allow marshaling between Flytekit and backend and other languages. - Example of TaskTemplate plugin which also allows plugin writers to supply a prebuilt container for runtime: [flytekit-sqlalchemy](./flytekit-sqlalchemy/) - Example of a SQL backend plugin where the actual query invocation is done by a backend plugin: [flytekit-snowflake](./flytekit-snowflake/) - Example of a Meta plugin that can wrap other tasks: [flytekit-papermill](./flytekit-papermill/) -- Example of a plugin that modifies the execution command: [flytekit-spark](./flytekit-spark/) OR [flytekit-aws-sagemaker](./flytekit-aws-sagemaker/) +- Example of a plugin that modifies the execution command: [flytekit-spark](./flytekit-spark/) - Example that allows executing the user container with some other context modifications: [flytekit-kf-tensorflow](./flytekit-kf-tensorflow/) - Example of a Persistence Plugin that allows data to be stored to different persistence layers: [flytekit-data-fsspec](./flytekit-data-fsspec/) diff --git a/plugins/flytekit-airflow/flytekitplugins/airflow/task.py b/plugins/flytekit-airflow/flytekitplugins/airflow/task.py index 17a023dfdb..cf8f992ad9 100644 --- a/plugins/flytekit-airflow/flytekitplugins/airflow/task.py +++ b/plugins/flytekit-airflow/flytekitplugins/airflow/task.py @@ -137,7 +137,7 @@ def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: def _get_airflow_instance( - airflow_obj: AirflowObj + airflow_obj: AirflowObj, ) -> typing.Union[airflow_models.BaseOperator, airflow_sensors.BaseSensorOperator, airflow_triggers.BaseTrigger]: # Set the GET_ORIGINAL_TASK attribute to True so that obj_def will return the original # airflow task instead of the Flyte task. diff --git a/plugins/flytekit-aws-sagemaker/README.md b/plugins/flytekit-aws-sagemaker/README.md index 9f133175b5..2d57333353 100644 --- a/plugins/flytekit-aws-sagemaker/README.md +++ b/plugins/flytekit-aws-sagemaker/README.md @@ -1,6 +1,11 @@ -# Flytekit AWS Sagemaker Plugin +# AWS SageMaker Plugin -Amazon SageMaker provides several built-in machine learning algorithms that you can use for a variety of problem types. Flyte Sagemaker plugin intends to greatly simplify using Sagemaker for training. We have tried to distill the API into a meaningful subset that makes it easier for users to adopt and run with Sagemaker. +The plugin currently features a SageMaker deployment agent. + +## Inference + +The deployment agent enables you to deploy models, create and trigger inference endpoints. +Additionally, you can entirely remove the SageMaker deployment using the `delete_sagemaker_deployment` workflow. To install the plugin, run the following command: @@ -8,6 +13,60 @@ To install the plugin, run the following command: pip install flytekitplugins-awssagemaker ``` -To install Sagemaker in the Flyte deployment's backend, go through the [prerequisites](https://docs.flyte.org/en/latest/flytesnacks/examples/sagemaker_training_plugin/index.html). +Here is a sample SageMaker deployment workflow: + +```python +from flytekitplugins.awssagemaker_inference import create_sagemaker_deployment + -[Built-in sagemaker](https://docs.flyte.org/en/latest/flytesnacks/examples/sagemaker_training_plugin/index.html#builtin-algorithms) and [custom sagemaker](https://docs.flyte.org/en/latest/flytesnacks/examples/sagemaker_training_plugin/index.html#training-a-custom-model) training models can be found in the documentation. +REGION = os.getenv("REGION") +MODEL_NAME = "xgboost" +ENDPOINT_CONFIG_NAME = "xgboost-endpoint-config" +ENDPOINT_NAME = "xgboost-endpoint" + +sagemaker_deployment_wf = create_sagemaker_deployment( + name="sagemaker-deployment", + model_input_types=kwtypes(model_path=str, execution_role_arn=str), + model_config={ + "ModelName": MODEL_NAME, + "PrimaryContainer": { + "Image": "{images.deployment_image}", + "ModelDataUrl": "{inputs.model_path}", + }, + "ExecutionRoleArn": "{inputs.execution_role_arn}", + }, + endpoint_config_input_types=kwtypes(instance_type=str), + endpoint_config_config={ + "EndpointConfigName": ENDPOINT_CONFIG_NAME, + "ProductionVariants": [ + { + "VariantName": "variant-name-1", + "ModelName": MODEL_NAME, + "InitialInstanceCount": 1, + "InstanceType": "{inputs.instance_type}", + }, + ], + "AsyncInferenceConfig": { + "OutputConfig": {"S3OutputPath": os.getenv("S3_OUTPUT_PATH")} + }, + }, + endpoint_config={ + "EndpointName": ENDPOINT_NAME, + "EndpointConfigName": ENDPOINT_CONFIG_NAME, + }, + images={"deployment_image": custom_image}, + region=REGION, +) + + +@workflow +def model_deployment_workflow( + model_path: str = os.getenv("MODEL_DATA_URL"), + execution_role_arn: str = os.getenv("EXECUTION_ROLE_ARN"), +) -> str: + return sagemaker_deployment_wf( + model_path=model_path, + execution_role_arn=execution_role_arn, + instance_type="ml.m4.xlarge", + ) +``` diff --git a/plugins/flytekit-aws-sagemaker/dev-requirements.txt b/plugins/flytekit-aws-sagemaker/dev-requirements.txt new file mode 100644 index 0000000000..2d73dba5b4 --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/dev-requirements.txt @@ -0,0 +1 @@ +pytest-asyncio diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/__init__.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/__init__.py deleted file mode 100644 index 6dce099dfb..0000000000 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/__init__.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -.. currentmodule:: flytekitplugins.awssagemaker - -This package contains things that are useful when extending Flytekit. - -.. autosummary:: - :template: custom.rst - :toctree: generated/ - - AlgorithmName - AlgorithmSpecification - CategoricalParameterRange - ContinuousParameterRange - DISTRIBUTED_TRAINING_CONTEXT_KEY - DistributedProtocol - DistributedTrainingContext - HPOJob - HyperparameterScalingType - HyperparameterTuningJobConfig - HyperparameterTuningObjective - HyperparameterTuningObjectiveType - HyperparameterTuningStrategy - InputContentType - InputMode - IntegerParameterRange - ParameterRangeOneOf - SagemakerCustomTrainingTask - SagemakerHPOTask - SagemakerTrainingJobConfig - TrainingJobEarlyStoppingType - TrainingJobResourceConfig -""" - -__all__ = [ - "AlgorithmName", - "AlgorithmSpecification", - "CategoricalParameterRange", - "ContinuousParameterRange", - "DISTRIBUTED_TRAINING_CONTEXT_KEY", - "DistributedProtocol", - "DistributedTrainingContext", - "HPOJob", - "HyperparameterScalingType", - "HyperparameterTuningJobConfig", - "HyperparameterTuningObjective", - "HyperparameterTuningObjectiveType", - "HyperparameterTuningStrategy", - "InputContentType", - "InputMode", - "IntegerParameterRange", - "ParameterRangeOneOf", - "SagemakerBuiltinAlgorithmsTask", - "SagemakerCustomTrainingTask", - "SagemakerHPOTask", - "SagemakerTrainingJobConfig", - "TrainingJobEarlyStoppingType", - "TrainingJobResourceConfig", -] - -from flytekitplugins.awssagemaker.models.hpo_job import ( - HyperparameterTuningJobConfig, - HyperparameterTuningObjective, - HyperparameterTuningObjectiveType, - HyperparameterTuningStrategy, - TrainingJobEarlyStoppingType, -) -from flytekitplugins.awssagemaker.models.parameter_ranges import ( - CategoricalParameterRange, - ContinuousParameterRange, - HyperparameterScalingType, - IntegerParameterRange, - ParameterRangeOneOf, -) -from flytekitplugins.awssagemaker.models.training_job import ( - AlgorithmName, - AlgorithmSpecification, - DistributedProtocol, - InputContentType, - InputMode, - TrainingJobResourceConfig, -) - -from .distributed_training import DISTRIBUTED_TRAINING_CONTEXT_KEY, DistributedTrainingContext -from .hpo import HPOJob, SagemakerHPOTask -from .training import SagemakerBuiltinAlgorithmsTask, SagemakerCustomTrainingTask, SagemakerTrainingJobConfig diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/distributed_training.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/distributed_training.py deleted file mode 100644 index 2b69bd430d..0000000000 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/distributed_training.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -import json -import os -import typing -from dataclasses import dataclass - -import retry - -SM_RESOURCE_CONFIG_FILE = "/opt/ml/input/config/resourceconfig.json" -SM_ENV_VAR_CURRENT_HOST = "SM_CURRENT_HOST" -SM_ENV_VAR_HOSTS = "SM_HOSTS" -SM_ENV_VAR_NETWORK_INTERFACE_NAME = "SM_NETWORK_INTERFACE_NAME" - - -def setup_envars_for_testing(): - """ - This method is useful in simulating the env variables that sagemaker will set on the execution environment - """ - os.environ[SM_ENV_VAR_CURRENT_HOST] = "host" - os.environ[SM_ENV_VAR_HOSTS] = '["host1","host2"]' - os.environ[SM_ENV_VAR_NETWORK_INTERFACE_NAME] = "nw" - - -@dataclass -class DistributedTrainingContext(object): - current_host: str - hosts: typing.List[str] - network_interface_name: str - - @classmethod - @retry.retry(exceptions=KeyError, delay=1, tries=10, backoff=1) - def from_env(cls) -> DistributedTrainingContext: - """ - SageMaker suggests "Hostname information might not be immediately available to the processing container. - We recommend adding a retry policy on hostname resolution operations as nodes become available in the cluster." - https://docs.aws.amazon.com/sagemaker/latest/dg/build-your-own-processing-container.html#byoc-config - This is why we have an automatic retry policy - """ - curr_host = os.environ.get(SM_ENV_VAR_CURRENT_HOST) - raw_hosts = os.environ.get(SM_ENV_VAR_HOSTS) - nw_iface = os.environ.get(SM_ENV_VAR_NETWORK_INTERFACE_NAME) - if not (curr_host and raw_hosts and nw_iface): - raise KeyError("Unable to locate Sagemaker Environment variables!") - hosts = json.loads(raw_hosts) - return DistributedTrainingContext(curr_host, hosts, nw_iface) - - @classmethod - @retry.retry(exceptions=FileNotFoundError, delay=1, tries=10, backoff=1) - def from_sagemaker_context_file(cls) -> DistributedTrainingContext: - with open(SM_RESOURCE_CONFIG_FILE, "r") as rc_file: - d = json.load(rc_file) - curr_host = d["current_host"] - hosts = d["hosts"] - nw_iface = d["network_interface_name"] - - if not (curr_host and hosts and nw_iface): - raise KeyError - - return DistributedTrainingContext(curr_host, hosts, nw_iface) - - @classmethod - def local_execute(cls) -> DistributedTrainingContext: - """ - Creates a dummy local execution context for distributed execution. - TODO revisit if this is a good idea - """ - return DistributedTrainingContext(hosts=["localhost"], current_host="localhost", network_interface_name="dummy") - - -DISTRIBUTED_TRAINING_CONTEXT_KEY = "DISTRIBUTED_TRAINING_CONTEXT" -""" -Use this key to retrieve the distributed training context of type :py:class:`DistributedTrainingContext`. -Usage: - -.. code-block:: python - - ctx = flytekit.current_context().distributed_training_context - # OR - ctx = flytekit.current_context().get(sagemaker.DISTRIBUTED_TRAINING_CONTEXT_KEY) - -""" diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/hpo.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/hpo.py deleted file mode 100644 index 1229b96195..0000000000 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/hpo.py +++ /dev/null @@ -1,168 +0,0 @@ -import json -from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Type, Union - -from flyteidl.plugins.sagemaker import hyperparameter_tuning_job_pb2 as _pb2_hpo_job -from flyteidl.plugins.sagemaker import parameter_ranges_pb2 as _pb2_params -from flytekitplugins.awssagemaker.training import SagemakerBuiltinAlgorithmsTask, SagemakerCustomTrainingTask -from google.protobuf import json_format -from google.protobuf.json_format import MessageToDict - -from flytekit import FlyteContext -from flytekit.configuration import SerializationSettings -from flytekit.extend import DictTransformer, PythonTask, TypeEngine, TypeTransformer -from flytekit.models.literals import Literal -from flytekit.models.types import LiteralType, SimpleType - -from .models import hpo_job as _hpo_job_model -from .models import parameter_ranges as _params -from .models import training_job as _training_job_model - - -@dataclass -class HPOJob(object): - """ - HPOJob Configuration should be used to configure the HPO Job. - - Args: - max_number_of_training_jobs: maximum number of jobs to run for a training round - max_parallel_training_jobs: limits the concurrency of the training jobs - tunable_params: [optional] should be a list of parameters for which we want to provide the tuning ranges - """ - - max_number_of_training_jobs: int - max_parallel_training_jobs: int - # TODO. we could make the tunable params a tuple of name and type of range? - tunable_params: Optional[List[str]] = None - - -# TODO Not done yet, but once we clean this up, the client interface should be simplified. The interface should -# Just take a list of Union of different types of Parameter Ranges. Lets see how simplify that -class SagemakerHPOTask(PythonTask[HPOJob]): - _SAGEMAKER_HYPERPARAMETER_TUNING_JOB_TASK = "sagemaker_hyperparameter_tuning_job_task" - - def __init__( - self, - name: str, - task_config: HPOJob, - training_task: Union[SagemakerCustomTrainingTask, SagemakerBuiltinAlgorithmsTask], - **kwargs, - ): - if training_task is None or not ( - isinstance(training_task, SagemakerCustomTrainingTask) - or isinstance(training_task, SagemakerBuiltinAlgorithmsTask) - ): - raise ValueError( - "Training Task of type SagemakerCustomTrainingTask/SagemakerBuiltinAlgorithmsTask is required to work" - " with Sagemaker HPO" - ) - - self._task_config = task_config - self._training_task = training_task - - extra_inputs = {"hyperparameter_tuning_job_config": _hpo_job_model.HyperparameterTuningJobConfig} - - if task_config.tunable_params: - extra_inputs.update({param: _params.ParameterRangeOneOf for param in task_config.tunable_params}) - - iface = training_task.python_interface - updated_iface = iface.with_inputs(extra_inputs) - super().__init__( - task_type=self._SAGEMAKER_HYPERPARAMETER_TUNING_JOB_TASK, - name=name, - interface=updated_iface, - task_config=task_config, - **kwargs, - ) - - def execute(self, **kwargs) -> Any: - raise NotImplementedError("Sagemaker HPO Task cannot be executed locally, to execute locally mock it!") - - def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: - training_job = _training_job_model.TrainingJob( - algorithm_specification=self._training_task.task_config.algorithm_specification, - training_job_resource_config=self._training_task.task_config.training_job_resource_config, - ) - return MessageToDict( - _hpo_job_model.HyperparameterTuningJob( - max_number_of_training_jobs=self.task_config.max_number_of_training_jobs, - max_parallel_training_jobs=self.task_config.max_parallel_training_jobs, - training_job=training_job, - ).to_flyte_idl() - ) - - -# %% -# HPO Task allows ParameterRangeOneOf and HyperparameterTuningJobConfig as inputs. In flytekit this is possible -# to allow these two types to be registered as valid input / output types and provide a custom transformer -# We will create custom transformers for them as follows and provide them once a user loads HPO task - - -class HPOTuningJobConfigTransformer(TypeTransformer[_hpo_job_model.HyperparameterTuningJobConfig]): - """ - Transformer to make ``HyperparameterTuningJobConfig`` an accepted value, for which a transformer is registered - """ - - def __init__(self): - super().__init__("sagemaker-hpojobconfig-transformer", _hpo_job_model.HyperparameterTuningJobConfig) - - def get_literal_type(self, t: Type[_hpo_job_model.HyperparameterTuningJobConfig]) -> LiteralType: - return LiteralType(simple=SimpleType.STRUCT, metadata=None) - - def to_literal( - self, - ctx: FlyteContext, - python_val: _hpo_job_model.HyperparameterTuningJobConfig, - python_type: Type[_hpo_job_model.HyperparameterTuningJobConfig], - expected: LiteralType, - ) -> Literal: - d = MessageToDict(python_val.to_flyte_idl()) - return DictTransformer.dict_to_generic_literal(d) - - def to_python_value( - self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[_hpo_job_model.HyperparameterTuningJobConfig] - ) -> _hpo_job_model.HyperparameterTuningJobConfig: - if lv and lv.scalar and lv.scalar.generic is not None: - d = json.loads(json_format.MessageToJson(lv.scalar.generic)) - o = _pb2_hpo_job.HyperparameterTuningJobConfig() - o = json_format.ParseDict(d, o) - return _hpo_job_model.HyperparameterTuningJobConfig.from_flyte_idl(o) - return None - - -class ParameterRangesTransformer(TypeTransformer[_params.ParameterRangeOneOf]): - """ - Transformer to make ``ParameterRange`` an accepted value, for which a transformer is registered - """ - - def __init__(self): - super().__init__("sagemaker-paramrange-transformer", _params.ParameterRangeOneOf) - - def get_literal_type(self, t: Type[_params.ParameterRangeOneOf]) -> LiteralType: - return LiteralType(simple=SimpleType.STRUCT, metadata=None) - - def to_literal( - self, - ctx: FlyteContext, - python_val: _params.ParameterRangeOneOf, - python_type: Type[_hpo_job_model.HyperparameterTuningJobConfig], - expected: LiteralType, - ) -> Literal: - d = MessageToDict(python_val.to_flyte_idl()) - return DictTransformer.dict_to_generic_literal(d) - - def to_python_value( - self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[_params.ParameterRangeOneOf] - ) -> _params.ParameterRangeOneOf: - if lv and lv.scalar and lv.scalar.generic is not None: - d = json.loads(json_format.MessageToJson(lv.scalar.generic)) - o = _pb2_params.ParameterRangeOneOf() - o = json_format.ParseDict(d, o) - return _params.ParameterRangeOneOf.from_flyte_idl(o) - return None - - -# %% -# Register the types -TypeEngine.register(HPOTuningJobConfigTransformer()) -TypeEngine.register(ParameterRangesTransformer()) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/__init__.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/hpo_job.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/hpo_job.py deleted file mode 100644 index 6d6b17189f..0000000000 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/hpo_job.py +++ /dev/null @@ -1,181 +0,0 @@ -from flyteidl.plugins.sagemaker import hyperparameter_tuning_job_pb2 as _pb2_hpo_job - -from flytekit.models import common as _common - -from . import training_job as _training_job - - -class HyperparameterTuningObjectiveType(object): - MINIMIZE = _pb2_hpo_job.HyperparameterTuningObjectiveType.MINIMIZE - MAXIMIZE = _pb2_hpo_job.HyperparameterTuningObjectiveType.MAXIMIZE - - -class HyperparameterTuningObjective(_common.FlyteIdlEntity): - """ - HyperparameterTuningObjective is a data structure that contains the target metric and the - objective of the hyperparameter tuning. - - https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-metrics.html - """ - - def __init__( - self, - objective_type: int, - metric_name: str, - ): - self._objective_type = objective_type - self._metric_name = metric_name - - @property - def objective_type(self) -> int: - """ - Enum value of HyperparameterTuningObjectiveType. objective_type determines the direction of the tuning of - the Hyperparameter Tuning Job with respect to the specified metric. - :rtype: int - """ - return self._objective_type - - @property - def metric_name(self) -> str: - """ - The target metric name, which is the user-defined name of the metric specified in the - training job's algorithm specification - :rtype: str - """ - return self._metric_name - - def to_flyte_idl(self) -> _pb2_hpo_job.HyperparameterTuningObjective: - return _pb2_hpo_job.HyperparameterTuningObjective( - objective_type=self.objective_type, - metric_name=self._metric_name, - ) - - @classmethod - def from_flyte_idl(cls, pb2_object: _pb2_hpo_job.HyperparameterTuningObjective): - return cls( - objective_type=pb2_object.objective_type, - metric_name=pb2_object.metric_name, - ) - - -class HyperparameterTuningStrategy: - BAYESIAN = _pb2_hpo_job.HyperparameterTuningStrategy.BAYESIAN - RANDOM = _pb2_hpo_job.HyperparameterTuningStrategy.RANDOM - - -class TrainingJobEarlyStoppingType: - OFF = _pb2_hpo_job.TrainingJobEarlyStoppingType.OFF - AUTO = _pb2_hpo_job.TrainingJobEarlyStoppingType.AUTO - - -class HyperparameterTuningJobConfig(_common.FlyteIdlEntity): - """ - The specification of the hyperparameter tuning process - https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-ex-tuning-job.html#automatic-model-tuning-ex-low-tuning-config - """ - - def __init__( - self, - tuning_strategy: int, - tuning_objective: HyperparameterTuningObjective, - training_job_early_stopping_type: TrainingJobEarlyStoppingType, - ): - self._tuning_strategy = tuning_strategy - self._tuning_objective = tuning_objective - self._training_job_early_stopping_type = training_job_early_stopping_type - - @property - def tuning_strategy(self) -> int: - """ - Enum value of HyperparameterTuningStrategy. Setting the strategy used when searching in the hyperparameter space - :rtype: int - """ - return self._tuning_strategy - - @property - def tuning_objective(self) -> HyperparameterTuningObjective: - """ - The target metric and the objective of the hyperparameter tuning. - :rtype: HyperparameterTuningObjective - """ - return self._tuning_objective - - @property - def training_job_early_stopping_type(self) -> int: - """ - Enum value of TrainingJobEarlyStoppingType. When the training jobs launched by the hyperparameter tuning job - are not improving significantly, a hyperparameter tuning job can be stopping early. This attribute determines - how the early stopping is to be done. - Note that there's only a subset of built-in algorithms that supports early stopping. - see: https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-early-stopping.html - :rtype: int - """ - return self._training_job_early_stopping_type - - def to_flyte_idl(self) -> _pb2_hpo_job.HyperparameterTuningJobConfig: - return _pb2_hpo_job.HyperparameterTuningJobConfig( - tuning_strategy=self._tuning_strategy, - tuning_objective=self._tuning_objective.to_flyte_idl(), - training_job_early_stopping_type=self._training_job_early_stopping_type, - ) - - @classmethod - def from_flyte_idl(cls, pb2_object: _pb2_hpo_job.HyperparameterTuningJobConfig): - return cls( - tuning_strategy=pb2_object.tuning_strategy, - tuning_objective=HyperparameterTuningObjective.from_flyte_idl(pb2_object.tuning_objective), - training_job_early_stopping_type=pb2_object.training_job_early_stopping_type, - ) - - -class HyperparameterTuningJob(_common.FlyteIdlEntity): - def __init__( - self, - max_number_of_training_jobs: int, - max_parallel_training_jobs: int, - training_job: _training_job.TrainingJob, - ): - self._max_number_of_training_jobs = max_number_of_training_jobs - self._max_parallel_training_jobs = max_parallel_training_jobs - self._training_job = training_job - - @property - def max_number_of_training_jobs(self) -> int: - """ - The maximum number of training jobs that a hyperparameter tuning job can launch. - https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_ResourceLimits.html - :rtype: int - """ - return self._max_number_of_training_jobs - - @property - def max_parallel_training_jobs(self) -> int: - """ - The maximum number of concurrent training job that an hpo job can launch - https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_ResourceLimits.html - :rtype: int - """ - return self._max_parallel_training_jobs - - @property - def training_job(self) -> _training_job.TrainingJob: - """ - The reference to the underlying training job that the hyperparameter tuning job will launch during the process - :rtype: _training_job.TrainingJob - """ - return self._training_job - - def to_flyte_idl(self) -> _pb2_hpo_job.HyperparameterTuningJob: - return _pb2_hpo_job.HyperparameterTuningJob( - max_number_of_training_jobs=self._max_number_of_training_jobs, - max_parallel_training_jobs=self._max_parallel_training_jobs, - training_job=self._training_job.to_flyte_idl(), # SDK task has already serialized it - ) - - @classmethod - def from_flyte_idl(cls, pb2_object: _pb2_hpo_job.HyperparameterTuningJob): - return cls( - max_number_of_training_jobs=pb2_object.max_number_of_training_jobs, - max_parallel_training_jobs=pb2_object.max_parallel_training_jobs, - training_job=_training_job.TrainingJob.from_flyte_idl(pb2_object.training_job), - ) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/parameter_ranges.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/parameter_ranges.py deleted file mode 100644 index 738f1820a2..0000000000 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/parameter_ranges.py +++ /dev/null @@ -1,315 +0,0 @@ -from typing import Dict, List, Optional, Union - -from flyteidl.plugins.sagemaker import parameter_ranges_pb2 as _idl_parameter_ranges - -from flytekit.exceptions import user -from flytekit.models import common as _common - - -class HyperparameterScalingType(object): - AUTO = _idl_parameter_ranges.HyperparameterScalingType.AUTO - LINEAR = _idl_parameter_ranges.HyperparameterScalingType.LINEAR - LOGARITHMIC = _idl_parameter_ranges.HyperparameterScalingType.LOGARITHMIC - REVERSELOGARITHMIC = _idl_parameter_ranges.HyperparameterScalingType.REVERSELOGARITHMIC - - -class ContinuousParameterRange(_common.FlyteIdlEntity): - def __init__( - self, - max_value: float, - min_value: float, - scaling_type: int, - ): - """ - - :param float max_value: - :param float min_value: - :param int scaling_type: - """ - self._max_value = max_value - self._min_value = min_value - self._scaling_type = scaling_type - - @property - def max_value(self) -> float: - """ - - :rtype: float - """ - return self._max_value - - @property - def min_value(self) -> float: - """ - - :rtype: float - """ - return self._min_value - - @property - def scaling_type(self) -> int: - """ - enum value from HyperparameterScalingType - :rtype: int - """ - return self._scaling_type - - def to_flyte_idl(self) -> _idl_parameter_ranges.ContinuousParameterRange: - """ - :rtype: _idl_parameter_ranges.ContinuousParameterRange - """ - - return _idl_parameter_ranges.ContinuousParameterRange( - max_value=self._max_value, - min_value=self._min_value, - scaling_type=self.scaling_type, - ) - - @classmethod - def from_flyte_idl(cls, pb2_object: _idl_parameter_ranges.ContinuousParameterRange): - """ - - :param pb2_object: - :rtype: ContinuousParameterRange - """ - return cls( - max_value=pb2_object.max_value, - min_value=pb2_object.min_value, - scaling_type=pb2_object.scaling_type, - ) - - -class IntegerParameterRange(_common.FlyteIdlEntity): - def __init__( - self, - max_value: int, - min_value: int, - scaling_type: int, - ): - """ - :param int max_value: - :param int min_value: - :param int scaling_type: - """ - self._max_value = max_value - self._min_value = min_value - self._scaling_type = scaling_type - - @property - def max_value(self) -> int: - """ - :rtype: int - """ - return self._max_value - - @property - def min_value(self) -> int: - """ - - :rtype: int - """ - return self._min_value - - @property - def scaling_type(self) -> int: - """ - enum value from HyperparameterScalingType - :rtype: int - """ - return self._scaling_type - - def to_flyte_idl(self) -> _idl_parameter_ranges.IntegerParameterRange: - """ - :rtype: _idl_parameter_ranges.IntegerParameterRange - """ - return _idl_parameter_ranges.IntegerParameterRange( - max_value=self._max_value, - min_value=self._min_value, - scaling_type=self.scaling_type, - ) - - @classmethod - def from_flyte_idl(cls, pb2_object: _idl_parameter_ranges.IntegerParameterRange): - """ - - :param pb2_object: - :rtype: IntegerParameterRange - """ - return cls( - max_value=pb2_object.max_value, - min_value=pb2_object.min_value, - scaling_type=pb2_object.scaling_type, - ) - - -class CategoricalParameterRange(_common.FlyteIdlEntity): - def __init__( - self, - values: List[str], - ): - """ - - :param List[str] values: list of strings representing categorical values - """ - self._values = values - - @property - def values(self) -> List[str]: - """ - :rtype: List[str] - """ - return self._values - - def to_flyte_idl(self) -> _idl_parameter_ranges.CategoricalParameterRange: - """ - :rtype: _idl_parameter_ranges.CategoricalParameterRange - """ - return _idl_parameter_ranges.CategoricalParameterRange(values=self._values) - - @classmethod - def from_flyte_idl(cls, pb2_object: _idl_parameter_ranges.CategoricalParameterRange): - """ - - :param pb2_object: - :rtype: CategoricalParameterRange - """ - return cls(values=[v for v in pb2_object.values]) - - -class ParameterRanges(_common.FlyteIdlEntity): - def __init__( - self, - parameter_range_map: Dict[str, _common.FlyteIdlEntity], - ): - self._parameter_range_map = parameter_range_map - - def to_flyte_idl(self) -> _idl_parameter_ranges.ParameterRanges: - """ - - :rtype: _idl_parameter_ranges.ParameterRanges - """ - converted = {} - for k, v in self._parameter_range_map.items(): - if isinstance(v, IntegerParameterRange): - converted[k] = _idl_parameter_ranges.ParameterRangeOneOf(integer_parameter_range=v.to_flyte_idl()) - elif isinstance(v, ContinuousParameterRange): - converted[k] = _idl_parameter_ranges.ParameterRangeOneOf(continuous_parameter_range=v.to_flyte_idl()) - elif isinstance(v, CategoricalParameterRange): - converted[k] = _idl_parameter_ranges.ParameterRangeOneOf(categorical_parameter_range=v.to_flyte_idl()) - else: - raise user.FlyteTypeException( - received_type=type(v), - expected_type=type( - Union[IntegerParameterRange, ContinuousParameterRange, CategoricalParameterRange] - ), - ) - - return _idl_parameter_ranges.ParameterRanges( - parameter_range_map=converted, - ) - - @classmethod - def from_flyte_idl(cls, pb2_object: _idl_parameter_ranges.ParameterRanges): - """ - - :param pb2_object: - :rtype: ParameterRanges - """ - converted = {} - for k, v in pb2_object.parameter_range_map.items(): - if v.HasField("continuous_parameter_range"): - converted[k] = ContinuousParameterRange.from_flyte_idl(v.continuous_parameter_range) - elif v.HasField("integer_parameter_range"): - converted[k] = IntegerParameterRange.from_flyte_idl(v.integer_parameter_range) - else: - converted[k] = CategoricalParameterRange.from_flyte_idl(v.categorical_parameter_range) - - return cls( - parameter_range_map=converted, - ) - - -class ParameterRangeOneOf(_common.FlyteIdlEntity): - def __init__(self, param: Union[IntegerParameterRange, ContinuousParameterRange, CategoricalParameterRange]): - """ - Initializes a new ParameterRangeOneOf. - - :param Union[IntegerParameterRange, ContinuousParameterRange, CategoricalParameterRange] param: One of the - supported parameter ranges. - """ - self._integer_parameter_range = param if isinstance(param, IntegerParameterRange) else None - self._continuous_parameter_range = param if isinstance(param, ContinuousParameterRange) else None - self._categorical_parameter_range = param if isinstance(param, CategoricalParameterRange) else None - - @property - def integer_parameter_range(self) -> Optional[IntegerParameterRange]: - """ - Retrieves the integer parameter range if one is set. None otherwise. - :rtype: Optional[IntegerParameterRange] - """ - if self._integer_parameter_range: - return self._integer_parameter_range - - return None - - @property - def continuous_parameter_range(self) -> Optional[ContinuousParameterRange]: - """ - Retrieves the continuous parameter range if one is set. None otherwise. - :rtype: Optional[ContinuousParameterRange] - """ - if self._continuous_parameter_range: - return self._continuous_parameter_range - - return None - - @property - def categorical_parameter_range(self) -> Optional[CategoricalParameterRange]: - """ - Retrieves the categorical parameter range if one is set. None otherwise. - :rtype: Optional[CategoricalParameterRange] - """ - if self._categorical_parameter_range: - return self._categorical_parameter_range - - return None - - def to_flyte_idl(self) -> _idl_parameter_ranges.ParameterRangeOneOf: - return _idl_parameter_ranges.ParameterRangeOneOf( - integer_parameter_range=self.integer_parameter_range.to_flyte_idl() - if self.integer_parameter_range - else None, - continuous_parameter_range=self.continuous_parameter_range.to_flyte_idl() - if self.continuous_parameter_range - else None, - categorical_parameter_range=self.categorical_parameter_range.to_flyte_idl() - if self.categorical_parameter_range - else None, - ) - - @classmethod - def from_flyte_idl( - cls, - pb_object: Union[ - _idl_parameter_ranges.ParameterRangeOneOf, - _idl_parameter_ranges.IntegerParameterRange, - _idl_parameter_ranges.ContinuousParameterRange, - _idl_parameter_ranges.CategoricalParameterRange, - ], - ): - param = None - if isinstance(pb_object, _idl_parameter_ranges.ParameterRangeOneOf): - if pb_object.HasField("continuous_parameter_range"): - param = ContinuousParameterRange.from_flyte_idl(pb_object.continuous_parameter_range) - elif pb_object.HasField("integer_parameter_range"): - param = IntegerParameterRange.from_flyte_idl(pb_object.integer_parameter_range) - elif pb_object.HasField("categorical_parameter_range"): - param = CategoricalParameterRange.from_flyte_idl(pb_object.categorical_parameter_range) - elif isinstance(pb_object, _idl_parameter_ranges.IntegerParameterRange): - param = IntegerParameterRange.from_flyte_idl(pb_object) - elif isinstance(pb_object, _idl_parameter_ranges.ContinuousParameterRange): - param = ContinuousParameterRange.from_flyte_idl(pb_object) - elif isinstance(pb_object, _idl_parameter_ranges.CategoricalParameterRange): - param = CategoricalParameterRange.from_flyte_idl(pb_object) - - return cls(param=param) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/training_job.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/training_job.py deleted file mode 100644 index 238aa27fa4..0000000000 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/models/training_job.py +++ /dev/null @@ -1,326 +0,0 @@ -from typing import List - -from flyteidl.plugins.sagemaker import training_job_pb2 as _training_job_pb2 - -from flytekit.models import common as _common - - -class DistributedProtocol(object): - """ - The distribution framework is used for determining which underlying distributed training mechanism to use. - This is only required for use cases where the user wants to train its custom training job in a distributed manner - """ - - UNSPECIFIED = _training_job_pb2.DistributedProtocol.UNSPECIFIED - MPI = _training_job_pb2.DistributedProtocol.MPI - - -class TrainingJobResourceConfig(_common.FlyteIdlEntity): - """ - TrainingJobResourceConfig is a pass-through, specifying the instance type to use for the training job, the - number of instances to launch, and the size of the ML storage volume the user wants to provision - Refer to SageMaker official doc for more details: https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateTrainingJob.html - """ - - def __init__( - self, - instance_type: str, - volume_size_in_gb: int, - instance_count: int = 1, - distributed_protocol: int = DistributedProtocol.UNSPECIFIED, - ): - self._instance_count = instance_count - self._instance_type = instance_type - self._volume_size_in_gb = volume_size_in_gb - self._distributed_protocol = distributed_protocol - - @property - def instance_count(self) -> int: - """ - The number of ML compute instances to use. For distributed training, provide a value greater than 1. - :rtype: int - """ - return self._instance_count - - @property - def instance_type(self) -> str: - """ - The ML compute instance type. - :rtype: str - """ - return self._instance_type - - @property - def volume_size_in_gb(self) -> int: - """ - The size of the ML storage volume that you want to provision to store the data and intermediate artifacts, etc. - :rtype: int - """ - return self._volume_size_in_gb - - @property - def distributed_protocol(self) -> int: - """ - The distribution framework is used to determine through which mechanism the distributed training is done. - enum value from DistributionFramework. - :rtype: int - """ - return self._distributed_protocol - - def to_flyte_idl(self) -> _training_job_pb2.TrainingJobResourceConfig: - """ - - :rtype: _training_job_pb2.TrainingJobResourceConfig - """ - return _training_job_pb2.TrainingJobResourceConfig( - instance_count=self.instance_count, - instance_type=self.instance_type, - volume_size_in_gb=self.volume_size_in_gb, - distributed_protocol=self.distributed_protocol, - ) - - @classmethod - def from_flyte_idl(cls, pb2_object: _training_job_pb2.TrainingJobResourceConfig): - """ - - :param pb2_object: - :rtype: TrainingJobResourceConfig - """ - return cls( - instance_count=pb2_object.instance_count, - instance_type=pb2_object.instance_type, - volume_size_in_gb=pb2_object.volume_size_in_gb, - distributed_protocol=pb2_object.distributed_protocol, - ) - - -class MetricDefinition(_common.FlyteIdlEntity): - def __init__( - self, - name: str, - regex: str, - ): - self._name = name - self._regex = regex - - @property - def name(self) -> str: - """ - The user-defined name of the metric - :rtype: str - """ - return self._name - - @property - def regex(self) -> str: - """ - SageMaker hyperparameter tuning using this regex to parses your algorithm’s stdout and stderr - streams to find the algorithm metrics on which the users want to track - :rtype: str - """ - return self._regex - - def to_flyte_idl(self) -> _training_job_pb2.MetricDefinition: - """ - - :rtype: _training_job_pb2.MetricDefinition - """ - return _training_job_pb2.MetricDefinition( - name=self.name, - regex=self.regex, - ) - - @classmethod - def from_flyte_idl(cls, pb2_object: _training_job_pb2.MetricDefinition): - """ - - :param pb2_object: _training_job_pb2.MetricDefinition - :rtype: MetricDefinition - """ - return cls( - name=pb2_object.name, - regex=pb2_object.regex, - ) - - -# TODO Convert to Enum -class InputMode(object): - """ - When using FILE input mode, different SageMaker built-in algorithms require different file types of input data - See https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html - https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html - """ - - PIPE = _training_job_pb2.InputMode.PIPE - FILE = _training_job_pb2.InputMode.FILE - - -# TODO Convert to enum -class AlgorithmName(object): - """ - The algorithm name is used for deciding which pre-built image to point to. - This is only required for use cases where SageMaker's built-in algorithm mode is used. - While we currently only support a subset of the algorithms, more will be added to the list. - See: https://docs.aws.amazon.com/sagemaker/latest/dg/algos.html - """ - - CUSTOM = _training_job_pb2.AlgorithmName.CUSTOM - XGBOOST = _training_job_pb2.AlgorithmName.XGBOOST - - -# TODO convert to enum -class InputContentType(object): - """ - Specifies the type of content for input data. Different SageMaker built-in algorithms require different content types of input data - See https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html - https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html - """ - - TEXT_CSV = _training_job_pb2.InputContentType.TEXT_CSV - - -class AlgorithmSpecification(_common.FlyteIdlEntity): - """ - Specifies the training algorithm to be used in the training job - This object is mostly a pass-through, with a couple of exceptions include: (1) in Flyte, users don't need to specify - TrainingImage; either use the built-in algorithm mode by using Flytekit's Simple Training Job and specifying an algorithm - name and an algorithm version or (2) when users want to supply custom algorithms they should set algorithm_name field to - CUSTOM. In this case, the value of the algorithm_version field has no effect - For pass-through use cases: refer to this AWS official document for more details - https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html - """ - - def __init__( - self, - algorithm_name: int = AlgorithmName.CUSTOM, - algorithm_version: str = "", - input_mode: int = InputMode.FILE, - metric_definitions: List[MetricDefinition] = None, - input_content_type: int = InputContentType.TEXT_CSV, - ): - self._input_mode = input_mode - self._input_content_type = input_content_type - self._algorithm_name = algorithm_name - self._algorithm_version = algorithm_version - self._metric_definitions = metric_definitions or [] - - @property - def input_mode(self) -> int: - """ - enum value from InputMode. The input mode can be either PIPE or FILE - :rtype: int - """ - return self._input_mode - - @property - def input_content_type(self) -> int: - """ - enum value from InputContentType. The content type of the input data - See https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-training.html - https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-algo-docker-registry-paths.html - :rtype: int - """ - return self._input_content_type - - @property - def algorithm_name(self) -> int: - """ - The algorithm name is used for deciding which pre-built image to point to. - enum value from AlgorithmName. - :rtype: int - """ - return self._algorithm_name - - @property - def algorithm_version(self) -> str: - """ - version of the algorithm (if using built-in algorithm mode). - :rtype: str - """ - return self._algorithm_version - - @property - def metric_definitions(self) -> List[MetricDefinition]: - """ - A list of metric definitions for SageMaker to evaluate/track on the progress of the training job - See this: https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_AlgorithmSpecification.html - - Note that, when you use one of the Amazon SageMaker built-in algorithms, you cannot define custom metrics. - If you are doing hyperparameter tuning, built-in algorithms automatically send metrics to hyperparameter tuning. - When using hyperparameter tuning, you do need to choose one of the metrics that the built-in algorithm emits as - the objective metric for the tuning job. - See this: https://docs.aws.amazon.com/sagemaker/latest/dg/automatic-model-tuning-define-metrics.html - :rtype: List[MetricDefinition] - """ - return self._metric_definitions - - def to_flyte_idl(self) -> _training_job_pb2.AlgorithmSpecification: - return _training_job_pb2.AlgorithmSpecification( - input_mode=self.input_mode, - algorithm_name=self.algorithm_name, - algorithm_version=self.algorithm_version, - metric_definitions=[m.to_flyte_idl() for m in self.metric_definitions], - input_content_type=self.input_content_type, - ) - - @classmethod - def from_flyte_idl(cls, pb2_object: _training_job_pb2.AlgorithmSpecification): - return cls( - input_mode=pb2_object.input_mode, - algorithm_name=pb2_object.algorithm_name, - algorithm_version=pb2_object.algorithm_version, - metric_definitions=[MetricDefinition.from_flyte_idl(m) for m in pb2_object.metric_definitions], - input_content_type=pb2_object.input_content_type, - ) - - -class TrainingJob(_common.FlyteIdlEntity): - def __init__( - self, - algorithm_specification: AlgorithmSpecification, - training_job_resource_config: TrainingJobResourceConfig, - ): - self._algorithm_specification = algorithm_specification - self._training_job_resource_config = training_job_resource_config - - @property - def algorithm_specification(self) -> AlgorithmSpecification: - """ - Contains the information related to the algorithm to use in the training job - :rtype: AlgorithmSpecification - """ - return self._algorithm_specification - - @property - def training_job_resource_config(self) -> TrainingJobResourceConfig: - """ - Specifies the information around the instances that will be used to run the training job. - :rtype: TrainingJobResourceConfig - """ - return self._training_job_resource_config - - def to_flyte_idl(self) -> _training_job_pb2.TrainingJob: - """ - :rtype: _training_job_pb2.TrainingJob - """ - - return _training_job_pb2.TrainingJob( - algorithm_specification=self.algorithm_specification.to_flyte_idl() - if self.algorithm_specification - else None, - training_job_resource_config=self.training_job_resource_config.to_flyte_idl() - if self.training_job_resource_config - else None, - ) - - @classmethod - def from_flyte_idl(cls, pb2_object: _training_job_pb2.TrainingJob): - """ - - :param pb2_object: - :rtype: TrainingJob - """ - return cls( - algorithm_specification=pb2_object.algorithm_specification, - training_job_resource_config=pb2_object.training_job_resource_config, - ) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/training.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/training.py deleted file mode 100644 index 7f456d19a0..0000000000 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker/training.py +++ /dev/null @@ -1,192 +0,0 @@ -import typing -from dataclasses import dataclass -from typing import Any, Callable, Dict - -from flytekitplugins.awssagemaker.distributed_training import DistributedTrainingContext -from google.protobuf.json_format import MessageToDict -from typing_extensions import Annotated - -import flytekit -from flytekit import ExecutionParameters, FlyteContextManager, PythonFunctionTask, kwtypes -from flytekit.configuration import SerializationSettings -from flytekit.extend import ExecutionState, IgnoreOutputs, Interface, PythonTask, TaskPlugins -from flytekit.loggers import logger -from flytekit.types.directory.types import FlyteDirectory -from flytekit.types.file import FileExt, FlyteFile - -from .models import training_job as _training_job_models - - -@dataclass -class SagemakerTrainingJobConfig(object): - """ - Configuration for Running Training Jobs on Sagemaker. This config can be used to run either the built-in algorithms - or custom algorithms. - - Args: - training_job_resource_config: Configuration for Resources to use during the training - algorithm_specification: Specification of the algorithm to use - should_persist_output: This method will be invoked and will decide if the generated model should be persisted - as the output. ``NOTE: Useful only for distributed training`` - ``default: single node training - always persist output`` - ``default: distributed training - always persist output on node with rank-0`` - """ - - training_job_resource_config: _training_job_models.TrainingJobResourceConfig - algorithm_specification: _training_job_models.AlgorithmSpecification - # The default output-persisting predicate. - # With this predicate, only the copy running on the first host in the list of hosts would persist its output - should_persist_output: typing.Callable[[DistributedTrainingContext], bool] = lambda dctx: ( - dctx.current_host == dctx.hosts[0] - ) - - -class SagemakerBuiltinAlgorithmsTask(PythonTask[SagemakerTrainingJobConfig]): - """ - Implements an interface that allows execution of a SagemakerBuiltinAlgorithms. - Refer to `Sagemaker Builtin Algorithms`_ for details. - """ - - _SAGEMAKER_TRAINING_JOB_TASK = "sagemaker_training_job_task" - - OUTPUT_TYPE = Annotated[str, FileExt("tar.gz")] - - def __init__( - self, - name: str, - task_config: SagemakerTrainingJobConfig, - **kwargs, - ): - """ - Args: - name: name of this specific task. This should be unique within the project. A good strategy is to prefix - with the module name - metadata: Metadata for the task - task_config: Config to use for the SagemakerBuiltinAlgorithms - """ - if ( - task_config is None - or task_config.algorithm_specification is None - or task_config.training_job_resource_config is None - ): - raise ValueError("TaskConfig, algorithm_specification, training_job_resource_config are required") - - input_type = Annotated[ - str, FileExt(self._content_type_to_blob_format(task_config.algorithm_specification.input_content_type)) - ] - - interface = Interface( - # TODO change train and validation to be FlyteDirectory when available - inputs=kwtypes( - static_hyperparameters=dict, train=FlyteDirectory[input_type], validation=FlyteDirectory[input_type] - ), - outputs=kwtypes(model=FlyteFile[self.OUTPUT_TYPE]), - ) - super().__init__( - self._SAGEMAKER_TRAINING_JOB_TASK, - name, - interface=interface, - task_config=task_config, - **kwargs, - ) - - def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: - training_job = _training_job_models.TrainingJob( - algorithm_specification=self._task_config.algorithm_specification, - training_job_resource_config=self._task_config.training_job_resource_config, - ) - return MessageToDict(training_job.to_flyte_idl()) - - def execute(self, **kwargs) -> Any: - raise NotImplementedError( - "Cannot execute Sagemaker Builtin Algorithms locally, for local testing, please mock!" - ) - - @classmethod - def _content_type_to_blob_format(cls, content_type: int) -> str: - """ - TODO Convert InputContentType to Enum and others - """ - if content_type == _training_job_models.InputContentType.TEXT_CSV: - return "csv" - else: - raise ValueError("Unsupported InputContentType: {}".format(content_type)) - - -class SagemakerCustomTrainingTask(PythonFunctionTask[SagemakerTrainingJobConfig]): - """ - Allows a custom training algorithm to be executed on Amazon Sagemaker. For this to work, make sure your container - is built according to Flyte plugin documentation (TODO point the link here) - """ - - _SAGEMAKER_CUSTOM_TRAINING_JOB_TASK = "sagemaker_custom_training_job_task" - - def __init__( - self, - task_config: SagemakerTrainingJobConfig, - task_function: Callable, - **kwargs, - ): - super().__init__( - task_config=task_config, - task_function=task_function, - task_type=self._SAGEMAKER_CUSTOM_TRAINING_JOB_TASK, - **kwargs, - ) - - def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: - training_job = _training_job_models.TrainingJob( - algorithm_specification=self.task_config.algorithm_specification, - training_job_resource_config=self.task_config.training_job_resource_config, - ) - return MessageToDict(training_job.to_flyte_idl()) - - def _is_distributed(self) -> bool: - """ - Only if more than one instance is specified, we assume it is a distributed training setup - """ - return ( - self.task_config.training_job_resource_config - and self.task_config.training_job_resource_config.instance_count > 1 - ) - - def pre_execute(self, user_params: ExecutionParameters) -> ExecutionParameters: - """ - Pre-execute for Sagemaker will automatically add the distributed context to the execution params, only - if the number of execution instances is > 1. Otherwise this is considered to be a single node execution - """ - if self._is_distributed(): - logger.info("Distributed context detected!") - exec_state = FlyteContextManager.current_context().execution_state - if exec_state and exec_state.mode == ExecutionState.Mode.TASK_EXECUTION: - """ - This mode indicates we are actually in a remote execute environment (within sagemaker in this case) - """ - dist_ctx = DistributedTrainingContext.from_env() - else: - dist_ctx = DistributedTrainingContext.local_execute() - return user_params.builder().add_attr("DISTRIBUTED_TRAINING_CONTEXT", dist_ctx).build() - - return user_params - - def post_execute(self, user_params: ExecutionParameters, rval: Any) -> Any: - """ - In the case of distributed execution, we check the should_persist_predicate in the configuration to determine - if the output should be persisted. This is because in distributed training, multiple nodes may produce partial - outputs and only the user process knows the output that should be generated. They can control the choice using - the predicate. - - To control if output is generated across every execution, we override the post_execute method and sometimes - return a None - """ - if self._is_distributed(): - logger.info("Distributed context detected!") - dctx = flytekit.current_context().distributed_training_context - if not self.task_config.should_persist_output(dctx): - logger.info("output persistence predicate not met, Flytekit will ignore outputs") - raise IgnoreOutputs(f"Distributed context - Persistence predicate not met. Ignoring outputs - {dctx}") - return rval - - -# Register the Tensorflow Plugin into the flytekit core plugin system -TaskPlugins.register_pythontask_plugin(SagemakerTrainingJobConfig, SagemakerCustomTrainingTask) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py new file mode 100644 index 0000000000..e907455182 --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py @@ -0,0 +1,36 @@ +""" +.. currentmodule:: flytekitplugins.awssagemaker_inference + +.. autosummary:: + :template: custom.rst + :toctree: generated/ + + BotoAgent + BotoTask + SageMakerModelTask + SageMakerEndpointConfigTask + SageMakerEndpointAgent + SageMakerEndpointTask + SageMakerDeleteEndpointConfigTask + SageMakerDeleteEndpointTask + SageMakerDeleteModelTask + SageMakerInvokeEndpointTask + create_sagemaker_deployment + delete_sagemaker_deployment +""" + +from .agent import SageMakerEndpointAgent +from .boto3_agent import BotoAgent +from .boto3_task import BotoConfig, BotoTask +from .task import ( + SageMakerDeleteEndpointConfigTask, + SageMakerDeleteEndpointTask, + SageMakerDeleteModelTask, + SageMakerEndpointConfigTask, + SageMakerEndpointTask, + SageMakerInvokeEndpointTask, + SageMakerModelTask, +) +from .workflow import create_sagemaker_deployment, delete_sagemaker_deployment + +triton_image_uri = "{account_id}.dkr.ecr.{region}.{base}/sagemaker-tritonserver:21.08-py3" diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py new file mode 100644 index 0000000000..ecd283fed8 --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py @@ -0,0 +1,99 @@ +import json +from datetime import datetime +from typing import Optional + +from flytekit.extend.backend.base_agent import ( + AgentRegistry, + AsyncAgentBase, + Resource, +) +from flytekit.extend.backend.utils import convert_to_flyte_phase, get_agent_secret +from flytekit.models.literals import LiteralMap +from flytekit.models.task import TaskTemplate + +from .boto3_mixin import Boto3AgentMixin +from .task import SageMakerEndpointMetadata + +states = { + "Creating": "Running", + "InService": "Success", + "Failed": "Failed", +} + + +class DateTimeEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, datetime): + return o.isoformat() + + return json.JSONEncoder.default(self, o) + + +class SageMakerEndpointAgent(Boto3AgentMixin, AsyncAgentBase): + """This agent creates an endpoint.""" + + name = "SageMaker Endpoint Agent" + + def __init__(self): + super().__init__( + service="sagemaker", + task_type_name="sagemaker-endpoint", + metadata_type=SageMakerEndpointMetadata, + ) + + async def create( + self, task_template: TaskTemplate, inputs: Optional[LiteralMap] = None, **kwargs + ) -> SageMakerEndpointMetadata: + custom = task_template.custom + config = custom.get("config") + region = custom.get("region") + + await self._call( + method="create_endpoint", + config=config, + inputs=inputs, + region=region, + aws_access_key_id=get_agent_secret(secret_key="aws-access-key"), + aws_secret_access_key=get_agent_secret(secret_key="aws-secret-access-key"), + aws_session_token=get_agent_secret(secret_key="aws-session-token"), + ) + + return SageMakerEndpointMetadata(config=config, region=region, inputs=inputs) + + async def get(self, resource_meta: SageMakerEndpointMetadata, **kwargs) -> Resource: + endpoint_status = await self._call( + method="describe_endpoint", + config={"EndpointName": resource_meta.config.get("EndpointName")}, + inputs=resource_meta.inputs, + region=resource_meta.region, + aws_access_key_id=get_agent_secret(secret_key="aws-access-key"), + aws_secret_access_key=get_agent_secret(secret_key="aws-secret-access-key"), + aws_session_token=get_agent_secret(secret_key="aws-session-token"), + ) + + current_state = endpoint_status.get("EndpointStatus") + flyte_phase = convert_to_flyte_phase(states[current_state]) + + message = None + if current_state == "Failed": + message = endpoint_status.get("FailureReason") + + res = None + if current_state == "InService": + res = {"result": json.dumps(endpoint_status, cls=DateTimeEncoder)} + + return Resource(phase=flyte_phase, outputs=res, message=message) + + async def delete(self, resource_meta: SageMakerEndpointMetadata, **kwargs): + await self._call( + "delete_endpoint", + config={"EndpointName": resource_meta.config.get("EndpointName")}, + region=resource_meta.region, + inputs=resource_meta.inputs, + aws_access_key_id=get_agent_secret(secret_key="aws-access-key"), + aws_secret_access_key=get_agent_secret(secret_key="aws-secret-access-key"), + aws_session_token=get_agent_secret(secret_key="aws-session-token"), + ) + + +AgentRegistry.register(SageMakerEndpointAgent()) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_agent.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_agent.py new file mode 100644 index 0000000000..adb1772248 --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_agent.py @@ -0,0 +1,68 @@ +from typing import Optional + +from flyteidl.core.execution_pb2 import TaskExecution + +from flytekit.extend.backend.base_agent import ( + AgentRegistry, + Resource, + SyncAgentBase, +) +from flytekit.extend.backend.utils import get_agent_secret +from flytekit.models.literals import LiteralMap +from flytekit.models.task import TaskTemplate + +from .boto3_mixin import Boto3AgentMixin + + +# https://github.com/flyteorg/flyte/issues/4505 +def convert_floats_with_no_fraction_to_ints(data): + if isinstance(data, dict): + for key, value in data.items(): + data[key] = convert_floats_with_no_fraction_to_ints(value) + elif isinstance(data, list): + for i, item in enumerate(data): + data[i] = convert_floats_with_no_fraction_to_ints(item) + elif isinstance(data, float) and data.is_integer(): + return int(data) + return data + + +class BotoAgent(SyncAgentBase): + """A general purpose boto3 agent that can be used to call any boto3 method.""" + + name = "Boto Agent" + + def __init__(self): + super().__init__(task_type_name="boto") + + async def do(self, task_template: TaskTemplate, inputs: Optional[LiteralMap] = None, **kwargs) -> Resource: + custom = task_template.custom + + service = custom.get("service") + raw_config = custom.get("config") + convert_floats_with_no_fraction_to_ints(raw_config) + config = raw_config + region = custom.get("region") + method = custom.get("method") + images = custom.get("images") + + boto3_object = Boto3AgentMixin(service=service, region=region) + + result = await boto3_object._call( + method=method, + config=config, + images=images, + inputs=inputs, + aws_access_key_id=get_agent_secret(secret_key="aws-access-key"), + aws_secret_access_key=get_agent_secret(secret_key="aws-secret-access-key"), + aws_session_token=get_agent_secret(secret_key="aws-session-token"), + ) + + outputs = None + if result: + outputs = {"result": result} + + return Resource(phase=TaskExecution.SUCCEEDED, outputs=outputs) + + +AgentRegistry.register(BotoAgent()) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py new file mode 100644 index 0000000000..045124afd0 --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py @@ -0,0 +1,185 @@ +from typing import Any, Dict, Optional + +import aioboto3 + +from flytekit.interaction.string_literals import literal_map_string_repr +from flytekit.models.literals import LiteralMap + +account_id_map = { + "us-east-1": "785573368785", + "us-east-2": "007439368137", + "us-west-1": "710691900526", + "us-west-2": "301217895009", + "eu-west-1": "802834080501", + "eu-west-2": "205493899709", + "eu-west-3": "254080097072", + "eu-north-1": "601324751636", + "eu-south-1": "966458181534", + "eu-central-1": "746233611703", + "ap-east-1": "110948597952", + "ap-south-1": "763008648453", + "ap-northeast-1": "941853720454", + "ap-northeast-2": "151534178276", + "ap-southeast-1": "324986816169", + "ap-southeast-2": "355873309152", + "cn-northwest-1": "474822919863", + "cn-north-1": "472730292857", + "sa-east-1": "756306329178", + "ca-central-1": "464438896020", + "me-south-1": "836785723513", + "af-south-1": "774647643957", +} + + +def update_dict_fn(original_dict: Any, update_dict: Dict[str, Any]) -> Any: + """ + Recursively update a dictionary with values from another dictionary. + For example, if original_dict is {"EndpointConfigName": "{endpoint_config_name}"}, + and update_dict is {"endpoint_config_name": "my-endpoint-config"}, + then the result will be {"EndpointConfigName": "my-endpoint-config"}. + + :param original_dict: The dictionary to update (in place) + :param update_dict: The dictionary to use for updating + :return: The updated dictionary + """ + if original_dict is None: + return None + + # If the original value is a string and contains placeholder curly braces + if isinstance(original_dict, str): + if "{" in original_dict and "}" in original_dict: + # Check if there are nested keys + if "." in original_dict: + # Create a copy of update_dict + update_dict_copy = update_dict.copy() + + # Fetch keys from the original_dict + keys = original_dict.strip("{}").split(".") + + # Get value from the nested dictionary + for key in keys: + try: + update_dict_copy = update_dict_copy[key] + except Exception: + raise ValueError(f"Could not find the key {key} in {update_dict_copy}.") + + return update_dict_copy + + # Retrieve the original value using the key without curly braces + original_value = update_dict.get(original_dict.strip("{}")) + + # Check if original_value exists; if so, return it, + # otherwise, raise a ValueError indicating that the value for the key original_dict could not be found. + if original_value: + return original_value + else: + raise ValueError(f"Could not find value for {original_dict}.") + + # If the string does not contain placeholders, return it as is + return original_dict + + # If the original value is a list, recursively update each element in the list + if isinstance(original_dict, list): + return [update_dict_fn(item, update_dict) for item in original_dict] + + # If the original value is a dictionary, recursively update each key-value pair + if isinstance(original_dict, dict): + for key, value in original_dict.items(): + original_dict[key] = update_dict_fn(value, update_dict) + + # Return the updated original dict + return original_dict + + +class Boto3AgentMixin: + """ + This mixin facilitates the creation of a Boto3 agent for any AWS service. + It provides a single method, `_call`, which can be employed to invoke any Boto3 method. + """ + + def __init__(self, *, service: str, region: Optional[str] = None, **kwargs): + """ + Initialize the Boto3AgentMixin. + + :param service: The AWS service to use, e.g., sagemaker. + :param region: The region for the boto3 client; can be overridden when calling boto3 methods. + """ + self._service = service + self._region = region + + super().__init__(**kwargs) + + async def _call( + self, + method: str, + config: Dict[str, Any], + images: Optional[Dict[str, str]] = None, + inputs: Optional[LiteralMap] = None, + region: Optional[str] = None, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_session_token: Optional[str] = None, + ) -> Any: + """ + Utilize this method to invoke any boto3 method (AWS service method). + + :param method: The boto3 method to invoke, e.g., create_endpoint_config. + :param config: The configuration for the method, e.g., {"EndpointConfigName": "my-endpoint-config"}. The config + may contain placeholders replaced by values from inputs. + For example, if the config is + {"EndpointConfigName": "{inputs.endpoint_config_name}", "EndpointName": "{inputs.endpoint_name}", + "Image": "{images.primary_container_image}"}, + the inputs contain a string literal for endpoint_config_name and endpoint_name and images contain primary_container_image, + then the config will be updated to {"EndpointConfigName": "my-endpoint-config", "EndpointName": "my-endpoint", + "Image": "my-image"} before invoking the boto3 method. + :param images: A dict of Docker images to use, for example, when deploying a model on SageMaker. + :param inputs: The inputs for the task being created. + :param region: The region for the boto3 client. If not provided, the region specified in the constructor will be used. + :param aws_access_key_id: The access key ID to use to access the AWS resources. + :param aws_secret_access_key: The secret access key to use to access the AWS resources + :param aws_session_token: An AWS session token used as part of the credentials to authenticate the user. + """ + args = {} + input_region = None + + if inputs: + args["inputs"] = literal_map_string_repr(inputs) + input_region = args["inputs"].get("region") + + final_region = input_region or region or self._region + if not final_region: + raise ValueError("Region parameter is required.") + + if images: + base = "amazonaws.com.cn" if final_region.startswith("cn-") else "amazonaws.com" + images = { + image_name: ( + image.format( + account_id=account_id_map[final_region], + region=final_region, + base=base, + ) + if isinstance(image, str) and "{region}" in image + else image + ) + for image_name, image in images.items() + } + args["images"] = images + + updated_config = update_dict_fn(config, args) + + # Asynchronous Boto3 session + session = aioboto3.Session() + async with session.client( + service_name=self._service, + region_name=final_region, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_session_token=aws_session_token, + ) as client: + try: + result = await getattr(client, method)(**updated_config) + except Exception as e: + raise e + + return result diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_task.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_task.py new file mode 100644 index 0000000000..2e7c8f5b7b --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_task.py @@ -0,0 +1,56 @@ +from dataclasses import dataclass +from typing import Any, Dict, Optional, Type, Union + +from flytekit import ImageSpec, kwtypes +from flytekit.configuration import SerializationSettings +from flytekit.core.base_task import PythonTask +from flytekit.core.interface import Interface +from flytekit.extend.backend.base_agent import SyncAgentExecutorMixin +from flytekit.image_spec.image_spec import ImageBuildEngine + + +@dataclass +class BotoConfig(object): + service: str + method: str + config: Dict[str, Any] + region: Optional[str] = None + images: Optional[Dict[str, Union[str, ImageSpec]]] = None + + +class BotoTask(SyncAgentExecutorMixin, PythonTask[BotoConfig]): + _TASK_TYPE = "boto" + + def __init__( + self, + name: str, + task_config: BotoConfig, + inputs: Optional[Dict[str, Type]] = None, + **kwargs, + ): + super().__init__( + name=name, + task_config=task_config, + task_type=self._TASK_TYPE, + interface=Interface(inputs=inputs, outputs=kwtypes(result=dict)), + **kwargs, + ) + + def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: + images = self.task_config.images + if images is not None: + for key, image in images.items(): + if isinstance(image, ImageSpec): + # Build the image + ImageBuildEngine.build(image) + + # Replace the value in the dictionary with image.image_name() + images[key] = image.image_name() + + return { + "service": self.task_config.service, + "config": self.task_config.config, + "region": self.task_config.region, + "method": self.task_config.method, + "images": images, + } diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/task.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/task.py new file mode 100644 index 0000000000..4ed538a410 --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/task.py @@ -0,0 +1,236 @@ +from dataclasses import dataclass +from typing import Any, Dict, Optional, Type, Union + +from flytekit import ImageSpec, kwtypes +from flytekit.configuration import SerializationSettings +from flytekit.core.base_task import PythonTask +from flytekit.core.interface import Interface +from flytekit.extend.backend.base_agent import AsyncAgentExecutorMixin +from flytekit.models.literals import LiteralMap + +from .boto3_task import BotoConfig, BotoTask + + +class SageMakerModelTask(BotoTask): + def __init__( + self, + name: str, + config: Dict[str, Any], + region: Optional[str] = None, + images: Optional[Dict[str, Union[str, ImageSpec]]] = None, + inputs: Optional[Dict[str, Type]] = None, + **kwargs, + ): + """ + Creates a SageMaker model. + + :param name: The name of the task. + :param config: The configuration to be provided to the boto3 API call. + :param region: The region for the boto3 client. + :param images: Images for SageMaker model creation. + :param inputs: The input literal map to be used for updating the configuration. + """ + + super(SageMakerModelTask, self).__init__( + name=name, + task_config=BotoConfig( + service="sagemaker", + method="create_model", + config=config, + region=region, + images=images, + ), + inputs=inputs, + **kwargs, + ) + + +class SageMakerEndpointConfigTask(BotoTask): + def __init__( + self, + name: str, + config: Dict[str, Any], + region: Optional[str] = None, + inputs: Optional[Dict[str, Type]] = None, + **kwargs, + ): + """ + Creates a SageMaker endpoint configuration. + + :param name: The name of the task. + :param config: The configuration to be provided to the boto3 API call. + :param region: The region for the boto3 client. + :param inputs: The input literal map to be used for updating the configuration. + """ + super(SageMakerEndpointConfigTask, self).__init__( + name=name, + task_config=BotoConfig( + service="sagemaker", + method="create_endpoint_config", + config=config, + region=region, + ), + inputs=inputs, + **kwargs, + ) + + +@dataclass +class SageMakerEndpointMetadata(object): + config: Dict[str, Any] + region: Optional[str] = None + inputs: Optional[LiteralMap] = None + + +class SageMakerEndpointTask(AsyncAgentExecutorMixin, PythonTask[SageMakerEndpointMetadata]): + _TASK_TYPE = "sagemaker-endpoint" + + def __init__( + self, + name: str, + config: Dict[str, Any], + region: Optional[str] = None, + inputs: Optional[Dict[str, Type]] = None, + **kwargs, + ): + """ + Creates a SageMaker endpoint. + + :param name: The name of the task. + :param config: The configuration to be provided to the boto3 API call. + :param region: The region for the boto3 client. + :param inputs: The input literal map to be used for updating the configuration. + """ + super().__init__( + name=name, + task_config=SageMakerEndpointMetadata( + config=config, + region=region, + ), + task_type=self._TASK_TYPE, + interface=Interface(inputs=inputs, outputs=kwtypes(result=str)), + **kwargs, + ) + + def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: + return {"config": self.task_config.config, "region": self.task_config.region} + + +class SageMakerDeleteEndpointTask(BotoTask): + def __init__( + self, + name: str, + config: Dict[str, Any], + region: Optional[str] = None, + inputs: Optional[Dict[str, Type]] = None, + **kwargs, + ): + """ + Deletes a SageMaker endpoint. + + :param name: The name of the task. + :param config: The configuration to be provided to the boto3 API call. + :param region: The region for the boto3 client. + :param inputs: The input literal map to be used for updating the configuration. + """ + super(SageMakerDeleteEndpointTask, self).__init__( + name=name, + task_config=BotoConfig( + service="sagemaker", + method="delete_endpoint", + config=config, + region=region, + ), + inputs=inputs, + **kwargs, + ) + + +class SageMakerDeleteEndpointConfigTask(BotoTask): + def __init__( + self, + name: str, + config: Dict[str, Any], + region: Optional[str] = None, + inputs: Optional[Dict[str, Type]] = None, + **kwargs, + ): + """ + Deletes a SageMaker endpoint config. + + :param name: The name of the task. + :param config: The configuration to be provided to the boto3 API call. + :param region: The region for the boto3 client. + :param inputs: The input literal map to be used for updating the configuration. + """ + super(SageMakerDeleteEndpointConfigTask, self).__init__( + name=name, + task_config=BotoConfig( + service="sagemaker", + method="delete_endpoint_config", + config=config, + region=region, + ), + inputs=inputs, + **kwargs, + ) + + +class SageMakerDeleteModelTask(BotoTask): + def __init__( + self, + name: str, + config: Dict[str, Any], + region: Optional[str] = None, + inputs: Optional[Dict[str, Type]] = None, + **kwargs, + ): + """ + Deletes a SageMaker model. + + :param name: The name of the task. + :param config: The configuration to be provided to the boto3 API call. + :param region: The region for the boto3 client. + :param inputs: The input literal map to be used for updating the configuration. + """ + super(SageMakerDeleteModelTask, self).__init__( + name=name, + task_config=BotoConfig( + service="sagemaker", + method="delete_model", + config=config, + region=region, + ), + inputs=inputs, + **kwargs, + ) + + +class SageMakerInvokeEndpointTask(BotoTask): + def __init__( + self, + name: str, + config: Dict[str, Any], + region: Optional[str] = None, + inputs: Optional[Dict[str, Type]] = None, + **kwargs, + ): + """ + Invokes a SageMaker endpoint. + + :param name: The name of the task. + :param config: The configuration to be provided to the boto3 API call. + :param region: The region for the boto3 client. + :param inputs: The input literal map to be used for updating the configuration. + """ + super(SageMakerInvokeEndpointTask, self).__init__( + name=name, + task_config=BotoConfig( + service="sagemaker-runtime", + method="invoke_endpoint_async", + config=config, + region=region, + ), + inputs=inputs, + **kwargs, + ) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/workflow.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/workflow.py new file mode 100644 index 0000000000..87a27c7497 --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/workflow.py @@ -0,0 +1,182 @@ +from typing import Any, Dict, Optional, Tuple, Type + +from flytekit import Workflow, kwtypes + +from .task import ( + SageMakerDeleteEndpointConfigTask, + SageMakerDeleteEndpointTask, + SageMakerDeleteModelTask, + SageMakerEndpointConfigTask, + SageMakerEndpointTask, + SageMakerModelTask, +) + + +def create_deployment_task( + name: str, + task_type: Any, + config: Dict[str, Any], + region: str, + inputs: Optional[Dict[str, Type]], + images: Optional[Dict[str, Any]], + region_at_runtime: bool, +) -> Tuple[Any, Optional[Dict[str, Type]]]: + if region_at_runtime: + if inputs: + inputs.update({"region": str}) + else: + inputs = kwtypes(region=str) + return ( + task_type(name=name, config=config, region=region, inputs=inputs, images=images), + inputs, + ) + + +def create_sagemaker_deployment( + name: str, + model_config: Dict[str, Any], + endpoint_config_config: Dict[str, Any], + endpoint_config: Dict[str, Any], + images: Optional[Dict[str, Any]] = None, + model_input_types: Optional[Dict[str, Type]] = None, + endpoint_config_input_types: Optional[Dict[str, Type]] = None, + endpoint_input_types: Optional[Dict[str, Type]] = None, + region: Optional[str] = None, + region_at_runtime: bool = False, +) -> Workflow: + """ + Creates SageMaker model, endpoint config and endpoint. + + :param model_config: Configuration for the SageMaker model creation API call. + :param endpoint_config_config: Configuration for the SageMaker endpoint configuration creation API call. + :param endpoint_config: Configuration for the SageMaker endpoint creation API call. + :param images: A dictionary of images for SageMaker model creation. + :param model_input_types: Mapping of SageMaker model configuration inputs to their types. + :param endpoint_config_input_types: Mapping of SageMaker endpoint configuration inputs to their types. + :param endpoint_input_types: Mapping of SageMaker endpoint inputs to their types. + :param region: The region for SageMaker API calls. + :param region_at_runtime: Set this to True if you want to provide the region at runtime. + """ + if not any((region, region_at_runtime)): + raise ValueError("Region parameter is required.") + + wf = Workflow(name=f"sagemaker-deploy-{name}") + + if region_at_runtime: + wf.add_workflow_input("region", str) + + inputs = { + SageMakerModelTask: { + "input_types": model_input_types, + "name": "sagemaker-model", + "images": True, + "config": model_config, + }, + SageMakerEndpointConfigTask: { + "input_types": endpoint_config_input_types, + "name": "sagemaker-endpoint-config", + "images": False, + "config": endpoint_config_config, + }, + SageMakerEndpointTask: { + "input_types": endpoint_input_types, + "name": "sagemaker-endpoint", + "images": False, + "config": endpoint_config, + }, + } + + nodes = [] + for key, value in inputs.items(): + input_types = value["input_types"] + obj, new_input_types = create_deployment_task( + name=f"{value['name']}-{name}", + task_type=key, + config=value["config"], + region=region, + inputs=input_types, + images=images if value["images"] else None, + region_at_runtime=region_at_runtime, + ) + input_dict = {} + if isinstance(new_input_types, dict): + for param, t in new_input_types.items(): + # Handles the scenario when the same input is present during different API calls. + if param not in wf.inputs.keys(): + wf.add_workflow_input(param, t) + input_dict[param] = wf.inputs[param] + node = wf.add_entity(obj, **input_dict) + if len(nodes) > 0: + nodes[-1] >> node + nodes.append(node) + + wf.add_workflow_output("wf_output", nodes[2].outputs["result"], str) + return wf + + +def create_delete_task( + name: str, + task_type: Any, + config: Dict[str, Any], + region: str, + value: str, + region_at_runtime: bool, +) -> Any: + return task_type( + name=name, + config=config, + region=region, + inputs=(kwtypes(**{value: str, "region": str}) if region_at_runtime else kwtypes(**{value: str})), + ) + + +def delete_sagemaker_deployment(name: str, region: Optional[str] = None, region_at_runtime: bool = False) -> Workflow: + """ + Deletes SageMaker model, endpoint config and endpoint. + + :param name: The prefix to be added to the task names. + :param region: The region to use for SageMaker API calls. + :param region_at_runtime: Set this to True if you want to provide the region at runtime. + """ + if not any((region, region_at_runtime)): + raise ValueError("Region parameter is required.") + + wf = Workflow(name=f"sagemaker-delete-deployment-{name}") + + if region_at_runtime: + wf.add_workflow_input("region", str) + + inputs = { + SageMakerDeleteEndpointTask: "endpoint_name", + SageMakerDeleteEndpointConfigTask: "endpoint_config_name", + SageMakerDeleteModelTask: "model_name", + } + + nodes = [] + for key, value in inputs.items(): + obj = create_delete_task( + name=f"sagemaker-delete-{value.replace('_name', '').replace('_', '-')}-{name}", + task_type=key, + config={value.title().replace("_", ""): f"{{inputs.{value}}}"}, + region=region, + value=value, + region_at_runtime=region_at_runtime, + ) + + wf.add_workflow_input(value, str) + node = wf.add_entity( + obj, + **( + { + value: wf.inputs[value], + "region": wf.inputs["region"], + } + if region_at_runtime + else {value: wf.inputs[value]} + ), + ) + if len(nodes) > 0: + nodes[-1] >> node + nodes.append(node) + + return wf diff --git a/plugins/flytekit-aws-sagemaker/scripts/flytekit_sagemaker_runner.py b/plugins/flytekit-aws-sagemaker/scripts/flytekit_sagemaker_runner.py deleted file mode 100644 index 4a3d94fab5..0000000000 --- a/plugins/flytekit-aws-sagemaker/scripts/flytekit_sagemaker_runner.py +++ /dev/null @@ -1,92 +0,0 @@ -import argparse -import logging -import os -import subprocess -import sys - -FLYTE_ARG_PREFIX = "--__FLYTE" -FLYTE_ENV_VAR_PREFIX = f"{FLYTE_ARG_PREFIX}_ENV_VAR_" -FLYTE_CMD_PREFIX = f"{FLYTE_ARG_PREFIX}_CMD_" -FLYTE_ARG_SUFFIX = "__" - - -# This script is the "entrypoint" script for SageMaker. An environment variable must be set on the container (typically -# in the Dockerfile) of SAGEMAKER_PROGRAM=flytekit_sagemaker_runner.py. When the container is launched in SageMaker, -# it'll run `train flytekit_sagemaker_runner.py `, the responsibility of this script is then to decode -# the known hyperparameters (passed as command line args) to recreate the original command that will actually run the -# virtual environment and execute the intended task (e.g. `service_venv pyflyte-execute --task-module ....`) - -# An example for a valid command: -# python flytekit_sagemaker_runner.py --__FLYTE_ENV_VAR_env1__ val1 --__FLYTE_ENV_VAR_env2__ val2 -# --__FLYTE_CMD_0_service_venv__ __FLYTE_CMD_DUMMY_VALUE__ -# --__FLYTE_CMD_1_pyflyte-execute__ __FLYTE_CMD_DUMMY_VALUE__ -# --__FLYTE_CMD_2_--task-module__ __FLYTE_CMD_DUMMY_VALUE__ -# --__FLYTE_CMD_3_blah__ __FLYTE_CMD_DUMMY_VALUE__ -# --__FLYTE_CMD_4_--task-name__ __FLYTE_CMD_DUMMY_VALUE__ -# --__FLYTE_CMD_5_bloh__ __FLYTE_CMD_DUMMY_VALUE__ -# --__FLYTE_CMD_6_--output-prefix__ __FLYTE_CMD_DUMMY_VALUE__ -# --__FLYTE_CMD_7_s3://fake-bucket__ __FLYTE_CMD_DUMMY_VALUE__ -# --__FLYTE_CMD_8_--inputs__ __FLYTE_CMD_DUMMY_VALUE__ -# --__FLYTE_CMD_9_s3://fake-bucket__ __FLYTE_CMD_DUMMY_VALUE__ - - -def parse_args(cli_args): - parser = argparse.ArgumentParser(description="Running sagemaker task") - args, unknowns = parser.parse_known_args(cli_args) - - # Parse the command line and env vars - flyte_cmd = [] - env_vars = {} - i = 0 - - while i < len(unknowns): - unknown = unknowns[i] - logging.info(f"Processing argument {unknown}") - if unknown.startswith(FLYTE_CMD_PREFIX) and unknown.endswith(FLYTE_ARG_SUFFIX): - processed = unknown[len(FLYTE_CMD_PREFIX) :][: -len(FLYTE_ARG_SUFFIX)] - # Parse the format `1_--task-module` - parts = processed.split("_", maxsplit=1) - flyte_cmd.append((parts[0], parts[1])) - i += 1 - elif unknown.startswith(FLYTE_ENV_VAR_PREFIX) and unknown.endswith(FLYTE_ARG_SUFFIX): - processed = unknown[len(FLYTE_ENV_VAR_PREFIX) :][: -len(FLYTE_ARG_SUFFIX)] - i += 1 - if unknowns[i].startswith(FLYTE_ARG_PREFIX) is False: - env_vars[processed] = unknowns[i] - i += 1 - else: - # To prevent SageMaker from ignoring our __FLYTE_CMD_*__ hyperparameters, we need to set a dummy value - # which serves as a placeholder for each of them. The dummy value placeholder `__FLYTE_CMD_DUMMY_VALUE__` - # falls into this branch and will be ignored - i += 1 - - return flyte_cmd, env_vars - - -def sort_flyte_cmd(flyte_cmd): - # Order the cmd using the index (the first element in each tuple) - flyte_cmd = sorted(flyte_cmd, key=lambda x: int(x[0])) - flyte_cmd = [x[1] for x in flyte_cmd] - return flyte_cmd - - -def set_env_vars(env_vars): - for key, val in env_vars.items(): - os.environ[key] = val - - -def run(cli_args): - flyte_cmd, env_vars = parse_args(cli_args) - flyte_cmd = sort_flyte_cmd(flyte_cmd) - set_env_vars(env_vars) - - logging.info(f"Cmd:{flyte_cmd}") - logging.info(f"Env vars:{env_vars}") - - # Launching a subprocess with the selected entrypoint script and the rest of the arguments - logging.info(f"Launching command: {flyte_cmd}") - subprocess.run(flyte_cmd, stdout=sys.stdout, stderr=sys.stderr, encoding="utf-8", check=True) - - -if __name__ == "__main__": - run(sys.argv) diff --git a/plugins/flytekit-aws-sagemaker/setup.py b/plugins/flytekit-aws-sagemaker/setup.py index 855dd32402..cdc4b816b6 100644 --- a/plugins/flytekit-aws-sagemaker/setup.py +++ b/plugins/flytekit-aws-sagemaker/setup.py @@ -1,37 +1,37 @@ from setuptools import setup PLUGIN_NAME = "awssagemaker" +INFERENCE_PACKAGE = "awssagemaker_inference" microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>=1.3.0b2,<2.0.0", "sagemaker-training>=3.6.2,<4.0.0", "retry2==0.9.5"] +plugin_requires = ["flytekit>=1.11.0", "aioboto3>=12.3.0"] __version__ = "0.0.0+develop" -# TODO: move sagemaker install script here. setup( name=microlib_name, version=__version__, author="flyteorg", author_email="admin@flyte.org", - description="AWS Plugins for flytekit", + description="Flytekit AWS SageMaker Plugin", namespace_packages=["flytekitplugins"], - packages=[f"flytekitplugins.{PLUGIN_NAME}", f"flytekitplugins.{PLUGIN_NAME}.models"], + packages=[f"flytekitplugins.{INFERENCE_PACKAGE}"], install_requires=plugin_requires, license="apache2", - python_requires=">=3.8", + python_requires=">=3.10", classifiers=[ "Intended Audience :: Science/Research", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Libraries :: Python Modules", ], - scripts=["scripts/flytekit_sagemaker_runner.py"], + entry_points={"flytekit.plugins": [f"{INFERENCE_PACKAGE}=flytekitplugins.{INFERENCE_PACKAGE}"]}, ) diff --git a/plugins/flytekit-aws-sagemaker/tests/test_boto3_agent.py b/plugins/flytekit-aws-sagemaker/tests/test_boto3_agent.py new file mode 100644 index 0000000000..7be62e216c --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/tests/test_boto3_agent.py @@ -0,0 +1,100 @@ +from datetime import timedelta +from unittest import mock + +import pytest +from flyteidl.core.execution_pb2 import TaskExecution + +from flytekit.extend.backend.base_agent import AgentRegistry +from flytekit.interfaces.cli_identifiers import Identifier +from flytekit.models import literals +from flytekit.models.core.identifier import ResourceType +from flytekit.models.task import RuntimeMetadata, TaskMetadata, TaskTemplate + + +@pytest.mark.asyncio +@mock.patch( + "flytekitplugins.awssagemaker_inference.boto3_agent.get_agent_secret", + return_value="mocked_secret", +) +@mock.patch( + "flytekitplugins.awssagemaker_inference.boto3_agent.Boto3AgentMixin._call", + return_value={ + "ResponseMetadata": { + "RequestId": "66f80391-348a-4ee0-9158-508914d16db2", + "HTTPStatusCode": 200.0, + "RetryAttempts": 0.0, + "HTTPHeaders": { + "content-type": "application/x-amz-json-1.1", + "date": "Wed, 31 Jan 2024 16:43:52 GMT", + "x-amzn-requestid": "66f80391-348a-4ee0-9158-508914d16db2", + "content-length": "114", + }, + }, + "EndpointConfigArn": "arn:aws:sagemaker:us-east-2:000000000:endpoint-config/sagemaker-xgboost-endpoint-config", + }, +) +async def test_agent(mock_boto_call, mock_secret): + agent = AgentRegistry.get_agent("boto") + task_id = Identifier( + resource_type=ResourceType.TASK, + project="project", + domain="domain", + name="name", + version="version", + ) + task_config = { + "service": "sagemaker", + "config": { + "EndpointConfigName": "endpoint-config-name", + "ProductionVariants": [ + { + "VariantName": "variant-name-1", + "ModelName": "{inputs.model_name}", + "InitialInstanceCount": 1, + "InstanceType": "ml.m4.xlarge", + }, + ], + "AsyncInferenceConfig": {"OutputConfig": {"S3OutputPath": "{inputs.s3_output_path}"}}, + }, + "region": "us-east-2", + "method": "create_endpoint_config", + "images": None, + } + task_metadata = TaskMetadata( + discoverable=True, + runtime=RuntimeMetadata(RuntimeMetadata.RuntimeType.FLYTE_SDK, "1.0.0", "python"), + timeout=timedelta(days=1), + retries=literals.RetryStrategy(3), + interruptible=True, + discovery_version="0.1.1b0", + deprecated_error_message="This is deprecated!", + cache_serializable=True, + pod_template_name="A", + cache_ignore_input_vars=(), + ) + + task_template = TaskTemplate( + id=task_id, + custom=task_config, + metadata=task_metadata, + interface=None, + type="boto", + ) + task_inputs = literals.LiteralMap( + { + "model_name": literals.Literal( + scalar=literals.Scalar(primitive=literals.Primitive(string_value="sagemaker-model")) + ), + "s3_output_path": literals.Literal( + scalar=literals.Scalar(primitive=literals.Primitive(string_value="s3-output-path")) + ), + }, + ) + + resource = await agent.do(task_template, task_inputs) + + assert resource.phase == TaskExecution.SUCCEEDED + assert ( + resource.outputs["result"]["EndpointConfigArn"] + == "arn:aws:sagemaker:us-east-2:000000000:endpoint-config/sagemaker-xgboost-endpoint-config" + ) diff --git a/plugins/flytekit-aws-sagemaker/tests/test_boto3_mixin.py b/plugins/flytekit-aws-sagemaker/tests/test_boto3_mixin.py new file mode 100644 index 0000000000..98c5686e2d --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/tests/test_boto3_mixin.py @@ -0,0 +1,119 @@ +import typing +from unittest.mock import AsyncMock, patch + +import pytest +from flytekitplugins.awssagemaker_inference import triton_image_uri +from flytekitplugins.awssagemaker_inference.boto3_mixin import ( + Boto3AgentMixin, + update_dict_fn, +) + +from flytekit import FlyteContext, StructuredDataset +from flytekit.core.type_engine import TypeEngine +from flytekit.interaction.string_literals import literal_map_string_repr +from flytekit.types.file import FlyteFile + + +def test_inputs(): + original_dict = { + "a": "{inputs.a}", + "b": "{inputs.b}", + "c": "{inputs.c}", + "d": "{inputs.d}", + "e": "{inputs.e}", + "f": "{inputs.f}", + "j": {"g": "{inputs.g}", "h": "{inputs.h}", "i": "{inputs.i}"}, + } + inputs = TypeEngine.dict_to_literal_map( + FlyteContext.current_context(), + { + "a": 1, + "b": "hello", + "c": True, + "d": 1.0, + "e": [1, 2, 3], + "f": {"a": "b"}, + "g": None, + "h": FlyteFile("s3://foo/bar", remote_path=False), + "i": StructuredDataset(uri="s3://foo/bar"), + }, + { + "a": int, + "b": str, + "c": bool, + "d": float, + "e": typing.List[int], + "f": typing.Dict[str, str], + "g": typing.Optional[str], + "h": FlyteFile, + "i": StructuredDataset, + }, + ) + + result = update_dict_fn( + original_dict=original_dict, + update_dict={"inputs": literal_map_string_repr(inputs)}, + ) + + assert result == { + "a": 1, + "b": "hello", + "c": True, + "d": 1.0, + "e": [1, 2, 3], + "f": {"a": "b"}, + "j": { + "g": None, + "h": "s3://foo/bar", + "i": "s3://foo/bar", + }, + } + + +def test_container(): + original_dict = {"a": "{images.primary_container_image}"} + images = {"primary_container_image": "cr.flyte.org/flyteorg/flytekit:py3.11-1.10.3"} + + result = update_dict_fn(original_dict=original_dict, update_dict={"images": images}) + + assert result == {"a": "cr.flyte.org/flyteorg/flytekit:py3.11-1.10.3"} + + +@pytest.mark.asyncio +@patch("flytekitplugins.awssagemaker_inference.boto3_mixin.aioboto3.Session") +async def test_call(mock_session): + mixin = Boto3AgentMixin(service="sagemaker") + + mock_client = AsyncMock() + mock_session.return_value.client.return_value.__aenter__.return_value = mock_client + mock_method = mock_client.create_model + + config = { + "ModelName": "{inputs.model_name}", + "PrimaryContainer": { + "Image": "{images.image}", + "ModelDataUrl": "s3://sagemaker-agent-xgboost/model.tar.gz", + }, + } + inputs = TypeEngine.dict_to_literal_map( + FlyteContext.current_context(), + {"model_name": "xgboost", "region": "us-west-2"}, + {"model_name": str, "region": str}, + ) + + result = await mixin._call( + method="create_model", + config=config, + inputs=inputs, + images={"image": triton_image_uri}, + ) + + mock_method.assert_called_with( + ModelName="xgboost", + PrimaryContainer={ + "Image": "301217895009.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tritonserver:21.08-py3", + "ModelDataUrl": "s3://sagemaker-agent-xgboost/model.tar.gz", + }, + ) + + assert result == mock_method.return_value diff --git a/plugins/flytekit-aws-sagemaker/tests/test_boto3_task.py b/plugins/flytekit-aws-sagemaker/tests/test_boto3_task.py new file mode 100644 index 0000000000..78dce7eae3 --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/tests/test_boto3_task.py @@ -0,0 +1,52 @@ +from flytekitplugins.awssagemaker_inference import BotoConfig, BotoTask + +from flytekit import kwtypes +from flytekit.configuration import Image, ImageConfig, SerializationSettings + + +def test_boto_task_and_config(): + boto_task = BotoTask( + name="boto_task", + task_config=BotoConfig( + service="sagemaker", + method="create_model", + config={ + "ModelName": "{inputs.model_name}", + "PrimaryContainer": { + "Image": "{container.image}", + "ModelDataUrl": "{inputs.model_data_url}", + }, + "ExecutionRoleArn": "{inputs.execution_role_arn}", + }, + region="us-east-2", + ), + inputs=kwtypes(model_name=str, model_data_url=str, execution_role_arn=str), + outputs=kwtypes(result=dict), + container_image="1234567890.dkr.ecr.us-east-2.amazonaws.com/sagemaker-xgboost", + ) + + assert len(boto_task.interface.inputs) == 3 + assert len(boto_task.interface.outputs) == 1 + + default_img = Image(name="default", fqn="test", tag="tag") + serialization_settings = SerializationSettings( + project="project", + domain="domain", + version="123", + image_config=ImageConfig(default_image=default_img, images=[default_img]), + env={}, + ) + + retrieved_setttings = boto_task.get_custom(serialization_settings) + + assert retrieved_setttings["service"] == "sagemaker" + assert retrieved_setttings["config"] == { + "ModelName": "{inputs.model_name}", + "PrimaryContainer": { + "Image": "{container.image}", + "ModelDataUrl": "{inputs.model_data_url}", + }, + "ExecutionRoleArn": "{inputs.execution_role_arn}", + } + assert retrieved_setttings["region"] == "us-east-2" + assert retrieved_setttings["method"] == "create_model" diff --git a/plugins/flytekit-aws-sagemaker/tests/test_flytekit_sagemaker_running.py b/plugins/flytekit-aws-sagemaker/tests/test_flytekit_sagemaker_running.py deleted file mode 100644 index f527deb5cc..0000000000 --- a/plugins/flytekit-aws-sagemaker/tests/test_flytekit_sagemaker_running.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import sys -from unittest import mock - -from scripts.flytekit_sagemaker_runner import run as _flyte_sagemaker_run - -cmd = [] -cmd.extend(["--__FLYTE_ENV_VAR_env1__", "val1"]) -cmd.extend(["--__FLYTE_ENV_VAR_env2__", "val2"]) -cmd.extend(["--__FLYTE_CMD_0_service_venv__", "__FLYTE_CMD_DUMMY_VALUE__"]) -cmd.extend(["--__FLYTE_CMD_1_pyflyte-execute__", "__FLYTE_CMD_DUMMY_VALUE__"]) -cmd.extend(["--__FLYTE_CMD_2_--task-module__", "__FLYTE_CMD_DUMMY_VALUE__"]) -cmd.extend(["--__FLYTE_CMD_3_blah__", "__FLYTE_CMD_DUMMY_VALUE__"]) -cmd.extend(["--__FLYTE_CMD_4_--task-name__", "__FLYTE_CMD_DUMMY_VALUE__"]) -cmd.extend(["--__FLYTE_CMD_5_bloh__", "__FLYTE_CMD_DUMMY_VALUE__"]) -cmd.extend(["--__FLYTE_CMD_6_--output-prefix__", "__FLYTE_CMD_DUMMY_VALUE__"]) -cmd.extend(["--__FLYTE_CMD_7_s3://fake-bucket__", "__FLYTE_CMD_DUMMY_VALUE__"]) -cmd.extend(["--__FLYTE_CMD_8_--inputs__", "__FLYTE_CMD_DUMMY_VALUE__"]) -cmd.extend(["--__FLYTE_CMD_9_s3://fake-bucket__", "__FLYTE_CMD_DUMMY_VALUE__"]) - - -@mock.patch.dict("os.environ") -@mock.patch("subprocess.run") -def test(mock_subprocess_run): - _flyte_sagemaker_run(cmd) - assert "env1" in os.environ - assert "env2" in os.environ - assert os.environ["env1"] == "val1" - assert os.environ["env2"] == "val2" - mock_subprocess_run.assert_called_with( - "service_venv pyflyte-execute --task-module blah --task-name bloh " - "--output-prefix s3://fake-bucket --inputs s3://fake-bucket".split(), - stdout=sys.stdout, - stderr=sys.stderr, - encoding="utf-8", - check=True, - ) diff --git a/plugins/flytekit-aws-sagemaker/tests/test_hpo.py b/plugins/flytekit-aws-sagemaker/tests/test_hpo.py deleted file mode 100644 index e52994c664..0000000000 --- a/plugins/flytekit-aws-sagemaker/tests/test_hpo.py +++ /dev/null @@ -1,124 +0,0 @@ -import os -import pathlib -import tempfile - -import pytest -from flytekitplugins.awssagemaker.hpo import ( - HPOJob, - HPOTuningJobConfigTransformer, - ParameterRangesTransformer, - SagemakerHPOTask, -) -from flytekitplugins.awssagemaker.models.hpo_job import ( - HyperparameterTuningJobConfig, - HyperparameterTuningObjective, - HyperparameterTuningObjectiveType, - TrainingJobEarlyStoppingType, -) -from flytekitplugins.awssagemaker.models.parameter_ranges import IntegerParameterRange, ParameterRangeOneOf -from flytekitplugins.awssagemaker.models.training_job import ( - AlgorithmName, - AlgorithmSpecification, - TrainingJobResourceConfig, -) -from flytekitplugins.awssagemaker.training import SagemakerBuiltinAlgorithmsTask, SagemakerTrainingJobConfig - -from flytekit import FlyteContext -from flytekit.models.types import LiteralType, SimpleType - -from .test_training import _get_reg_settings - - -def test_hpo_for_builtin(): - trainer = SagemakerBuiltinAlgorithmsTask( - name="builtin-trainer", - task_config=SagemakerTrainingJobConfig( - training_job_resource_config=TrainingJobResourceConfig( - instance_count=1, - instance_type="ml-xlarge", - volume_size_in_gb=1, - ), - algorithm_specification=AlgorithmSpecification( - algorithm_name=AlgorithmName.XGBOOST, - ), - ), - ) - - hpo = SagemakerHPOTask( - name="test", - task_config=HPOJob(10, 10, ["x"]), - training_task=trainer, - ) - - assert hpo.python_interface.inputs.keys() == { - "static_hyperparameters", - "train", - "validation", - "hyperparameter_tuning_job_config", - "x", - } - assert hpo.python_interface.outputs.keys() == {"model"} - - assert hpo.get_custom(_get_reg_settings()) == { - "maxNumberOfTrainingJobs": "10", - "maxParallelTrainingJobs": "10", - "trainingJob": { - "algorithmSpecification": {"algorithmName": "XGBOOST"}, - "trainingJobResourceConfig": {"instanceCount": "1", "instanceType": "ml-xlarge", "volumeSizeInGb": "1"}, - }, - } - - with pytest.raises(NotImplementedError): - with tempfile.TemporaryDirectory() as tmp: - x = pathlib.Path(os.path.join(tmp, "x")) - y = pathlib.Path(os.path.join(tmp, "y")) - x.mkdir(parents=True, exist_ok=True) - y.mkdir(parents=True, exist_ok=True) - - hpo( - static_hyperparameters={}, - train=f"{x}", # file transformer doesn't handle pathlib.Path yet - validation=f"{y}", # file transformer doesn't handle pathlib.Path yet - hyperparameter_tuning_job_config=HyperparameterTuningJobConfig( - tuning_strategy=1, - tuning_objective=HyperparameterTuningObjective( - objective_type=HyperparameterTuningObjectiveType.MINIMIZE, - metric_name="x", - ), - training_job_early_stopping_type=TrainingJobEarlyStoppingType.OFF, - ), - x=ParameterRangeOneOf(param=IntegerParameterRange(10, 1, 1)), - ) - - -def test_hpoconfig_transformer(): - t = HPOTuningJobConfigTransformer() - assert t.get_literal_type(HyperparameterTuningJobConfig) == LiteralType(simple=SimpleType.STRUCT) - o = HyperparameterTuningJobConfig( - tuning_strategy=1, - tuning_objective=HyperparameterTuningObjective( - objective_type=HyperparameterTuningObjectiveType.MINIMIZE, - metric_name="x", - ), - training_job_early_stopping_type=TrainingJobEarlyStoppingType.OFF, - ) - ctx = FlyteContext.current_context() - lit = t.to_literal(ctx, python_val=o, python_type=HyperparameterTuningJobConfig, expected=None) - assert lit is not None - assert lit.scalar.generic is not None - ro = t.to_python_value(ctx, lit, HyperparameterTuningJobConfig) - assert ro is not None - assert ro == o - - -def test_parameter_ranges_transformer(): - t = ParameterRangesTransformer() - assert t.get_literal_type(ParameterRangeOneOf) == LiteralType(simple=SimpleType.STRUCT) - o = ParameterRangeOneOf(param=IntegerParameterRange(10, 0, 1)) - ctx = FlyteContext.current_context() - lit = t.to_literal(ctx, python_val=o, python_type=ParameterRangeOneOf, expected=None) - assert lit is not None - assert lit.scalar.generic is not None - ro = t.to_python_value(ctx, lit, ParameterRangeOneOf) - assert ro is not None - assert ro == o diff --git a/plugins/flytekit-aws-sagemaker/tests/test_hpo_job.py b/plugins/flytekit-aws-sagemaker/tests/test_hpo_job.py deleted file mode 100644 index 494eecd2ab..0000000000 --- a/plugins/flytekit-aws-sagemaker/tests/test_hpo_job.py +++ /dev/null @@ -1,79 +0,0 @@ -from flytekitplugins.awssagemaker.models import hpo_job, training_job - - -def test_hyperparameter_tuning_objective(): - obj = hpo_job.HyperparameterTuningObjective( - objective_type=hpo_job.HyperparameterTuningObjectiveType.MAXIMIZE, metric_name="test_metric" - ) - obj2 = hpo_job.HyperparameterTuningObjective.from_flyte_idl(obj.to_flyte_idl()) - - assert obj == obj2 - - -def test_hyperparameter_job_config(): - jc = hpo_job.HyperparameterTuningJobConfig( - tuning_strategy=hpo_job.HyperparameterTuningStrategy.BAYESIAN, - tuning_objective=hpo_job.HyperparameterTuningObjective( - objective_type=hpo_job.HyperparameterTuningObjectiveType.MAXIMIZE, metric_name="test_metric" - ), - training_job_early_stopping_type=hpo_job.TrainingJobEarlyStoppingType.AUTO, - ) - - jc2 = hpo_job.HyperparameterTuningJobConfig.from_flyte_idl(jc.to_flyte_idl()) - assert jc2.tuning_strategy == jc.tuning_strategy - assert jc2.tuning_objective == jc.tuning_objective - assert jc2.training_job_early_stopping_type == jc.training_job_early_stopping_type - - -def test_hyperparameter_tuning_job(): - rc = training_job.TrainingJobResourceConfig( - instance_type="test_type", - instance_count=10, - volume_size_in_gb=25, - distributed_protocol=training_job.DistributedProtocol.MPI, - ) - alg = training_job.AlgorithmSpecification( - algorithm_name=training_job.AlgorithmName.CUSTOM, - algorithm_version="", - input_mode=training_job.InputMode.FILE, - input_content_type=training_job.InputContentType.TEXT_CSV, - ) - tj = training_job.TrainingJob( - training_job_resource_config=rc, - algorithm_specification=alg, - ) - hpo = hpo_job.HyperparameterTuningJob(max_number_of_training_jobs=10, max_parallel_training_jobs=5, training_job=tj) - - hpo2 = hpo_job.HyperparameterTuningJob.from_flyte_idl(hpo.to_flyte_idl()) - - assert hpo.max_number_of_training_jobs == hpo2.max_number_of_training_jobs - assert hpo.max_parallel_training_jobs == hpo2.max_parallel_training_jobs - assert ( - hpo2.training_job.training_job_resource_config.instance_type - == hpo.training_job.training_job_resource_config.instance_type - ) - assert ( - hpo2.training_job.training_job_resource_config.instance_count - == hpo.training_job.training_job_resource_config.instance_count - ) - assert ( - hpo2.training_job.training_job_resource_config.distributed_protocol - == hpo.training_job.training_job_resource_config.distributed_protocol - ) - assert ( - hpo2.training_job.training_job_resource_config.volume_size_in_gb - == hpo.training_job.training_job_resource_config.volume_size_in_gb - ) - assert ( - hpo2.training_job.algorithm_specification.algorithm_name - == hpo.training_job.algorithm_specification.algorithm_name - ) - assert ( - hpo2.training_job.algorithm_specification.algorithm_version - == hpo.training_job.algorithm_specification.algorithm_version - ) - assert hpo2.training_job.algorithm_specification.input_mode == hpo.training_job.algorithm_specification.input_mode - assert ( - hpo2.training_job.algorithm_specification.input_content_type - == hpo.training_job.algorithm_specification.input_content_type - ) diff --git a/plugins/flytekit-aws-sagemaker/tests/test_inference_agent.py b/plugins/flytekit-aws-sagemaker/tests/test_inference_agent.py new file mode 100644 index 0000000000..e4003c0735 --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/tests/test_inference_agent.py @@ -0,0 +1,122 @@ +import json +from datetime import timedelta +from unittest import mock + +import pytest +from flyteidl.core.execution_pb2 import TaskExecution +from flytekitplugins.awssagemaker_inference.agent import SageMakerEndpointMetadata + +from flytekit.extend.backend.base_agent import AgentRegistry +from flytekit.interfaces.cli_identifiers import Identifier +from flytekit.models import literals +from flytekit.models.core.identifier import ResourceType +from flytekit.models.task import RuntimeMetadata, TaskMetadata, TaskTemplate + + +@pytest.mark.asyncio +@mock.patch( + "flytekitplugins.awssagemaker_inference.agent.get_agent_secret", + return_value="mocked_secret", +) +@mock.patch( + "flytekitplugins.awssagemaker_inference.agent.Boto3AgentMixin._call", + return_value={ + "EndpointName": "sagemaker-xgboost-endpoint", + "EndpointArn": "arn:aws:sagemaker:us-east-2:1234567890:endpoint/sagemaker-xgboost-endpoint", + "EndpointConfigName": "sagemaker-xgboost-endpoint-config", + "ProductionVariants": [ + { + "VariantName": "variant-name-1", + "DeployedImages": [ + { + "SpecifiedImage": "1234567890.dkr.ecr.us-east-2.amazonaws.com/sagemaker-xgboost:iL3_jIEY3lQPB4wnlS7HKA..", + "ResolvedImage": "1234567890.dkr.ecr.us-east-2.amazonaws.com/sagemaker-xgboost@sha256:0725042bf15f384c46e93bbf7b22c0502859981fc8830fd3aea4127469e8cf1e", + "ResolutionTime": "2024-01-31T22:14:07.193000+05:30", + } + ], + "CurrentWeight": 1.0, + "DesiredWeight": 1.0, + "CurrentInstanceCount": 1, + "DesiredInstanceCount": 1, + } + ], + "EndpointStatus": "InService", + "CreationTime": "2024-01-31T22:14:06.553000+05:30", + "LastModifiedTime": "2024-01-31T22:16:37.001000+05:30", + "AsyncInferenceConfig": { + "OutputConfig": {"S3OutputPath": "s3://sagemaker-agent-xgboost/inference-output/output"} + }, + "ResponseMetadata": { + "RequestId": "50d8bfa7-d84-4bd9-bf11-992832f42793", + "HTTPStatusCode": 200, + "HTTPHeaders": { + "x-amzn-requestid": "50d8bfa7-d840-4bd9-bf11-992832f42793", + "content-type": "application/x-amz-json-1.1", + "content-length": "865", + "date": "Wed, 31 Jan 2024 16:46:38 GMT", + }, + "RetryAttempts": 0, + }, + }, +) +async def test_agent(mock_boto_call, mock_secret): + agent = AgentRegistry.get_agent("sagemaker-endpoint") + task_id = Identifier( + resource_type=ResourceType.TASK, + project="project", + domain="domain", + name="name", + version="version", + ) + task_config = { + "service": "sagemaker", + "config": { + "EndpointName": "sagemaker-endpoint", + "EndpointConfigName": "sagemaker-endpoint-config", + }, + "region": "us-east-2", + "method": "create_endpoint", + } + task_metadata = TaskMetadata( + discoverable=True, + runtime=RuntimeMetadata(RuntimeMetadata.RuntimeType.FLYTE_SDK, "1.0.0", "python"), + timeout=timedelta(days=1), + retries=literals.RetryStrategy(3), + interruptible=True, + discovery_version="0.1.1b0", + deprecated_error_message="This is deprecated!", + cache_serializable=True, + pod_template_name="A", + cache_ignore_input_vars=(), + ) + + task_template = TaskTemplate( + id=task_id, + custom=task_config, + metadata=task_metadata, + interface=None, + type="sagemaker-endpoint", + ) + + # CREATE + metadata = SageMakerEndpointMetadata( + config={ + "EndpointName": "sagemaker-endpoint", + "EndpointConfigName": "sagemaker-endpoint-config", + }, + region="us-east-2", + ) + response = await agent.create(task_template) + assert response == metadata + + # GET + resource = await agent.get(metadata) + assert resource.phase == TaskExecution.SUCCEEDED + + from_json = json.loads(resource.outputs["result"]) + assert from_json["EndpointName"] == "sagemaker-xgboost-endpoint" + assert from_json["EndpointArn"] == "arn:aws:sagemaker:us-east-2:1234567890:endpoint/sagemaker-xgboost-endpoint" + + # DELETE + delete_response = await agent.delete(metadata) + assert delete_response is None diff --git a/plugins/flytekit-aws-sagemaker/tests/test_inference_task.py b/plugins/flytekit-aws-sagemaker/tests/test_inference_task.py new file mode 100644 index 0000000000..93e61d909d --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/tests/test_inference_task.py @@ -0,0 +1,189 @@ +import pytest +from flytekitplugins.awssagemaker_inference import ( + SageMakerDeleteEndpointConfigTask, + SageMakerDeleteEndpointTask, + SageMakerDeleteModelTask, + SageMakerEndpointConfigTask, + SageMakerEndpointTask, + SageMakerInvokeEndpointTask, + SageMakerModelTask, +) + +from flytekit import kwtypes +from flytekit.configuration import Image, ImageConfig, SerializationSettings + + +@pytest.mark.parametrize( + "name,config,service,method,inputs,images,no_of_inputs,no_of_outputs,region,task", + [ + ( + "sagemaker_model", + { + "ModelName": "{inputs.model_name}", + "PrimaryContainer": { + "Image": "{images.primary_container_image}", + "ModelDataUrl": "{inputs.model_data_url}", + }, + "ExecutionRoleArn": "{inputs.execution_role_arn}", + }, + "sagemaker", + "create_model", + kwtypes(model_name=str, model_data_url=str, execution_role_arn=str), + {"primary_container_image": "1234567890.dkr.ecr.us-east-2.amazonaws.com/sagemaker-xgboost"}, + 3, + 1, + "us-east-2", + SageMakerModelTask, + ), + ( + "sagemaker_endpoint_config", + { + "EndpointConfigName": "{inputs.endpoint_config_name}", + "ProductionVariants": [ + { + "VariantName": "variant-name-1", + "ModelName": "{inputs.model_name}", + "InitialInstanceCount": 1, + "InstanceType": "ml.m4.xlarge", + }, + ], + "AsyncInferenceConfig": {"OutputConfig": {"S3OutputPath": "{inputs.s3_output_path}"}}, + }, + "sagemaker", + "create_endpoint_config", + kwtypes(endpoint_config_name=str, model_name=str, s3_output_path=str), + None, + 3, + 1, + "us-east-2", + SageMakerEndpointConfigTask, + ), + ( + "sagemaker_endpoint", + { + "EndpointName": "{inputs.endpoint_name}", + "EndpointConfigName": "{inputs.endpoint_config_name}", + }, + None, + None, + kwtypes(endpoint_name=str, endpoint_config_name=str), + None, + 2, + 1, + "us-east-2", + SageMakerEndpointTask, + ), + ( + "sagemaker_delete_endpoint", + {"EndpointName": "{inputs.endpoint_name}"}, + "sagemaker", + "delete_endpoint", + kwtypes(endpoint_name=str), + None, + 1, + 1, + "us-east-2", + SageMakerDeleteEndpointTask, + ), + ( + "sagemaker_delete_endpoint_config", + {"EndpointConfigName": "{inputs.endpoint_config_name}"}, + "sagemaker", + "delete_endpoint_config", + kwtypes(endpoint_config_name=str), + None, + 1, + 1, + "us-east-2", + SageMakerDeleteEndpointConfigTask, + ), + ( + "sagemaker_delete_model", + {"ModelName": "{inputs.model_name}"}, + "sagemaker", + "delete_model", + kwtypes(model_name=str), + None, + 1, + 1, + "us-east-2", + SageMakerDeleteModelTask, + ), + ( + "sagemaker_invoke_endpoint", + { + "EndpointName": "{inputs.endpoint_name}", + "InputLocation": "s3://sagemaker-agent-xgboost/inference_input", + }, + "sagemaker-runtime", + "invoke_endpoint_async", + kwtypes(endpoint_name=str), + None, + 1, + 1, + "us-east-2", + SageMakerInvokeEndpointTask, + ), + ( + "sagemaker_invoke_endpoint_with_region_at_runtime", + { + "EndpointName": "{inputs.endpoint_name}", + "InputLocation": "s3://sagemaker-agent-xgboost/inference_input", + }, + "sagemaker-runtime", + "invoke_endpoint_async", + kwtypes(endpoint_name=str, region=str), + None, + 2, + 1, + None, + SageMakerInvokeEndpointTask, + ), + ], +) +def test_sagemaker_task( + name, + config, + service, + method, + inputs, + images, + no_of_inputs, + no_of_outputs, + region, + task, +): + if images: + sagemaker_task = task( + name=name, + config=config, + region=region, + inputs=inputs, + images=images, + ) + else: + sagemaker_task = task( + name=name, + config=config, + region=region, + inputs=inputs, + ) + + assert len(sagemaker_task.interface.inputs) == no_of_inputs + assert len(sagemaker_task.interface.outputs) == no_of_outputs + + default_img = Image(name="default", fqn="test", tag="tag") + serialization_settings = SerializationSettings( + project="project", + domain="domain", + version="123", + image_config=ImageConfig(default_image=default_img, images=[default_img]), + env={}, + ) + + retrieved_settings = sagemaker_task.get_custom(serialization_settings) + + assert retrieved_settings.get("service") == service + assert retrieved_settings["config"] == config + assert retrieved_settings["region"] == region + assert retrieved_settings.get("method") == method diff --git a/plugins/flytekit-aws-sagemaker/tests/test_inference_workflow.py b/plugins/flytekit-aws-sagemaker/tests/test_inference_workflow.py new file mode 100644 index 0000000000..f98bb557fa --- /dev/null +++ b/plugins/flytekit-aws-sagemaker/tests/test_inference_workflow.py @@ -0,0 +1,93 @@ +from flytekitplugins.awssagemaker_inference import create_sagemaker_deployment, delete_sagemaker_deployment + +from flytekit import kwtypes + + +def test_sagemaker_deployment_workflow(): + sagemaker_deployment_wf = create_sagemaker_deployment( + name="sagemaker-deployment", + model_input_types=kwtypes(model_path=str, execution_role_arn=str), + model_config={ + "ModelName": "sagemaker-xgboost", + "PrimaryContainer": { + "Image": "{images.primary_container_image}", + "ModelDataUrl": "{inputs.model_path}", + }, + "ExecutionRoleArn": "{inputs.execution_role_arn}", + }, + endpoint_config_input_types=kwtypes(instance_type=str), + endpoint_config_config={ + "EndpointConfigName": "sagemaker-xgboost-endpoint-config", + "ProductionVariants": [ + { + "VariantName": "variant-name-1", + "ModelName": "sagemaker-xgboost", + "InitialInstanceCount": 1, + "InstanceType": "{inputs.instance_type}", + }, + ], + "AsyncInferenceConfig": { + "OutputConfig": {"S3OutputPath": "s3://sagemaker-agent-xgboost/inference-output/output"} + }, + }, + endpoint_config={ + "EndpointName": "sagemaker-xgboost-endpoint", + "EndpointConfigName": "sagemaker-xgboost-endpoint-config", + }, + images={"primary_container_image": "1234567890.dkr.ecr.us-east-2.amazonaws.com/sagemaker-xgboost"}, + region="us-east-2", + ) + + assert len(sagemaker_deployment_wf.interface.inputs) == 3 + assert len(sagemaker_deployment_wf.interface.outputs) == 1 + assert len(sagemaker_deployment_wf.nodes) == 3 + + +def test_sagemaker_deployment_workflow_with_region_at_runtime(): + sagemaker_deployment_wf = create_sagemaker_deployment( + name="sagemaker-deployment-region-runtime", + model_input_types=kwtypes(model_path=str, execution_role_arn=str), + model_config={ + "ModelName": "sagemaker-xgboost", + "PrimaryContainer": { + "Image": "{images.primary_container_image}", + "ModelDataUrl": "{inputs.model_path}", + }, + "ExecutionRoleArn": "{inputs.execution_role_arn}", + }, + endpoint_config_input_types=kwtypes(instance_type=str), + endpoint_config_config={ + "EndpointConfigName": "sagemaker-xgboost-endpoint-config", + "ProductionVariants": [ + { + "VariantName": "variant-name-1", + "ModelName": "sagemaker-xgboost", + "InitialInstanceCount": 1, + "InstanceType": "{inputs.instance_type}", + }, + ], + "AsyncInferenceConfig": { + "OutputConfig": {"S3OutputPath": "s3://sagemaker-agent-xgboost/inference-output/output"} + }, + }, + endpoint_config={ + "EndpointName": "sagemaker-xgboost-endpoint", + "EndpointConfigName": "sagemaker-xgboost-endpoint-config", + }, + images={"primary_container_image": "1234567890.dkr.ecr.us-east-2.amazonaws.com/sagemaker-xgboost"}, + region_at_runtime=True, + ) + + assert len(sagemaker_deployment_wf.interface.inputs) == 4 + assert len(sagemaker_deployment_wf.interface.outputs) == 1 + assert len(sagemaker_deployment_wf.nodes) == 3 + + +def test_sagemaker_deployment_deletion_workflow(): + sagemaker_deployment_deletion_wf = delete_sagemaker_deployment( + name="sagemaker-deployment-deletion", region_at_runtime=True + ) + + assert len(sagemaker_deployment_deletion_wf.interface.inputs) == 4 + assert len(sagemaker_deployment_deletion_wf.interface.outputs) == 0 + assert len(sagemaker_deployment_deletion_wf.nodes) == 3 diff --git a/plugins/flytekit-aws-sagemaker/tests/test_parameter_ranges.py b/plugins/flytekit-aws-sagemaker/tests/test_parameter_ranges.py deleted file mode 100644 index 6d33388c33..0000000000 --- a/plugins/flytekit-aws-sagemaker/tests/test_parameter_ranges.py +++ /dev/null @@ -1,93 +0,0 @@ -import unittest - -import pytest -from flytekitplugins.awssagemaker.models import parameter_ranges - - -# assert statements cannot be written inside lambda expressions. This is a convenient function to work around that. -def assert_equal(a, b): - assert a == b - - -def test_continuous_parameter_range(): - pr = parameter_ranges.ContinuousParameterRange( - max_value=10, min_value=0.5, scaling_type=parameter_ranges.HyperparameterScalingType.REVERSELOGARITHMIC - ) - - pr2 = parameter_ranges.ContinuousParameterRange.from_flyte_idl(pr.to_flyte_idl()) - assert pr == pr2 - assert type(pr2.max_value) == float - assert type(pr2.min_value) == float - assert pr2.max_value == 10.0 - assert pr2.min_value == 0.5 - assert pr2.scaling_type == parameter_ranges.HyperparameterScalingType.REVERSELOGARITHMIC - - -def test_integer_parameter_range(): - pr = parameter_ranges.IntegerParameterRange( - max_value=1, min_value=0, scaling_type=parameter_ranges.HyperparameterScalingType.LOGARITHMIC - ) - - pr2 = parameter_ranges.IntegerParameterRange.from_flyte_idl(pr.to_flyte_idl()) - assert pr == pr2 - assert type(pr2.max_value) == int - assert type(pr2.min_value) == int - assert pr2.max_value == 1 - assert pr2.min_value == 0 - assert pr2.scaling_type == parameter_ranges.HyperparameterScalingType.LOGARITHMIC - - -def test_categorical_parameter_range(): - case = unittest.TestCase() - pr = parameter_ranges.CategoricalParameterRange(values=["abc", "cat"]) - - pr2 = parameter_ranges.CategoricalParameterRange.from_flyte_idl(pr.to_flyte_idl()) - assert pr == pr2 - assert isinstance(pr2.values, list) - case.assertCountEqual(pr2.values, pr.values) - - -def test_parameter_ranges(): - pr = parameter_ranges.ParameterRanges( - { - "a": parameter_ranges.CategoricalParameterRange(values=["a-1", "a-2"]), - "b": parameter_ranges.IntegerParameterRange( - min_value=1, max_value=5, scaling_type=parameter_ranges.HyperparameterScalingType.LINEAR - ), - "c": parameter_ranges.ContinuousParameterRange( - min_value=0.1, max_value=1.0, scaling_type=parameter_ranges.HyperparameterScalingType.LOGARITHMIC - ), - }, - ) - pr2 = parameter_ranges.ParameterRanges.from_flyte_idl(pr.to_flyte_idl()) - assert pr == pr2 - - -LIST_OF_PARAMETERS = [ - ( - parameter_ranges.IntegerParameterRange( - min_value=1, max_value=5, scaling_type=parameter_ranges.HyperparameterScalingType.LINEAR - ), - lambda param: assert_equal(param.integer_parameter_range.max_value, 5), - ), - ( - parameter_ranges.ContinuousParameterRange( - min_value=0.1, max_value=1.0, scaling_type=parameter_ranges.HyperparameterScalingType.LOGARITHMIC - ), - lambda param: assert_equal(param.continuous_parameter_range.max_value, 1), - ), - ( - parameter_ranges.CategoricalParameterRange(values=["a-1", "a-2"]), - lambda param: assert_equal(len(param.categorical_parameter_range.values), 2), - ), -] - - -@pytest.mark.parametrize("param_tuple", LIST_OF_PARAMETERS) -def test_parameter_ranges_oneof(param_tuple): - param, assertion = param_tuple - oneof = parameter_ranges.ParameterRangeOneOf(param=param) - oneof2 = parameter_ranges.ParameterRangeOneOf.from_flyte_idl(oneof.to_flyte_idl()) - assert oneof2 == oneof - assertion(oneof) - assertion(oneof2) diff --git a/plugins/flytekit-aws-sagemaker/tests/test_training.py b/plugins/flytekit-aws-sagemaker/tests/test_training.py deleted file mode 100644 index 4d33a9e4bb..0000000000 --- a/plugins/flytekit-aws-sagemaker/tests/test_training.py +++ /dev/null @@ -1,134 +0,0 @@ -import os -import pathlib -import tempfile - -import pytest -from flytekitplugins.awssagemaker.distributed_training import setup_envars_for_testing -from flytekitplugins.awssagemaker.models.training_job import ( - AlgorithmName, - AlgorithmSpecification, - DistributedProtocol, - TrainingJobResourceConfig, -) -from flytekitplugins.awssagemaker.training import SagemakerBuiltinAlgorithmsTask, SagemakerTrainingJobConfig - -import flytekit -from flytekit import task -from flytekit.configuration import Image, ImageConfig, SerializationSettings -from flytekit.core.context_manager import ExecutionParameters - - -def _get_reg_settings(): - default_img = Image(name="default", fqn="test", tag="tag") - settings = SerializationSettings( - project="project", - domain="domain", - version="version", - env={"FOO": "baz"}, - image_config=ImageConfig(default_image=default_img, images=[default_img]), - ) - return settings - - -def test_builtin_training(): - trainer = SagemakerBuiltinAlgorithmsTask( - name="builtin-trainer", - task_config=SagemakerTrainingJobConfig( - training_job_resource_config=TrainingJobResourceConfig( - instance_count=1, - instance_type="ml-xlarge", - volume_size_in_gb=1, - ), - algorithm_specification=AlgorithmSpecification( - algorithm_name=AlgorithmName.XGBOOST, - ), - ), - ) - - assert trainer.python_interface.inputs.keys() == {"static_hyperparameters", "train", "validation"} - assert trainer.python_interface.outputs.keys() == {"model"} - - with tempfile.TemporaryDirectory() as tmp: - x = pathlib.Path(os.path.join(tmp, "x")) - y = pathlib.Path(os.path.join(tmp, "y")) - x.mkdir(parents=True, exist_ok=True) - y.mkdir(parents=True, exist_ok=True) - with pytest.raises(NotImplementedError): - # Type engine doesn't support pathlib.Path yet - trainer(static_hyperparameters={}, train=f"{x}", validation=f"{y}") - - assert trainer.get_custom(_get_reg_settings()) == { - "algorithmSpecification": {"algorithmName": "XGBOOST"}, - "trainingJobResourceConfig": {"instanceCount": "1", "instanceType": "ml-xlarge", "volumeSizeInGb": "1"}, - } - - -def test_custom_training(): - @task( - task_config=SagemakerTrainingJobConfig( - training_job_resource_config=TrainingJobResourceConfig( - instance_type="ml-xlarge", - volume_size_in_gb=1, - ), - algorithm_specification=AlgorithmSpecification( - algorithm_name=AlgorithmName.CUSTOM, - ), - ) - ) - def my_custom_trainer(x: int) -> int: - return x - - assert my_custom_trainer.python_interface.inputs == {"x": int} - assert my_custom_trainer.python_interface.outputs == {"o0": int} - - assert my_custom_trainer(x=10) == 10 - - assert my_custom_trainer.get_custom(_get_reg_settings()) == { - "algorithmSpecification": {}, - "trainingJobResourceConfig": {"instanceCount": "1", "instanceType": "ml-xlarge", "volumeSizeInGb": "1"}, - } - - -def test_distributed_custom_training(): - setup_envars_for_testing() - - @task( - task_config=SagemakerTrainingJobConfig( - training_job_resource_config=TrainingJobResourceConfig( - instance_type="ml-xlarge", - volume_size_in_gb=1, - instance_count=2, # Indicates distributed training - distributed_protocol=DistributedProtocol.MPI, - ), - algorithm_specification=AlgorithmSpecification( - algorithm_name=AlgorithmName.CUSTOM, - ), - ) - ) - def my_custom_trainer(x: int) -> int: - assert flytekit.current_context().distributed_training_context is not None - return x - - assert my_custom_trainer.python_interface.inputs == {"x": int} - assert my_custom_trainer.python_interface.outputs == {"o0": int} - - assert my_custom_trainer(x=10) == 10 - - assert my_custom_trainer._is_distributed() is True - - pb = ExecutionParameters.new_builder() - pb.working_dir = "/tmp" - p = pb.build() - new_p = my_custom_trainer.pre_execute(p) - assert new_p is not None - assert new_p.has_attr("distributed_training_context") - - assert my_custom_trainer.get_custom(_get_reg_settings()) == { - "algorithmSpecification": {}, - "trainingJobResourceConfig": { - "distributedProtocol": "MPI", - "instanceCount": "2", - "instanceType": "ml-xlarge", - "volumeSizeInGb": "1", - }, - } diff --git a/plugins/flytekit-aws-sagemaker/tests/test_training_job.py b/plugins/flytekit-aws-sagemaker/tests/test_training_job.py deleted file mode 100644 index 8774857b1f..0000000000 --- a/plugins/flytekit-aws-sagemaker/tests/test_training_job.py +++ /dev/null @@ -1,87 +0,0 @@ -import unittest - -from flytekitplugins.awssagemaker.models import training_job - - -def test_training_job_resource_config(): - rc = training_job.TrainingJobResourceConfig( - instance_count=1, - instance_type="random.instance", - volume_size_in_gb=25, - distributed_protocol=training_job.DistributedProtocol.MPI, - ) - - rc2 = training_job.TrainingJobResourceConfig.from_flyte_idl(rc.to_flyte_idl()) - assert rc2 == rc - assert rc2.distributed_protocol == training_job.DistributedProtocol.MPI - assert rc != training_job.TrainingJobResourceConfig( - instance_count=1, - instance_type="random.instance", - volume_size_in_gb=25, - distributed_protocol=training_job.DistributedProtocol.UNSPECIFIED, - ) - - assert rc != training_job.TrainingJobResourceConfig( - instance_count=1, - instance_type="oops", - volume_size_in_gb=25, - distributed_protocol=training_job.DistributedProtocol.MPI, - ) - - -def test_metric_definition(): - md = training_job.MetricDefinition(name="test-metric", regex="[a-zA-Z]*") - - md2 = training_job.MetricDefinition.from_flyte_idl(md.to_flyte_idl()) - assert md == md2 - assert md2.name == "test-metric" - assert md2.regex == "[a-zA-Z]*" - - -def test_algorithm_specification(): - case = unittest.TestCase() - alg_spec = training_job.AlgorithmSpecification( - algorithm_name=training_job.AlgorithmName.CUSTOM, - algorithm_version="v100", - input_mode=training_job.InputMode.FILE, - metric_definitions=[training_job.MetricDefinition(name="a", regex="b")], - input_content_type=training_job.InputContentType.TEXT_CSV, - ) - - alg_spec2 = training_job.AlgorithmSpecification.from_flyte_idl(alg_spec.to_flyte_idl()) - - assert alg_spec2.algorithm_name == training_job.AlgorithmName.CUSTOM - assert alg_spec2.algorithm_version == "v100" - assert alg_spec2.input_mode == training_job.InputMode.FILE - case.assertCountEqual(alg_spec.metric_definitions, alg_spec2.metric_definitions) - assert alg_spec == alg_spec2 - - -def test_training_job(): - rc = training_job.TrainingJobResourceConfig( - instance_type="test_type", - instance_count=10, - volume_size_in_gb=25, - distributed_protocol=training_job.DistributedProtocol.MPI, - ) - alg = training_job.AlgorithmSpecification( - algorithm_name=training_job.AlgorithmName.CUSTOM, - algorithm_version="", - input_mode=training_job.InputMode.FILE, - input_content_type=training_job.InputContentType.TEXT_CSV, - ) - tj = training_job.TrainingJob( - training_job_resource_config=rc, - algorithm_specification=alg, - ) - - tj2 = training_job.TrainingJob.from_flyte_idl(tj.to_flyte_idl()) - # checking tj == tj2 would return false because we don't have the __eq__ magic method defined - assert tj.training_job_resource_config.instance_type == tj2.training_job_resource_config.instance_type - assert tj.training_job_resource_config.instance_count == tj2.training_job_resource_config.instance_count - assert tj.training_job_resource_config.distributed_protocol == tj2.training_job_resource_config.distributed_protocol - assert tj.training_job_resource_config.volume_size_in_gb == tj2.training_job_resource_config.volume_size_in_gb - assert tj.algorithm_specification.algorithm_name == tj2.algorithm_specification.algorithm_name - assert tj.algorithm_specification.algorithm_version == tj2.algorithm_specification.algorithm_version - assert tj.algorithm_specification.input_mode == tj2.algorithm_specification.input_mode - assert tj.algorithm_specification.input_content_type == tj2.algorithm_specification.input_content_type diff --git a/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py b/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py index 4cd0d5a3e9..64dd84faf8 100644 --- a/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py +++ b/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py @@ -88,7 +88,9 @@ def __init__(self): super().__init__(name="GreatExpectations Transformer", t=GreatExpectationsType) @staticmethod - def get_config(t: Type[GreatExpectationsType]) -> Tuple[Type, GreatExpectationsFlyteConfig]: + def get_config( + t: Type[GreatExpectationsType], + ) -> Tuple[Type, GreatExpectationsFlyteConfig]: return t.config() def get_literal_type(self, t: Type[GreatExpectationsType]) -> LiteralType: @@ -138,13 +140,20 @@ def _flyte_schema( # copy parquet file to user-given directory if lv.scalar.structured_dataset: - ctx.file_access.get_data(lv.scalar.structured_dataset.uri, ge_conf.local_file_path, is_multipart=True) + ctx.file_access.get_data( + lv.scalar.structured_dataset.uri, + ge_conf.local_file_path, + is_multipart=True, + ) else: ctx.file_access.get_data(lv.scalar.schema.uri, ge_conf.local_file_path, is_multipart=True) temp_dataset = os.path.basename(ge_conf.local_file_path) - return FlyteSchemaTransformer().to_python_value(ctx, lv, expected_python_type), temp_dataset + return ( + FlyteSchemaTransformer().to_python_value(ctx, lv, expected_python_type), + temp_dataset, + ) def _flyte_file( self, @@ -199,7 +208,12 @@ def to_python_value( context = ge.data_context.DataContext(ge_conf.context_root_dir) # type: ignore # determine the type of data connector - selected_datasource = list(filter(lambda x: x["name"] == ge_conf.datasource_name, context.list_datasources())) + selected_datasource = list( + filter( + lambda x: x["name"] == ge_conf.datasource_name, + context.list_datasources(), + ) + ) if not selected_datasource: raise ValueError("Datasource doesn't exist!") @@ -226,7 +240,11 @@ def to_python_value( # FlyteSchema if lv.scalar.schema or lv.scalar.structured_dataset: return_dataset, temp_dataset = self._flyte_schema( - is_runtime=is_runtime, ctx=ctx, ge_conf=ge_conf, lv=lv, expected_python_type=type_conf[0] + is_runtime=is_runtime, + ctx=ctx, + ge_conf=ge_conf, + lv=lv, + expected_python_type=type_conf[0], ) # FlyteFile diff --git a/plugins/flytekit-papermill/tests/test_task.py b/plugins/flytekit-papermill/tests/test_task.py index 8c229f71f9..9c7b778afb 100644 --- a/plugins/flytekit-papermill/tests/test_task.py +++ b/plugins/flytekit-papermill/tests/test_task.py @@ -63,7 +63,11 @@ def test_notebook_task_simple(): sqr, out, render = nb_simple.execute(pi=4) assert sqr == 16.0 assert nb_simple.python_interface.inputs == {"pi": float} - assert nb_simple.python_interface.outputs.keys() == {"square", "out_nb", "out_rendered_nb"} + assert nb_simple.python_interface.outputs.keys() == { + "square", + "out_nb", + "out_rendered_nb", + } assert nb_simple.output_notebook_path == out == _get_nb_path(nb_name, suffix="-out") assert nb_simple.rendered_output_path == render == _get_nb_path(nb_name, suffix="-out", ext=".html") assert ( @@ -86,7 +90,14 @@ def test_notebook_task_multi_values(): assert h == "blah world!" assert type(n) == datetime.datetime assert nb.python_interface.inputs == {"x": int, "y": int, "h": str} - assert nb.python_interface.outputs.keys() == {"z", "m", "h", "n", "out_nb", "out_rendered_nb"} + assert nb.python_interface.outputs.keys() == { + "z", + "m", + "h", + "n", + "out_nb", + "out_rendered_nb", + } assert nb.output_notebook_path == out == _get_nb_path(nb_name, suffix="-out") assert nb.rendered_output_path == render == _get_nb_path(nb_name, suffix="-out", ext=".html") @@ -104,7 +115,13 @@ def test_notebook_task_complex(): assert w is not None assert x.x == 10 assert nb.python_interface.inputs == {"n": int, "h": str, "w": str} - assert nb.python_interface.outputs.keys() == {"h", "w", "x", "out_nb", "out_rendered_nb"} + assert nb.python_interface.outputs.keys() == { + "h", + "w", + "x", + "out_nb", + "out_rendered_nb", + } assert nb.output_notebook_path == out == _get_nb_path(nb_name, suffix="-out") assert nb.rendered_output_path == render == _get_nb_path(nb_name, suffix="-out", ext=".html") @@ -241,7 +258,10 @@ def wf(a: float) -> typing.List[float]: def test_register_notebook_task(mock_client, mock_remote): mock_remote._client = mock_client mock_remote.return_value._version_from_hash.return_value = "dummy_version_from_hash" - mock_remote.return_value.fast_package.return_value = "dummy_md5_bytes", "dummy_native_url" + mock_remote.return_value.fast_package.return_value = ( + "dummy_md5_bytes", + "dummy_native_url", + ) runner = CliRunner() context_manager.FlyteEntities.entities.clear() notebook_task = """ diff --git a/pyproject.toml b/pyproject.toml index 2ea7997f8f..df457ac3ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,8 +116,8 @@ branch = true [tool.ruff] line-length = 120 -select = ["E", "W", "F", "I"] -ignore = [ +lint.select = ["E", "W", "F", "I"] +lint.ignore = [ # Whitespace before '{symbol}' "E203", # Too many leading # before block comment @@ -134,7 +134,7 @@ ignore = [ "E731", ] -[tool.ruff.extend-per-file-ignores] +[tool.ruff.lint.extend-per-file-ignores] "*/__init__.py" = [ # unused-import "F401", From 4522dc90d8da70312be26a0542ed5b0d1084e3f3 Mon Sep 17 00:00:00 2001 From: Noah Jackson Date: Mon, 25 Mar 2024 06:58:28 -0700 Subject: [PATCH 12/50] Pass additional fields to agent create (#2272) Signed-off-by: noahjax Signed-off-by: Kevin Su Co-authored-by: Kevin Su --- flytekit/extend/backend/agent_service.py | 7 ++++++- flytekit/extend/backend/base_agent.py | 5 ++++- tests/flytekit/unit/extend/test_agent.py | 12 +++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/flytekit/extend/backend/agent_service.py b/flytekit/extend/backend/agent_service.py index e6b9f49639..3e1527c5c5 100644 --- a/flytekit/extend/backend/agent_service.py +++ b/flytekit/extend/backend/agent_service.py @@ -117,7 +117,12 @@ async def CreateTask(self, request: CreateTaskRequest, context: grpc.ServicerCon agent = AgentRegistry.get_agent(template.type, template.task_type_version) logger.info(f"{agent.name} start creating the job") - resource_mata = await mirror_async_methods(agent.create, task_template=template, inputs=inputs) + resource_mata = await mirror_async_methods( + agent.create, + task_template=template, + inputs=inputs, + output_prefix=request.output_prefix, + ) return CreateTaskResponse(resource_meta=resource_mata.encode()) @record_agent_metrics diff --git a/flytekit/extend/backend/base_agent.py b/flytekit/extend/backend/base_agent.py index 0f1b71068d..3c1a149abc 100644 --- a/flytekit/extend/backend/base_agent.py +++ b/flytekit/extend/backend/base_agent.py @@ -145,7 +145,9 @@ def metadata_type(self) -> ResourceMeta: return self._metadata_type @abstractmethod - def create(self, task_template: TaskTemplate, inputs: Optional[LiteralMap], **kwargs) -> ResourceMeta: + def create( + self, task_template: TaskTemplate, inputs: Optional[LiteralMap], output_prefix: Optional[str], **kwargs + ) -> ResourceMeta: """ Return a resource meta that can be used to get the status of the task. """ @@ -312,6 +314,7 @@ async def _create( self._agent.create, task_template=task_template, inputs=literal_map, + output_prefix=output_prefix, ) signal.signal(signal.SIGINT, partial(self.signal_handler, resource_meta)) # type: ignore diff --git a/tests/flytekit/unit/extend/test_agent.py b/tests/flytekit/unit/extend/test_agent.py index 8a1289f974..1bb4976cbd 100644 --- a/tests/flytekit/unit/extend/test_agent.py +++ b/tests/flytekit/unit/extend/test_agent.py @@ -47,6 +47,7 @@ @dataclass class DummyMetadata(ResourceMeta): job_id: str + output_path: typing.Optional[str] = None class DummyAgent(AsyncAgentBase): @@ -72,9 +73,14 @@ def __init__(self): super().__init__(task_type_name="async_dummy", metadata_type=DummyMetadata) async def create( - self, task_template: TaskTemplate, inputs: typing.Optional[LiteralMap] = None, **kwargs + self, + task_template: TaskTemplate, + inputs: typing.Optional[LiteralMap] = None, + output_prefix: typing.Optional[str] = None, + **kwargs, ) -> DummyMetadata: - return DummyMetadata(job_id=dummy_id) + output_path = f"{output_prefix}/{dummy_id}" if output_prefix else None + return DummyMetadata(job_id=dummy_id, output_path=output_path) async def get(self, resource_meta: DummyMetadata, **kwargs) -> Resource: return Resource(phase=TaskExecution.SUCCEEDED, log_links=[TaskLog(name="console", uri="localhost:3000")]) @@ -164,7 +170,7 @@ async def test_async_agent_service(agent): inputs_proto = task_inputs.to_flyte_idl() output_prefix = "/tmp" - metadata_bytes = DummyMetadata(job_id=dummy_id).encode() + metadata_bytes = DummyMetadata(job_id=dummy_id, output_path=f"{output_prefix}/{dummy_id}").encode() tmp = get_task_template(agent.task_category.name).to_flyte_idl() task_category = TaskCategory(name=agent.task_category.name, version=0) From d9cea30be22e840fa206975badae4a8405e480e1 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 25 Mar 2024 11:44:12 -0700 Subject: [PATCH 13/50] feat: Add retry mechanism to file access methods (#2287) Signed-off-by: Kevin Su --- dev-requirements.in | 1 + flytekit/core/data_persistence.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/dev-requirements.in b/dev-requirements.in index 93e054ac14..fb90c597b9 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -38,6 +38,7 @@ python-magic; (platform_system=='Darwin' or platform_system=='Linux') types-protobuf types-croniter +types-decorator types-mock autoflake diff --git a/flytekit/core/data_persistence.py b/flytekit/core/data_persistence.py index cc6cd8bcdb..f507e491b1 100644 --- a/flytekit/core/data_persistence.py +++ b/flytekit/core/data_persistence.py @@ -22,10 +22,12 @@ import pathlib import tempfile import typing +from time import sleep from typing import Any, Dict, Optional, Union, cast from uuid import UUID import fsspec +from decorator import decorator from fsspec.utils import get_protocol from typing_extensions import Unpack @@ -101,6 +103,24 @@ def get_fsspec_storage_options( return {} +@decorator +def retry_request(func, *args, **kwargs): + # TODO: Remove this method once s3fs has a new release. https://github.com/fsspec/s3fs/pull/865 + retries = kwargs.pop("retries", 5) + for retry in range(retries): + try: + if retry > 0: + sleep(random.randint(0, min(2**retry, 32))) + return func(*args, **kwargs) + except Exception as e: + # Catch this specific error message from S3 since S3FS doesn't catch it and retry the request. + if "Please reduce your request rate" in str(e): + if retry == retries - 1: + raise e + else: + raise e + + class FileAccessProvider(object): """ This is the class that is available through the FlyteContext and can be used for persisting data to the remote @@ -246,6 +266,7 @@ def exists(self, path: str) -> bool: return anon_fs.exists(path) raise oe + @retry_request def get(self, from_path: str, to_path: str, recursive: bool = False, **kwargs): file_system = self.get_filesystem_for_path(from_path) if recursive: @@ -272,6 +293,7 @@ def get(self, from_path: str, to_path: str, recursive: bool = False, **kwargs): return file_system.get(from_path, to_path, recursive=recursive, **kwargs) raise oe + @retry_request def put(self, from_path: str, to_path: str, recursive: bool = False, **kwargs): file_system = self.get_filesystem_for_path(to_path) from_path = self.strip_file_header(from_path) From ecc783566fa254fad31a9ccc3c443172955212bc Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 25 Mar 2024 11:44:32 -0700 Subject: [PATCH 14/50] pyflyte run spark task (#2280) Signed-off-by: Kevin Su --- flytekit/core/container_task.py | 12 ++++++++---- flytekit/core/python_auto_container.py | 12 ++++++++---- flytekit/remote/remote.py | 1 + plugins/flytekit-spark/flytekitplugins/spark/task.py | 8 ++++++++ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/flytekit/core/container_task.py b/flytekit/core/container_task.py index 1b078f83a7..7773226c1a 100644 --- a/flytekit/core/container_task.py +++ b/flytekit/core/container_task.py @@ -112,14 +112,18 @@ def _get_data_loading_config(self) -> _task_model.DataLoadingConfig: io_strategy=self._io_strategy.value if self._io_strategy else None, ) + def _get_image(self, settings: SerializationSettings) -> str: + if settings.fast_serialization_settings is None or not settings.fast_serialization_settings.enabled: + if isinstance(self._image, ImageSpec): + # Set the source root for the image spec if it's non-fast registration + self._image.source_root = settings.source_root + return get_registerable_container_image(self._image, settings.image_config) + def _get_container(self, settings: SerializationSettings) -> _task_model.Container: env = settings.env or {} env = {**env, **self.environment} if self.environment else env - if isinstance(self._image, ImageSpec): - if settings.fast_serialization_settings is None or not settings.fast_serialization_settings.enabled: - self._image.source_root = settings.source_root return _get_container_definition( - image=get_registerable_container_image(self._image, settings.image_config), + image=self._get_image(settings), command=self._cmd, args=self._args, data_loading_config=self._get_data_loading_config(), diff --git a/flytekit/core/python_auto_container.py b/flytekit/core/python_auto_container.py index c43e3d4d14..7099456e5b 100644 --- a/flytekit/core/python_auto_container.py +++ b/flytekit/core/python_auto_container.py @@ -175,6 +175,13 @@ def get_command(self, settings: SerializationSettings) -> List[str]: """ return self._get_command_fn(settings) + def get_image(self, settings: SerializationSettings) -> str: + if settings.fast_serialization_settings is None or not settings.fast_serialization_settings.enabled: + if isinstance(self.container_image, ImageSpec): + # Set the source root for the image spec if it's non-fast registration + self.container_image.source_root = settings.source_root + return get_registerable_container_image(self.container_image, settings.image_config) + def get_container(self, settings: SerializationSettings) -> _task_model.Container: # if pod_template is not None, return None here but in get_k8s_pod, return pod_template merged with container if self.pod_template is not None: @@ -187,11 +194,8 @@ def _get_container(self, settings: SerializationSettings) -> _task_model.Contain for elem in (settings.env, self.environment): if elem: env.update(elem) - if settings.fast_serialization_settings is None or not settings.fast_serialization_settings.enabled: - if isinstance(self.container_image, ImageSpec): - self.container_image.source_root = settings.source_root return _get_container_definition( - image=get_registerable_container_image(self.container_image, settings.image_config), + image=self.get_image(settings), command=[], args=self.get_command(settings=settings), data_loading_config=None, diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 4fd17fe40b..aad5adbd3f 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -986,6 +986,7 @@ def register_script( destination_dir=destination_dir, distribution_location=upload_native_url, ), + source_root=source_path, ) if version is None: diff --git a/plugins/flytekit-spark/flytekitplugins/spark/task.py b/plugins/flytekit-spark/flytekitplugins/spark/task.py index 39a93afd06..079cf8815c 100644 --- a/plugins/flytekit-spark/flytekitplugins/spark/task.py +++ b/plugins/flytekit-spark/flytekitplugins/spark/task.py @@ -7,6 +7,7 @@ from flytekit import FlyteContextManager, PythonFunctionTask, lazy_module, logger from flytekit.configuration import DefaultImages, SerializationSettings from flytekit.core.context_manager import ExecutionParameters +from flytekit.core.python_auto_container import get_registerable_container_image from flytekit.extend import ExecutionState, TaskPlugins from flytekit.extend.backend.base_agent import AsyncAgentExecutorMixin from flytekit.image_spec import ImageSpec @@ -136,6 +137,13 @@ def __init__( **kwargs, ) + def get_image(self, settings: SerializationSettings) -> str: + if isinstance(self.container_image, ImageSpec): + # Ensure that the code is always copied into the image, even during fast-registration. + self.container_image.source_root = settings.source_root + + return get_registerable_container_image(self.container_image, settings.image_config) + def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: job = SparkJob( spark_conf=self.task_config.spark_conf, From 53299d676c907e17d9964e4d28a4cfbbc96a1929 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 25 Mar 2024 21:21:17 -0700 Subject: [PATCH 15/50] Automatically create envd context for users (#2266) Signed-off-by: Kevin Su --- flytekit/image_spec/image_spec.py | 2 +- .../flytekitplugins/envd/image_builder.py | 62 ++++++++++++------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/flytekit/image_spec/image_spec.py b/flytekit/image_spec/image_spec.py index e740e72449..766e4e4a97 100644 --- a/flytekit/image_spec/image_spec.py +++ b/flytekit/image_spec/image_spec.py @@ -142,7 +142,7 @@ def __hash__(self): def with_commands(self, commands: Union[str, List[str]]) -> "ImageSpec": """ - Builder that returns a new image spec with additional list of commands that will be executed during the building process. + Builder that returns a new image spec with an additional list of commands that will be executed during the building process. """ new_image_spec = copy.deepcopy(self) if new_image_spec.commands is None: diff --git a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py index 3d424d584e..2409f0a25d 100644 --- a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py +++ b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py @@ -12,42 +12,62 @@ from flytekit.core.constants import REQUIREMENTS_FILE_NAME from flytekit.image_spec.image_spec import _F_IMG_ID, ImageBuildEngine, ImageSpec, ImageSpecBuilder +FLYTE_LOCAL_REGISTRY = "localhost:30000" + class EnvdImageSpecBuilder(ImageSpecBuilder): """ This class is used to build a docker image using envd. """ - def execute_command(self, command): - click.secho(f"Run command: {command} ", fg="blue") - p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - result = [] - for line in iter(p.stdout.readline, ""): - if p.poll() is not None: - break - if line.decode().strip() != "": - output = line.decode().strip() - click.secho(output, fg="blue") - result.append(output) - - if p.returncode != 0: - _, stderr = p.communicate() - raise Exception(f"failed to run command {command} with error {stderr}") - - return result - def build_image(self, image_spec: ImageSpec): cfg_path = create_envd_config(image_spec) if image_spec.registry_config: bootstrap_command = f"envd bootstrap --registry-config {image_spec.registry_config}" - self.execute_command(bootstrap_command) + execute_command(bootstrap_command) build_command = f"envd build --path {pathlib.Path(cfg_path).parent} --platform {image_spec.platform}" if image_spec.registry: build_command += f" --output type=image,name={image_spec.image_name()},push=true" - self.execute_command(build_command) + envd_context_switch(image_spec.registry) + execute_command(build_command) + + +def envd_context_switch(registry: str): + if registry == FLYTE_LOCAL_REGISTRY: + # Assume buildkit daemon is running within the sandbox and exposed on port 30003 + command = "envd context create --name flyte-sandbox --builder tcp --builder-address localhost:30003 --use" + p = subprocess.run(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if p.returncode != 0: + # Assume the context already exists + execute_command("envd context use --name flyte-sandbox") + else: + command = "envd context create --name flyte --builder docker-container --builder-address buildkitd-demo -use" + p = subprocess.run(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if p.returncode != 0: + # Assume the context already exists + execute_command("envd context use --name flyte") + + +def execute_command(command: str): + click.secho(f"Run command: {command} ", fg="blue") + p = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + result = [] + for line in iter(p.stdout.readline, ""): + if p.poll() is not None: + break + if line.decode().strip() != "": + output = line.decode().strip() + click.secho(output, fg="blue") + result.append(output) + + if p.returncode != 0: + _, stderr = p.communicate() + raise Exception(f"failed to run command {command} with error {stderr}") + + return result def _create_str_from_package_list(packages): From e1339fa25aedfc49e47d1e0e1a11ce3813f2fe64 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 25 Mar 2024 21:21:36 -0700 Subject: [PATCH 16/50] [CI] Add an action to remove cache from runners (#2265) Signed-off-by: Kevin Su --- .github/actions/clear-action-cache/action.yml | 11 +++++++++++ .github/workflows/pythonbuild.yml | 14 ++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 .github/actions/clear-action-cache/action.yml diff --git a/.github/actions/clear-action-cache/action.yml b/.github/actions/clear-action-cache/action.yml new file mode 100644 index 0000000000..a29347b61c --- /dev/null +++ b/.github/actions/clear-action-cache/action.yml @@ -0,0 +1,11 @@ +name: 'Clear action cache' +description: 'As suggested by GitHub to prevent low disk space: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173' +runs: + using: 'composite' + steps: + - shell: bash + run: | + rm -rf /usr/share/dotnet + rm -rf /opt/ghc + rm -rf "/usr/local/share/boost" + rm -rf "$AGENT_TOOLSDIRECTORY" diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 2b444df511..b58b61ac95 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -41,8 +41,9 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}} steps: - - uses: insightsengineering/disk-space-reclaimer@v1 - uses: actions/checkout@v4 + - name: 'Clear action cache' + uses: ./.github/actions/clear-action-cache - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -78,8 +79,9 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}} steps: - - uses: insightsengineering/disk-space-reclaimer@v1 - uses: actions/checkout@v4 + - name: 'Clear action cache' + uses: ./.github/actions/clear-action-cache - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -119,8 +121,9 @@ jobs: python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}} pandas: ["pandas<2.0.0", "pandas>=2.0.0"] steps: - - uses: insightsengineering/disk-space-reclaimer@v1 - uses: actions/checkout@v4 + - name: 'Clear action cache' + uses: ./.github/actions/clear-action-cache - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -189,7 +192,6 @@ jobs: os: [ubuntu-latest] python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}} steps: - - uses: insightsengineering/disk-space-reclaimer@v1 # As described in https://github.com/pypa/setuptools_scm/issues/414, SCM needs git history # and tags to work. - uses: actions/checkout@v4 @@ -346,9 +348,9 @@ jobs: - python-version: 3.12 plugin-names: "flytekit-kf-pytorch" steps: - - uses: insightsengineering/disk-space-reclaimer@v1 - if: ${{ matrix.plugin-names == 'flytekit-envd' }} - uses: actions/checkout@v4 + - name: 'Clear action cache' + uses: ./.github/actions/clear-action-cache - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 71df7d69fa57c529d11008076143dd1760e08d52 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 25 Mar 2024 21:21:47 -0700 Subject: [PATCH 17/50] feat: Support ImageSpec as base image (#2277) Signed-off-by: Kevin Su --- flytekit/image_spec/image_spec.py | 8 +++++++- plugins/flytekit-envd/tests/test_image_spec.py | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/flytekit/image_spec/image_spec.py b/flytekit/image_spec/image_spec.py index 766e4e4a97..c7c9235a4e 100644 --- a/flytekit/image_spec/image_spec.py +++ b/flytekit/image_spec/image_spec.py @@ -61,7 +61,7 @@ class ImageSpec: apt_packages: Optional[List[str]] = None cuda: Optional[str] = None cudnn: Optional[str] = None - base_image: Optional[str] = None + base_image: Optional[Union[str, "ImageSpec"]] = None platform: str = "linux/amd64" pip_index: Optional[str] = None pip_extra_index_url: Optional[List[str]] = None @@ -228,6 +228,10 @@ def register(cls, builder_type: str, image_spec_builder: ImageSpecBuilder, prior @classmethod @lru_cache def build(cls, image_spec: ImageSpec) -> str: + if isinstance(image_spec.base_image, ImageSpec): + cls.build(image_spec.base_image) + image_spec.base_image = image_spec.base_image.image_name() + if image_spec.builder is None and cls._REGISTRY: builder = max(cls._REGISTRY, key=lambda name: cls._REGISTRY[name][1]) else: @@ -269,6 +273,8 @@ def calculate_hash_from_image_spec(image_spec: ImageSpec): """ # copy the image spec to avoid modifying the original image spec. otherwise, the hash will be different. spec = copy.deepcopy(image_spec) + if isinstance(spec.base_image, ImageSpec): + spec.base_image = spec.base_image.image_name() spec.source_root = hash_directory(image_spec.source_root) if image_spec.source_root else b"" if spec.requirements: spec.requirements = hashlib.sha1(pathlib.Path(spec.requirements).read_bytes()).hexdigest() diff --git a/plugins/flytekit-envd/tests/test_image_spec.py b/plugins/flytekit-envd/tests/test_image_spec.py index 8e5d8b1631..f7c8e3f370 100644 --- a/plugins/flytekit-envd/tests/test_image_spec.py +++ b/plugins/flytekit-envd/tests/test_image_spec.py @@ -21,17 +21,27 @@ def register_envd_higher_priority(): def test_image_spec(): + base_image = ImageSpec( + packages=["numpy"], + python_version="3.8", + registry="", + base_image="cr.flyte.org/flyteorg/flytekit:py3.8-latest", + ) + # Replace the base image name with the default flytekit image name, + # so Envd can find the base image when building imageSpec below + ImageBuildEngine._IMAGE_NAME_TO_REAL_NAME[base_image.image_name()] = "cr.flyte.org/flyteorg/flytekit:py3.8-latest" + image_spec = ImageSpec( packages=["pandas"], apt_packages=["git"], python_version="3.8", - base_image="cr.flyte.org/flyteorg/flytekit:py3.8-latest", + base_image=base_image, pip_index="https://private-pip-index/simple", ) image_spec = image_spec.with_commands("echo hello") - EnvdImageSpecBuilder().build_image(image_spec) + ImageBuildEngine.build(image_spec) config_path = create_envd_config(image_spec) assert image_spec.platform == "linux/amd64" image_name = image_spec.image_name() From 6a63c1fffdaa2ee1f94d4c184c79511b73a8cce0 Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Tue, 26 Mar 2024 00:24:53 -0400 Subject: [PATCH 18/50] Makes the deviceflow auth URL simplier (#2293) Signed-off-by: Thomas J. Fan --- flytekit/clients/auth/authenticator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flytekit/clients/auth/authenticator.py b/flytekit/clients/auth/authenticator.py index fdf1d13eae..0ed780509e 100644 --- a/flytekit/clients/auth/authenticator.py +++ b/flytekit/clients/auth/authenticator.py @@ -302,7 +302,9 @@ def refresh_credentials(self): self._verify, self._session, ) - text = f"To Authenticate, navigate in a browser to the following URL: {click.style(resp.verification_uri, fg='blue', underline=True)} and enter code: {click.style(resp.user_code, fg='blue')}" + + full_uri = f"{resp.verification_uri}?user_code={resp.user_code}" + text = f"To Authenticate, navigate in a browser to the following URL: {click.style(full_uri, fg='blue', underline=True)}" click.secho(text) try: # Currently the refresh token is not retrieved. We may want to add support for refreshTokens so that From 133e8d5b1a5f833e55267a15b035d237b040968a Mon Sep 17 00:00:00 2001 From: Noah Jackson Date: Tue, 26 Mar 2024 14:38:24 -0700 Subject: [PATCH 19/50] Add task execution metadata to agent create (#2282) Signed-off-by: noahjax Signed-off-by: Kevin Su Co-authored-by: Kevin Su --- flytekit/extend/backend/agent_service.py | 4 +- flytekit/extend/backend/base_agent.py | 9 ++- flytekit/extend/backend/utils.py | 4 +- flytekit/models/task.py | 89 ++++++++++++++++++++++++ tests/flytekit/unit/extend/test_agent.py | 50 +++++++++++-- 5 files changed, 145 insertions(+), 11 deletions(-) diff --git a/flytekit/extend/backend/agent_service.py b/flytekit/extend/backend/agent_service.py index 3e1527c5c5..eb2838ca41 100644 --- a/flytekit/extend/backend/agent_service.py +++ b/flytekit/extend/backend/agent_service.py @@ -30,7 +30,7 @@ from flytekit.exceptions.system import FlyteAgentNotFound from flytekit.extend.backend.base_agent import AgentRegistry, SyncAgentBase, mirror_async_methods from flytekit.models.literals import LiteralMap -from flytekit.models.task import TaskTemplate +from flytekit.models.task import TaskExecutionMetadata, TaskTemplate metric_prefix = "flyte_agent_" create_operation = "create" @@ -115,6 +115,7 @@ async def CreateTask(self, request: CreateTaskRequest, context: grpc.ServicerCon template = TaskTemplate.from_flyte_idl(request.template) inputs = LiteralMap.from_flyte_idl(request.inputs) if request.inputs else None agent = AgentRegistry.get_agent(template.type, template.task_type_version) + task_execution_metadata = TaskExecutionMetadata.from_flyte_idl(request.task_execution_metadata) logger.info(f"{agent.name} start creating the job") resource_mata = await mirror_async_methods( @@ -122,6 +123,7 @@ async def CreateTask(self, request: CreateTaskRequest, context: grpc.ServicerCon task_template=template, inputs=inputs, output_prefix=request.output_prefix, + task_execution_metadata=task_execution_metadata, ) return CreateTaskResponse(resource_meta=resource_mata.encode()) diff --git a/flytekit/extend/backend/base_agent.py b/flytekit/extend/backend/base_agent.py index 3c1a149abc..ac942a3642 100644 --- a/flytekit/extend/backend/base_agent.py +++ b/flytekit/extend/backend/base_agent.py @@ -26,7 +26,7 @@ from flytekit.exceptions.user import FlyteUserException from flytekit.extend.backend.utils import is_terminal_phase, mirror_async_methods, render_task_template from flytekit.models.literals import LiteralMap -from flytekit.models.task import TaskTemplate +from flytekit.models.task import TaskExecutionMetadata, TaskTemplate class TaskCategory: @@ -146,7 +146,12 @@ def metadata_type(self) -> ResourceMeta: @abstractmethod def create( - self, task_template: TaskTemplate, inputs: Optional[LiteralMap], output_prefix: Optional[str], **kwargs + self, + task_template: TaskTemplate, + inputs: Optional[LiteralMap], + output_prefix: Optional[str], + task_execution_metadata: Optional[TaskExecutionMetadata], + **kwargs, ) -> ResourceMeta: """ Return a resource meta that can be used to get the status of the task. diff --git a/flytekit/extend/backend/utils.py b/flytekit/extend/backend/utils.py index 5199536b5d..dcea3e6b34 100644 --- a/flytekit/extend/backend/utils.py +++ b/flytekit/extend/backend/utils.py @@ -1,4 +1,5 @@ import asyncio +import functools import inspect from typing import Callable, Coroutine @@ -11,8 +12,7 @@ def mirror_async_methods(func: Callable, **kwargs) -> Coroutine: if inspect.iscoroutinefunction(func): return func(**kwargs) - args = [v for _, v in kwargs.items()] - return asyncio.get_running_loop().run_in_executor(None, func, *args) + return asyncio.get_running_loop().run_in_executor(None, functools.partial(func, **kwargs)) def convert_to_flyte_phase(state: str) -> TaskExecution.Phase: diff --git a/flytekit/models/task.py b/flytekit/models/task.py index b6e8222fb9..198adf2859 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -1,6 +1,7 @@ import json as _json import typing +from flyteidl.admin import agent_pb2 as _admin_agent from flyteidl.admin import task_pb2 as _admin_task from flyteidl.core import compiler_pb2 as _compiler from flyteidl.core import literals_pb2 as _literals_pb2 @@ -518,6 +519,94 @@ def from_flyte_idl(cls, pb2_object): ) +class TaskExecutionMetadata(_common.FlyteIdlEntity): + def __init__( + self, + task_execution_id, + namespace, + labels, + annotations, + k8s_service_account, + environment_variables, + ): + """ + Runtime task execution metadata. + + :param flytekit.models.core.identifier.TaskExecutionIdentifier task_execution_id: This is generated by the system and uniquely identifies + this execution of the task. + :param Text namespace: This is the namespace the task is executing in. + :param dict[str, str] labels: Labels to use for the execution of this task. + :param dict[str, str] annotations: Annotations to use for the execution of this task. + :param Text k8s_service_account: Service account to use for execution of this task. + :param dict[str, str] environment_variables: Environment variables for this task. + """ + self._task_execution_id = task_execution_id + self._namespace = namespace + self._labels = labels + self._annotations = annotations + self._k8s_service_account = k8s_service_account + self._environment_variables = environment_variables + + @property + def task_execution_id(self): + return self._task_execution_id + + @property + def namespace(self): + return self._namespace + + @property + def labels(self): + return self._labels + + @property + def annotations(self): + return self._annotations + + @property + def k8s_service_account(self): + return self._k8s_service_account + + @property + def environment_variables(self): + return self._environment_variables + + def to_flyte_idl(self): + """ + :rtype: flyteidl.admin.agent_pb2.TaskExecutionMetadata + """ + task_execution_metadata = _admin_agent.TaskExecutionMetadata( + task_execution_id=self.task_execution_id.to_flyte_idl(), + namespace=self.namespace, + labels={k: v for k, v in self.labels.items()} if self.labels is not None else None, + annotations={k: v for k, v in self.annotations.items()} if self.annotations is not None else None, + k8s_service_account=self.k8s_service_account, + environment_variables={k: v for k, v in self.environment_variables.items()} + if self.labels is not None + else None, + ) + return task_execution_metadata + + @classmethod + def from_flyte_idl(cls, pb2_object): + """ + :param flyteidl.admin.agent_pb2.TaskExecutionMetadata pb2_object: + :rtype: TaskExecutionMetadata + """ + return cls( + task_execution_id=_identifier.TaskExecutionIdentifier.from_flyte_idl(pb2_object.task_execution_id), + namespace=pb2_object.namespace, + labels={k: v for k, v in pb2_object.labels.items()} if pb2_object.labels is not None else None, + annotations={k: v for k, v in pb2_object.annotations.items()} + if pb2_object.annotations is not None + else None, + k8s_service_account=pb2_object.k8s_service_account, + environment_variables={k: v for k, v in pb2_object.environment_variables.items()} + if pb2_object.environment_variables is not None + else None, + ) + + class TaskSpec(_common.FlyteIdlEntity): def __init__(self, template: TaskTemplate, docs: typing.Optional[Documentation] = None): """ diff --git a/tests/flytekit/unit/extend/test_agent.py b/tests/flytekit/unit/extend/test_agent.py index 1bb4976cbd..2bf23abb25 100644 --- a/tests/flytekit/unit/extend/test_agent.py +++ b/tests/flytekit/unit/extend/test_agent.py @@ -17,6 +17,7 @@ TaskCategory, ) from flyteidl.core.execution_pb2 import TaskExecution, TaskLog +from flyteidl.core.identifier_pb2 import ResourceType from flytekit import PythonFunctionTask, task from flytekit.configuration import FastSerializationSettings, Image, ImageConfig, SerializationSettings @@ -37,8 +38,14 @@ ) from flytekit.extend.backend.utils import convert_to_flyte_phase, get_agent_secret from flytekit.models import literals +from flytekit.models.core.identifier import ( + Identifier, + NodeExecutionIdentifier, + TaskExecutionIdentifier, + WorkflowExecutionIdentifier, +) from flytekit.models.literals import LiteralMap -from flytekit.models.task import TaskTemplate +from flytekit.models.task import TaskExecutionMetadata, TaskTemplate from flytekit.tools.translator import get_serializable dummy_id = "dummy_id" @@ -48,6 +55,7 @@ class DummyMetadata(ResourceMeta): job_id: str output_path: typing.Optional[str] = None + task_name: typing.Optional[str] = None class DummyAgent(AsyncAgentBase): @@ -77,10 +85,12 @@ async def create( task_template: TaskTemplate, inputs: typing.Optional[LiteralMap] = None, output_prefix: typing.Optional[str] = None, + task_execution_metadata: typing.Optional[TaskExecutionMetadata] = None, **kwargs, ) -> DummyMetadata: output_path = f"{output_prefix}/{dummy_id}" if output_prefix else None - return DummyMetadata(job_id=dummy_id, output_path=output_path) + task_name = task_execution_metadata.task_execution_id.task_id.name if task_execution_metadata else "default" + return DummyMetadata(job_id=dummy_id, output_path=output_path, task_name=task_name) async def get(self, resource_meta: DummyMetadata, **kwargs) -> Resource: return Resource(phase=TaskExecution.SUCCEEDED, log_links=[TaskLog(name="console", uri="localhost:3000")]) @@ -136,6 +146,19 @@ def simple_task(i: int): }, ) +task_execution_metadata = TaskExecutionMetadata( + task_execution_id=TaskExecutionIdentifier( + task_id=Identifier(ResourceType.TASK, "project", "domain", "name", "version"), + node_execution_id=NodeExecutionIdentifier("node_id", WorkflowExecutionIdentifier("project", "domain", "name")), + retry_attempt=1, + ), + namespace="namespace", + labels={"label_key": "label_val"}, + annotations={"annotation_key": "annotation_val"}, + k8s_service_account="k8s service account", + environment_variables={"env_var_key": "env_var_val"}, +) + def test_dummy_agent(): AgentRegistry.register(DummyAgent(), override=True) @@ -161,20 +184,35 @@ def __init__(self, **kwargs): t.execute() -@pytest.mark.parametrize("agent", [DummyAgent(), AsyncDummyAgent()], ids=["sync", "async"]) +@pytest.mark.parametrize( + "agent,consume_metadata", [(DummyAgent(), False), (AsyncDummyAgent(), True)], ids=["sync", "async"] +) @pytest.mark.asyncio -async def test_async_agent_service(agent): +async def test_async_agent_service(agent, consume_metadata): AgentRegistry.register(agent, override=True) service = AsyncAgentService() ctx = MagicMock(spec=grpc.ServicerContext) inputs_proto = task_inputs.to_flyte_idl() output_prefix = "/tmp" - metadata_bytes = DummyMetadata(job_id=dummy_id, output_path=f"{output_prefix}/{dummy_id}").encode() + metadata_bytes = ( + DummyMetadata( + job_id=dummy_id, + output_path=f"{output_prefix}/{dummy_id}", + task_name=task_execution_metadata.task_execution_id.task_id.name, + ).encode() + if consume_metadata + else DummyMetadata(job_id=dummy_id).encode() + ) tmp = get_task_template(agent.task_category.name).to_flyte_idl() task_category = TaskCategory(name=agent.task_category.name, version=0) - req = CreateTaskRequest(inputs=inputs_proto, output_prefix=output_prefix, template=tmp) + req = CreateTaskRequest( + inputs=inputs_proto, + template=tmp, + output_prefix=output_prefix, + task_execution_metadata=task_execution_metadata.to_flyte_idl(), + ) res = await service.CreateTask(req, ctx) assert res.resource_meta == metadata_bytes From fa2aa0b215cd2fca3d90d8de92e41c6fadb88ed6 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 27 Mar 2024 06:34:56 +0800 Subject: [PATCH 20/50] [flytekitplugins] Rename ChatGPT module to OpenAI module (#2263) * refactor chagpt module to openai module Signed-off-by: Future-Outlier * nit Signed-off-by: Future-Outlier --------- Signed-off-by: Future-Outlier --- plugins/flytekit-openai/README.md | 4 ++-- .../flytekitplugins/{chatgpt => openai}/__init__.py | 6 +++--- .../flytekitplugins/openai/chatgpt/__init__.py | 0 .../flytekitplugins/{ => openai}/chatgpt/agent.py | 0 .../flytekitplugins/{ => openai}/chatgpt/task.py | 0 plugins/flytekit-openai/setup.py | 8 ++++---- plugins/flytekit-openai/tests/__init__.py | 0 plugins/flytekit-openai/tests/test_chatgpt.py | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) rename plugins/flytekit-openai/flytekitplugins/{chatgpt => openai}/__init__.py (59%) create mode 100644 plugins/flytekit-openai/flytekitplugins/openai/chatgpt/__init__.py rename plugins/flytekit-openai/flytekitplugins/{ => openai}/chatgpt/agent.py (100%) rename plugins/flytekit-openai/flytekitplugins/{ => openai}/chatgpt/task.py (100%) create mode 100644 plugins/flytekit-openai/tests/__init__.py diff --git a/plugins/flytekit-openai/README.md b/plugins/flytekit-openai/README.md index 21c5553ce7..f93b634735 100644 --- a/plugins/flytekit-openai/README.md +++ b/plugins/flytekit-openai/README.md @@ -4,7 +4,7 @@ ChatGPT plugin allows you to run ChatGPT tasks in the Flyte workflow without cha ## Example ```python from flytekit import task, workflow -from flytekitplugins.chatgpt import ChatGPTTask, ChatGPTConfig +from flytekitplugins.openai import ChatGPTTask, ChatGPTConfig chatgpt_small_job = ChatGPTTask( name="chatgpt gpt-3.5-turbo", @@ -40,5 +40,5 @@ if __name__ == "__main__": To install the plugin, run the following command: ```bash -pip install flytekitplugins-chatgpt +pip install flytekitplugins-openai ``` diff --git a/plugins/flytekit-openai/flytekitplugins/chatgpt/__init__.py b/plugins/flytekit-openai/flytekitplugins/openai/__init__.py similarity index 59% rename from plugins/flytekit-openai/flytekitplugins/chatgpt/__init__.py rename to plugins/flytekit-openai/flytekitplugins/openai/__init__.py index 64dd73fb35..58e99f747e 100644 --- a/plugins/flytekit-openai/flytekitplugins/chatgpt/__init__.py +++ b/plugins/flytekit-openai/flytekitplugins/openai/__init__.py @@ -1,5 +1,5 @@ """ -.. currentmodule:: flytekitplugins.chatgpt +.. currentmodule:: flytekitplugins.openai This package contains things that are useful when extending Flytekit. .. autosummary:: :template: custom.rst @@ -8,5 +8,5 @@ ChatGPTTask """ -from .agent import ChatGPTAgent -from .task import ChatGPTTask +from .chatgpt.agent import ChatGPTAgent +from .chatgpt.task import ChatGPTTask diff --git a/plugins/flytekit-openai/flytekitplugins/openai/chatgpt/__init__.py b/plugins/flytekit-openai/flytekitplugins/openai/chatgpt/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/flytekit-openai/flytekitplugins/chatgpt/agent.py b/plugins/flytekit-openai/flytekitplugins/openai/chatgpt/agent.py similarity index 100% rename from plugins/flytekit-openai/flytekitplugins/chatgpt/agent.py rename to plugins/flytekit-openai/flytekitplugins/openai/chatgpt/agent.py diff --git a/plugins/flytekit-openai/flytekitplugins/chatgpt/task.py b/plugins/flytekit-openai/flytekitplugins/openai/chatgpt/task.py similarity index 100% rename from plugins/flytekit-openai/flytekitplugins/chatgpt/task.py rename to plugins/flytekit-openai/flytekitplugins/openai/chatgpt/task.py diff --git a/plugins/flytekit-openai/setup.py b/plugins/flytekit-openai/setup.py index 82257c2435..9a7fff284a 100644 --- a/plugins/flytekit-openai/setup.py +++ b/plugins/flytekit-openai/setup.py @@ -1,10 +1,10 @@ from setuptools import setup -PLUGIN_NAME = "chatgpt" +PLUGIN_NAME = "openai" microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -plugin_requires = ["flytekit>1.10.7", "openai>=1.12.0", "flyteidl>=1.11.0b0"] +plugin_requires = ["flytekit>1.10.7", "openai>=1.12.0", "flyteidl>=1.11.0"] __version__ = "0.0.0+develop" @@ -13,9 +13,9 @@ version=__version__, author="flyteorg", author_email="admin@flyte.org", - description="This package holds the ChatGPT plugins for flytekit", + description="This package holds the openai plugins for flytekit", namespace_packages=["flytekitplugins"], - packages=[f"flytekitplugins.{PLUGIN_NAME}"], + packages=[f"flytekitplugins.{PLUGIN_NAME}", f"flytekitplugins.{PLUGIN_NAME}.chatgpt"], install_requires=plugin_requires, license="apache2", python_requires=">=3.8", diff --git a/plugins/flytekit-openai/tests/__init__.py b/plugins/flytekit-openai/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/flytekit-openai/tests/test_chatgpt.py b/plugins/flytekit-openai/tests/test_chatgpt.py index f85f94cc7b..6298bdf52c 100644 --- a/plugins/flytekit-openai/tests/test_chatgpt.py +++ b/plugins/flytekit-openai/tests/test_chatgpt.py @@ -1,6 +1,6 @@ from collections import OrderedDict -from flytekitplugins.chatgpt import ChatGPTTask +from flytekitplugins.openai import ChatGPTTask from flytekit.configuration import Image, ImageConfig, SerializationSettings from flytekit.extend import get_serializable From a150899c8ac8b49cc7fed28c8771a2c0039f9681 Mon Sep 17 00:00:00 2001 From: Troy Chiu <114708546+troychiu@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:55:37 -0700 Subject: [PATCH 21/50] Add support for execution of reference entities (#1808) Signed-off-by: troychiu --- flytekit/remote/remote.py | 198 +++++++++++++++++- .../integration/remote/test_remote.py | 84 ++++++++ 2 files changed, 280 insertions(+), 2 deletions(-) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index aad5adbd3f..437468a57e 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -36,11 +36,12 @@ from flytekit.core.base_task import PythonTask from flytekit.core.context_manager import FlyteContext, FlyteContextManager from flytekit.core.data_persistence import FileAccessProvider -from flytekit.core.launch_plan import LaunchPlan +from flytekit.core.launch_plan import LaunchPlan, ReferenceLaunchPlan from flytekit.core.python_auto_container import PythonAutoContainerTask from flytekit.core.reference_entity import ReferenceSpec +from flytekit.core.task import ReferenceTask from flytekit.core.type_engine import LiteralsResolver, TypeEngine -from flytekit.core.workflow import WorkflowBase, WorkflowFailurePolicy +from flytekit.core.workflow import ReferenceWorkflow, WorkflowBase, WorkflowFailurePolicy from flytekit.exceptions import user as user_exceptions from flytekit.exceptions.user import ( FlyteEntityAlreadyExistsException, @@ -1297,6 +1298,48 @@ def execute( tags=tags, cluster_pool=cluster_pool, ) + if isinstance(entity, ReferenceTask): + return self.execute_reference_task( + entity=entity, + inputs=inputs, + execution_name=execution_name, + execution_name_prefix=execution_name_prefix, + options=options, + wait=wait, + type_hints=type_hints, + overwrite_cache=overwrite_cache, + envs=envs, + tags=tags, + cluster_pool=cluster_pool, + ) + if isinstance(entity, ReferenceWorkflow): + return self.execute_reference_workflow( + entity=entity, + inputs=inputs, + execution_name=execution_name, + execution_name_prefix=execution_name_prefix, + options=options, + wait=wait, + type_hints=type_hints, + overwrite_cache=overwrite_cache, + envs=envs, + tags=tags, + cluster_pool=cluster_pool, + ) + if isinstance(entity, ReferenceLaunchPlan): + return self.execute_reference_launch_plan( + entity=entity, + inputs=inputs, + execution_name=execution_name, + execution_name_prefix=execution_name_prefix, + options=options, + wait=wait, + type_hints=type_hints, + overwrite_cache=overwrite_cache, + envs=envs, + tags=tags, + cluster_pool=cluster_pool, + ) if isinstance(entity, PythonTask): return self.execute_local_task( entity=entity, @@ -1427,6 +1470,157 @@ def execute_remote_wf( cluster_pool=cluster_pool, ) + # Flyte Reference Entities + # --------------------- + def execute_reference_task( + self, + entity: ReferenceTask, + inputs: typing.Dict[str, typing.Any], + execution_name: typing.Optional[str] = None, + execution_name_prefix: typing.Optional[str] = None, + options: typing.Optional[Options] = None, + wait: bool = False, + type_hints: typing.Optional[typing.Dict[str, typing.Type]] = None, + overwrite_cache: typing.Optional[bool] = None, + envs: typing.Optional[typing.Dict[str, str]] = None, + tags: typing.Optional[typing.List[str]] = None, + cluster_pool: typing.Optional[str] = None, + ) -> FlyteWorkflowExecution: + """Execute a ReferenceTask.""" + resolved_identifiers = ResolvedIdentifiers( + project=entity.reference.project, + domain=entity.reference.domain, + name=entity.reference.name, + version=entity.reference.version, + ) + resolved_identifiers_dict = asdict(resolved_identifiers) + try: + flyte_task: FlyteTask = self.fetch_task(**resolved_identifiers_dict) + except FlyteEntityNotExistException: + raise ValueError( + f'missing entity of type ReferenceTask with identifier project:"{entity.reference.project}" domain:"{entity.reference.domain}" name:"{entity.reference.name}" version:"{entity.reference.version}"' + ) + + return self.execute( + flyte_task, + inputs, + project=resolved_identifiers.project, + domain=resolved_identifiers.domain, + execution_name=execution_name, + execution_name_prefix=execution_name_prefix, + options=options, + wait=wait, + type_hints=type_hints, + overwrite_cache=overwrite_cache, + envs=envs, + tags=tags, + cluster_pool=cluster_pool, + ) + + def execute_reference_workflow( + self, + entity: ReferenceWorkflow, + inputs: typing.Dict[str, typing.Any], + execution_name: typing.Optional[str] = None, + execution_name_prefix: typing.Optional[str] = None, + options: typing.Optional[Options] = None, + wait: bool = False, + type_hints: typing.Optional[typing.Dict[str, typing.Type]] = None, + overwrite_cache: typing.Optional[bool] = None, + envs: typing.Optional[typing.Dict[str, str]] = None, + tags: typing.Optional[typing.List[str]] = None, + cluster_pool: typing.Optional[str] = None, + ) -> FlyteWorkflowExecution: + """Execute a ReferenceWorkflow.""" + resolved_identifiers = ResolvedIdentifiers( + project=entity.reference.project, + domain=entity.reference.domain, + name=entity.reference.name, + version=entity.reference.version, + ) + resolved_identifiers_dict = asdict(resolved_identifiers) + try: + self.fetch_workflow(**resolved_identifiers_dict) + except FlyteEntityNotExistException: + raise ValueError( + f'missing entity of type ReferenceWorkflow with identifier project:"{entity.reference.project}" domain:"{entity.reference.domain}" name:"{entity.reference.name}" version:"{entity.reference.version}"' + ) + + try: + flyte_lp = self.fetch_launch_plan(**resolved_identifiers_dict) + except FlyteEntityNotExistException: + remote_logger.info("Try to register default launch plan because it wasn't found in Flyte Admin!") + default_lp = LaunchPlan.get_default_launch_plan(self.context, entity) + self.register_launch_plan( + default_lp, + project=resolved_identifiers.project, + domain=resolved_identifiers.domain, + version=resolved_identifiers.version, + options=options, + ) + flyte_lp = self.fetch_launch_plan(**resolved_identifiers_dict) + + return self.execute( + flyte_lp, + inputs, + project=resolved_identifiers.project, + domain=resolved_identifiers.domain, + execution_name=execution_name, + execution_name_prefix=execution_name_prefix, + wait=wait, + options=options, + type_hints=type_hints, + overwrite_cache=overwrite_cache, + envs=envs, + tags=tags, + cluster_pool=cluster_pool, + ) + + def execute_reference_launch_plan( + self, + entity: ReferenceLaunchPlan, + inputs: typing.Dict[str, typing.Any], + execution_name: typing.Optional[str] = None, + execution_name_prefix: typing.Optional[str] = None, + options: typing.Optional[Options] = None, + wait: bool = False, + type_hints: typing.Optional[typing.Dict[str, typing.Type]] = None, + overwrite_cache: typing.Optional[bool] = None, + envs: typing.Optional[typing.Dict[str, str]] = None, + tags: typing.Optional[typing.List[str]] = None, + cluster_pool: typing.Optional[str] = None, + ) -> FlyteWorkflowExecution: + """Execute a ReferenceLaunchPlan.""" + resolved_identifiers = ResolvedIdentifiers( + project=entity.reference.project, + domain=entity.reference.domain, + name=entity.reference.name, + version=entity.reference.version, + ) + resolved_identifiers_dict = asdict(resolved_identifiers) + try: + flyte_launchplan: FlyteLaunchPlan = self.fetch_launch_plan(**resolved_identifiers_dict) + except FlyteEntityNotExistException: + raise ValueError( + f'missing entity of type ReferenceLaunchPlan with identifier project:"{entity.reference.project}" domain:"{entity.reference.domain}" name:"{entity.reference.name}" version:"{entity.reference.version}"' + ) + + return self.execute( + flyte_launchplan, + inputs, + project=resolved_identifiers.project, + domain=resolved_identifiers.domain, + execution_name=execution_name, + execution_name_prefix=execution_name_prefix, + options=options, + wait=wait, + type_hints=type_hints, + overwrite_cache=overwrite_cache, + envs=envs, + tags=tags, + cluster_pool=cluster_pool, + ) + # Flytekit Entities # ----------------- diff --git a/tests/flytekit/integration/remote/test_remote.py b/tests/flytekit/integration/remote/test_remote.py index 3398c771f3..1e03b55098 100644 --- a/tests/flytekit/integration/remote/test_remote.py +++ b/tests/flytekit/integration/remote/test_remote.py @@ -11,6 +11,9 @@ from flytekit import LaunchPlan, kwtypes from flytekit.configuration import Config, ImageConfig, SerializationSettings +from flytekit.core.launch_plan import reference_launch_plan +from flytekit.core.task import reference_task +from flytekit.core.workflow import reference_workflow from flytekit.exceptions.user import FlyteAssertion, FlyteEntityNotExistException from flytekit.extras.sqlite3.task import SQLite3Config, SQLite3Task from flytekit.remote.remote import FlyteRemote @@ -352,3 +355,84 @@ def test_fetch_not_exist_launch_plan(register): remote = FlyteRemote(Config.auto(config_file=CONFIG), PROJECT, DOMAIN) with pytest.raises(FlyteEntityNotExistException): remote.fetch_launch_plan(name="basic.list_float_wf.fake_wf", version=VERSION) + + +def test_execute_reference_task(register): + @reference_task( + project=PROJECT, + domain=DOMAIN, + name="basic.basic_workflow.t1", + version=VERSION, + ) + def t1(a: int) -> typing.NamedTuple("OutputsBC", t1_int_output=int, c=str): + ... + + remote = FlyteRemote(Config.auto(config_file=CONFIG), PROJECT, DOMAIN) + execution = remote.execute( + t1, + inputs={"a": 10}, + wait=True, + overwrite_cache=True, + envs={"foo": "bar"}, + tags=["flyte"], + cluster_pool="gpu", + ) + assert execution.outputs["t1_int_output"] == 12 + assert execution.outputs["c"] == "world" + assert execution.spec.envs.envs == {"foo": "bar"} + assert execution.spec.tags == ["flyte"] + assert execution.spec.cluster_assignment.cluster_pool == "gpu" + + +def test_execute_reference_workflow(register): + @reference_workflow( + project=PROJECT, + domain=DOMAIN, + name="basic.basic_workflow.my_wf", + version=VERSION, + ) + def my_wf(a: int, b: str) -> (int, str): + ... + + remote = FlyteRemote(Config.auto(config_file=CONFIG), PROJECT, DOMAIN) + execution = remote.execute( + my_wf, + inputs={"a": 10, "b": "xyz"}, + wait=True, + overwrite_cache=True, + envs={"foo": "bar"}, + tags=["flyte"], + cluster_pool="gpu", + ) + assert execution.outputs["o0"] == 12 + assert execution.outputs["o1"] == "xyzworld" + assert execution.spec.envs.envs == {"foo": "bar"} + assert execution.spec.tags == ["flyte"] + assert execution.spec.cluster_assignment.cluster_pool == "gpu" + + +def test_execute_reference_launchplan(register): + @reference_launch_plan( + project=PROJECT, + domain=DOMAIN, + name="basic.basic_workflow.my_wf", + version=VERSION, + ) + def my_wf(a: int, b: str) -> (int, str): + ... + + remote = FlyteRemote(Config.auto(config_file=CONFIG), PROJECT, DOMAIN) + execution = remote.execute( + my_wf, + inputs={"a": 10, "b": "xyz"}, + wait=True, + overwrite_cache=True, + envs={"foo": "bar"}, + tags=["flyte"], + cluster_pool="gpu", + ) + assert execution.outputs["o0"] == 12 + assert execution.outputs["o1"] == "xyzworld" + assert execution.spec.envs.envs == {"foo": "bar"} + assert execution.spec.tags == ["flyte"] + assert execution.spec.cluster_assignment.cluster_pool == "gpu" From 08fa982823b532800e56cbbaa955fe8cbd589b45 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 27 Mar 2024 13:56:18 -0700 Subject: [PATCH 22/50] test: Add serial marker to cache related tests (#2300) --- tests/flytekit/unit/core/test_local_cache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py index 6a9570fd7a..4799ce3f64 100644 --- a/tests/flytekit/unit/core/test_local_cache.py +++ b/tests/flytekit/unit/core/test_local_cache.py @@ -600,6 +600,7 @@ def test_checkpoint_cached_task(): assert t2(n=5) == 6 +@pytest.mark.serial def test_cache_ignore_input_vars(): @task(cache=True, cache_version="v1", cache_ignore_input_vars=["a"]) def add(a: int, b: int) -> int: @@ -614,6 +615,7 @@ def add_wf(a: int, b: int) -> int: assert add_wf(a=20, b=8) == 28 +@pytest.mark.serial def test_set_cache_ignore_input_vars_without_set_cache(): with pytest.raises( ValueError, From 312c92f5119b24ee4b6d2b4002a30812f9dbf1ce Mon Sep 17 00:00:00 2001 From: Greg Gydush <35151789+ggydush@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:37:42 -0700 Subject: [PATCH 23/50] feat: Support pep604 union operator (#2298) * feat: Support pep604 union operator Signed-off-by: ggydush * fix: check union early Signed-off-by: ggydush * refactor: Change name to 604 Signed-off-by: ggydush * fix lint Signed-off-by: ggydush * fix: Fix duplicated code Signed-off-by: ggydush * fix: Remove code for testing Signed-off-by: ggydush * test: Add simple tests Signed-off-by: ggydush * Add more tests Signed-off-by: ggydush * fix: Fix names Signed-off-by: ggydush * fix: Lint Signed-off-by: ggydush * fix: Fix again Signed-off-by: ggydush * fix: Fix default Signed-off-by: ggydush * test: Add test for parameter and defaults Signed-off-by: ggydush * fix: Fix code coverage by ignoring Signed-off-by: ggydush * refactor: Use is_union_type Signed-off-by: ggydush * fix import sort Signed-off-by: ggydush * fix: cleanup Signed-off-by: ggydush * fix: fix Signed-off-by: ggydush * refactor: Clean it up Signed-off-by: ggydush * fix: Fix lint Signed-off-by: ggydush * fix: Fix pydantic plugin test failure Signed-off-by: ggydush * Update flytekit/core/type_engine.py Co-authored-by: Kevin Su * Address comment Signed-off-by: ggydush * fix: Use UnionTransformer Signed-off-by: ggydush * fix: Fix lint Signed-off-by: ggydush * Skip tests with | syntax on < 3.10 Signed-off-by: ggydush * fix: Fix test Signed-off-by: ggydush * fix: More review comments Signed-off-by: ggydush * fix: Fix lint Signed-off-by: ggydush --------- Signed-off-by: ggydush Co-authored-by: Kevin Su --- flytekit/core/interface.py | 6 +++--- flytekit/core/type_engine.py | 20 ++++++++++++++----- .../flytekitplugins/pydantic/commons.py | 5 ++++- pyproject.toml | 1 + tests/flytekit/unit/core/test_interface.py | 14 +++++++++++++ tests/flytekit/unit/core/test_type_engine.py | 19 +++++++++++++++++- 6 files changed, 55 insertions(+), 10 deletions(-) diff --git a/flytekit/core/interface.py b/flytekit/core/interface.py index 8a92d58591..aecca2936d 100644 --- a/flytekit/core/interface.py +++ b/flytekit/core/interface.py @@ -8,13 +8,13 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Type, TypeVar, Union, cast from flyteidl.core import artifact_id_pb2 as art_id -from typing_extensions import get_args, get_origin, get_type_hints +from typing_extensions import get_args, get_type_hints from flytekit.core import context_manager from flytekit.core.artifact import Artifact, ArtifactIDSpecification, ArtifactQuery from flytekit.core.docstring import Docstring from flytekit.core.sentinel import DYNAMIC_INPUT_BINDING -from flytekit.core.type_engine import TypeEngine +from flytekit.core.type_engine import TypeEngine, UnionTransformer from flytekit.exceptions.user import FlyteValidationException from flytekit.loggers import logger from flytekit.models import interface as _interface_models @@ -218,7 +218,7 @@ def transform_inputs_to_parameters( inputs_with_def = interface.inputs_with_defaults for k, v in inputs_vars.items(): val, _default = inputs_with_def[k] - if _default is None and get_origin(val) is typing.Union and type(None) in get_args(val): + if _default is None and UnionTransformer.is_optional_type(val): literal = Literal(scalar=Scalar(none_type=Void())) params[k] = _interface_models.Parameter(var=v, default=literal, required=False) else: diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 9153fca032..df90991b3c 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -9,6 +9,7 @@ import json import json as _json import mimetypes +import sys import textwrap import typing from abc import ABC, abstractmethod @@ -26,6 +27,7 @@ from marshmallow_enum import EnumField, LoadDumpOptions from mashumaro.mixins.json import DataClassJSONMixin from typing_extensions import Annotated, get_args, get_origin +from typing_inspect import is_union_type from flytekit.core.annotation import FlyteAnnotation from flytekit.core.context_manager import FlyteContext @@ -547,7 +549,7 @@ def _serialize_flyte_type(self, python_val: T, python_type: Type[T]) -> typing.A from flytekit.types.structured.structured_dataset import StructuredDataset # Handle Optional - if get_origin(python_type) is typing.Union and type(None) in get_args(python_type): + if UnionTransformer.is_optional_type(python_type): if python_val is None: return None return self._serialize_flyte_type(python_val, get_args(python_type)[0]) @@ -600,7 +602,7 @@ def _deserialize_flyte_type(self, python_val: T, expected_python_type: Type) -> from flytekit.types.structured.structured_dataset import StructuredDataset, StructuredDatasetTransformerEngine # Handle Optional - if get_origin(expected_python_type) is typing.Union and type(None) in get_args(expected_python_type): + if UnionTransformer.is_optional_type(expected_python_type): if python_val is None: return None return self._deserialize_flyte_type(python_val, get_args(expected_python_type)[0]) @@ -694,7 +696,7 @@ def _fix_val_int(self, t: typing.Type, val: typing.Any) -> typing.Any: if val is None: return val - if get_origin(t) is typing.Union and type(None) in get_args(t): + if UnionTransformer.is_optional_type(t): # Handle optional type. e.g. Optional[int], Optional[dataclass] # Marshmallow doesn't support union type, so the type here is always an optional type. # https://github.com/marshmallow-code/marshmallow/issues/1191#issuecomment-480831796 @@ -961,6 +963,9 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: Step 5: if v is of type data class, use the dataclass transformer + + Step 6: + Pickle transformer is used """ cls.lazy_import_transformers() # Step 1 @@ -1496,7 +1501,7 @@ def __init__(self): @staticmethod def is_optional_type(t: Type[T]) -> bool: - return get_origin(t) is typing.Union and type(None) in get_args(t) + return is_union_type(t) and type(None) in get_args(t) @staticmethod def get_sub_type_in_optional(t: Type[T]) -> Type[T]: @@ -1968,7 +1973,12 @@ def _register_default_type_transformers(): [None], ) TypeEngine.register(ListTransformer()) - TypeEngine.register(UnionTransformer()) + if sys.version_info >= (3, 10): + from types import UnionType + + TypeEngine.register(UnionTransformer(), [UnionType]) + else: + TypeEngine.register(UnionTransformer()) TypeEngine.register(DictTransformer()) TypeEngine.register(TextIOTransformer()) TypeEngine.register(BinaryIOTransformer()) diff --git a/plugins/flytekit-pydantic/flytekitplugins/pydantic/commons.py b/plugins/flytekit-pydantic/flytekitplugins/pydantic/commons.py index 79d56d8354..36358ddbd8 100644 --- a/plugins/flytekit-pydantic/flytekitplugins/pydantic/commons.py +++ b/plugins/flytekit-pydantic/flytekitplugins/pydantic/commons.py @@ -1,5 +1,6 @@ import builtins import datetime +import types import typing from typing import Set @@ -11,7 +12,9 @@ numpy = lazy_module("numpy") pyarrow = lazy_module("pyarrow") -MODULES_TO_EXCLUDE_FROM_FLYTE_TYPES: Set[str] = {m.__name__ for m in [builtins, typing, datetime, pyarrow, numpy]} +MODULES_TO_EXCLUDE_FROM_FLYTE_TYPES: Set[str] = { + m.__name__ for m in [builtins, types, typing, datetime, pyarrow, numpy] +} def include_in_flyte_types(t: type) -> bool: diff --git a/pyproject.toml b/pyproject.toml index df457ac3ad..5678d342b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "s3fs>=2023.3.0,!=2024.3.1", "statsd>=3.0.0,<4.0.0", "typing_extensions", + "typing-inspect", "urllib3>=1.22,<2.0.0", ] classifiers = [ diff --git a/tests/flytekit/unit/core/test_interface.py b/tests/flytekit/unit/core/test_interface.py index b8f743118b..ecf25fb6e0 100644 --- a/tests/flytekit/unit/core/test_interface.py +++ b/tests/flytekit/unit/core/test_interface.py @@ -1,4 +1,5 @@ import os +import sys import typing from typing import Dict, List @@ -156,6 +157,7 @@ def t1() -> FlyteFile[typing.TypeVar("svg")]: assert return_type["o0"].extension() == FlyteFile[typing.TypeVar("svg")].extension() +@pytest.mark.skipif(sys.version_info < (3, 10), reason="PEP604 requires >=3.10.") def test_parameters_and_defaults(): ctx = context_manager.FlyteContext.current_context() @@ -213,6 +215,18 @@ def z( assert not params.parameters["c"].required assert params.parameters["c"].default.scalar.none_type == Void() + def z(a: int | None = None, b: str | None = None, c: typing.List[int] | None = None) -> typing.Tuple[int, str]: + ... + + our_interface = transform_function_to_interface(z) + params = transform_inputs_to_parameters(ctx, our_interface) + assert not params.parameters["a"].required + assert params.parameters["a"].default.scalar.none_type == Void() + assert not params.parameters["b"].required + assert params.parameters["b"].default.scalar.none_type == Void() + assert not params.parameters["c"].required + assert params.parameters["c"].default.scalar.none_type == Void() + def test_parameters_with_docstring(): ctx = context_manager.FlyteContext.current_context() diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index b9c9dac8f2..b94249f804 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -1368,9 +1368,13 @@ def union_type_tags_unique(t: LiteralType): return True +@pytest.mark.skipif(sys.version_info < (3, 10), reason="PEP604 requires >=3.10.") def test_union_type(): pt = typing.Union[str, int] lt = TypeEngine.to_literal_type(pt) + pt_604 = str | int + lt_604 = TypeEngine.to_literal_type(pt_604) + assert lt == lt_604 assert lt.union_type.variants == [ LiteralType(simple=SimpleType.STRING, structure=TypeStructure(tag="str")), LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="int")), @@ -1526,10 +1530,13 @@ class Bar(DataClassJSONMixin): DataclassTransformer().assert_type(gt, pv) +@pytest.mark.skipif(sys.version_info < (3, 10), reason="PEP604 requires >=3.10.") def test_union_transformer(): assert UnionTransformer.is_optional_type(typing.Optional[int]) + assert UnionTransformer.is_optional_type(int | None) assert not UnionTransformer.is_optional_type(str) assert UnionTransformer.get_sub_type_in_optional(typing.Optional[int]) == int + assert UnionTransformer.get_sub_type_in_optional(int | None) == int def test_union_guess_type(): @@ -1597,9 +1604,13 @@ def test_annotated_union_type(): assert v == "hello" +@pytest.mark.skipif(sys.version_info < (3, 10), reason="PEP604 requires >=3.10.") def test_optional_type(): pt = typing.Optional[int] lt = TypeEngine.to_literal_type(pt) + pt_604 = int | None + lt_604 = TypeEngine.to_literal_type(pt_604) + assert lt == lt_604 assert lt.union_type.variants == [ LiteralType(simple=SimpleType.INTEGER, structure=TypeStructure(tag="int")), LiteralType(simple=SimpleType.NONE, structure=TypeStructure(tag="none")), @@ -1791,9 +1802,13 @@ def test_union_of_lists(): assert v == [1, 3] +@pytest.mark.skipif(sys.version_info < (3, 10), reason="PEP604 requires >=3.10.") def test_list_of_unions(): pt = typing.List[typing.Union[str, int]] lt = TypeEngine.to_literal_type(pt) + pt_604 = typing.List[str | int] + lt_604 = TypeEngine.to_literal_type(pt_604) + assert lt == lt_604 # todo(maximsmol): seems like the order here is non-deterministic assert lt.collection_type.union_type.variants == [ LiteralType(simple=SimpleType.STRING, structure=TypeStructure(tag="str")), @@ -1804,8 +1819,10 @@ def test_list_of_unions(): ctx = FlyteContextManager.current_context() lv = TypeEngine.to_literal(ctx, ["hello", 123, "world"], pt, lt) v = TypeEngine.to_python_value(ctx, lv, pt) + lv_604 = TypeEngine.to_literal(ctx, ["hello", 123, "world"], pt_604, lt_604) + v_604 = TypeEngine.to_python_value(ctx, lv_604, pt_604) assert [x.scalar.union.stored_type.structure.tag for x in lv.collection.literals] == ["str", "int", "str"] - assert v == ["hello", 123, "world"] + assert v == v_604 == ["hello", 123, "world"] def test_pickle_type(): From 8052c81a5aa58226b3ca768488f3729744f0bf3f Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Thu, 28 Mar 2024 12:25:38 -0700 Subject: [PATCH 24/50] Partition limit (#2301) Add a partition limit. Signed-off-by: Yee Hing Tong --- flytekit/core/artifact.py | 4 ++++ tests/flytekit/unit/core/test_artifacts.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/flytekit/core/artifact.py b/flytekit/core/artifact.py index 742270a9ae..e9a7909809 100644 --- a/flytekit/core/artifact.py +++ b/flytekit/core/artifact.py @@ -15,6 +15,7 @@ from flytekit.loggers import logger TIME_PARTITION_KWARG = "time_partition" +MAX_PARTITIONS = 10 class InputsBase(object): @@ -337,6 +338,9 @@ def __init__( self._partitions = Partitions(p) self._partitions.set_reference_artifact(self) + if self.partition_keys and len(self.partition_keys) > MAX_PARTITIONS: + raise ValueError("There is a hard limit of 10 partition keys per artifact currently.") + def __call__(self, *args, **kwargs) -> ArtifactIDSpecification: """ This __call__ should only ever happen in the context of a task or workflow's output, to be diff --git a/tests/flytekit/unit/core/test_artifacts.py b/tests/flytekit/unit/core/test_artifacts.py index ea3734f8aa..1580832426 100644 --- a/tests/flytekit/unit/core/test_artifacts.py +++ b/tests/flytekit/unit/core/test_artifacts.py @@ -583,3 +583,9 @@ def test_tp_math(): assert tp2.other == datetime.timedelta(days=1) assert tp2.granularity == Granularity.HOUR assert tp2 is not tp + + +def test_lims(): + # test an artifact with 11 partition keys + with pytest.raises(ValueError): + Artifact(name="test artifact", time_partitioned=True, partition_keys=[f"key_{i}" for i in range(11)]) From 8ab9a3c7c3ad2064296b6d1282953c77cbc26728 Mon Sep 17 00:00:00 2001 From: Nikki Everett Date: Thu, 28 Mar 2024 16:09:59 -0500 Subject: [PATCH 25/50] make artifact documentation visible (#2302) Signed-off-by: nikki everett --- flytekit/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index 8fc0311e6d..63ad935b47 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -179,6 +179,16 @@ HashMethod +Artifacts +========= + +.. autosummary:: + :nosignatures: + :template: custom.rst + :toctree: generated/ + + Artifact + Documentation ============= @@ -207,6 +217,7 @@ from flytekit._version import __version__ from flytekit.core.array_node_map_task import map_task +from flytekit.core.artifact import Artifact from flytekit.core.base_sql_task import SQLTask from flytekit.core.base_task import SecurityContext, TaskMetadata, kwtypes from flytekit.core.checkpointer import Checkpoint From 6c917ed16265e00f8d33339b6b78235a395b99dd Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 29 Mar 2024 13:19:41 +0530 Subject: [PATCH 26/50] remove secrets in sagemaker agent (#2308) Signed-off-by: Samhita Alla --- .../flytekitplugins/awssagemaker_inference/agent.py | 11 +---------- .../awssagemaker_inference/boto3_agent.py | 4 ---- .../awssagemaker_inference/boto3_mixin.py | 6 ------ .../flytekit-aws-sagemaker/tests/test_boto3_agent.py | 6 +----- .../tests/test_inference_agent.py | 6 +----- 5 files changed, 3 insertions(+), 30 deletions(-) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py index ecd283fed8..831c70135c 100644 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py @@ -7,7 +7,7 @@ AsyncAgentBase, Resource, ) -from flytekit.extend.backend.utils import convert_to_flyte_phase, get_agent_secret +from flytekit.extend.backend.utils import convert_to_flyte_phase from flytekit.models.literals import LiteralMap from flytekit.models.task import TaskTemplate @@ -53,9 +53,6 @@ async def create( config=config, inputs=inputs, region=region, - aws_access_key_id=get_agent_secret(secret_key="aws-access-key"), - aws_secret_access_key=get_agent_secret(secret_key="aws-secret-access-key"), - aws_session_token=get_agent_secret(secret_key="aws-session-token"), ) return SageMakerEndpointMetadata(config=config, region=region, inputs=inputs) @@ -66,9 +63,6 @@ async def get(self, resource_meta: SageMakerEndpointMetadata, **kwargs) -> Resou config={"EndpointName": resource_meta.config.get("EndpointName")}, inputs=resource_meta.inputs, region=resource_meta.region, - aws_access_key_id=get_agent_secret(secret_key="aws-access-key"), - aws_secret_access_key=get_agent_secret(secret_key="aws-secret-access-key"), - aws_session_token=get_agent_secret(secret_key="aws-session-token"), ) current_state = endpoint_status.get("EndpointStatus") @@ -90,9 +84,6 @@ async def delete(self, resource_meta: SageMakerEndpointMetadata, **kwargs): config={"EndpointName": resource_meta.config.get("EndpointName")}, region=resource_meta.region, inputs=resource_meta.inputs, - aws_access_key_id=get_agent_secret(secret_key="aws-access-key"), - aws_secret_access_key=get_agent_secret(secret_key="aws-secret-access-key"), - aws_session_token=get_agent_secret(secret_key="aws-session-token"), ) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_agent.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_agent.py index adb1772248..ca605a103d 100644 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_agent.py +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_agent.py @@ -7,7 +7,6 @@ Resource, SyncAgentBase, ) -from flytekit.extend.backend.utils import get_agent_secret from flytekit.models.literals import LiteralMap from flytekit.models.task import TaskTemplate @@ -53,9 +52,6 @@ async def do(self, task_template: TaskTemplate, inputs: Optional[LiteralMap] = N config=config, images=images, inputs=inputs, - aws_access_key_id=get_agent_secret(secret_key="aws-access-key"), - aws_secret_access_key=get_agent_secret(secret_key="aws-secret-access-key"), - aws_session_token=get_agent_secret(secret_key="aws-session-token"), ) outputs = None diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py index 045124afd0..1daa16bc73 100644 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py @@ -116,9 +116,6 @@ async def _call( images: Optional[Dict[str, str]] = None, inputs: Optional[LiteralMap] = None, region: Optional[str] = None, - aws_access_key_id: Optional[str] = None, - aws_secret_access_key: Optional[str] = None, - aws_session_token: Optional[str] = None, ) -> Any: """ Utilize this method to invoke any boto3 method (AWS service method). @@ -173,9 +170,6 @@ async def _call( async with session.client( service_name=self._service, region_name=final_region, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - aws_session_token=aws_session_token, ) as client: try: result = await getattr(client, method)(**updated_config) diff --git a/plugins/flytekit-aws-sagemaker/tests/test_boto3_agent.py b/plugins/flytekit-aws-sagemaker/tests/test_boto3_agent.py index 7be62e216c..2974711f88 100644 --- a/plugins/flytekit-aws-sagemaker/tests/test_boto3_agent.py +++ b/plugins/flytekit-aws-sagemaker/tests/test_boto3_agent.py @@ -12,10 +12,6 @@ @pytest.mark.asyncio -@mock.patch( - "flytekitplugins.awssagemaker_inference.boto3_agent.get_agent_secret", - return_value="mocked_secret", -) @mock.patch( "flytekitplugins.awssagemaker_inference.boto3_agent.Boto3AgentMixin._call", return_value={ @@ -33,7 +29,7 @@ "EndpointConfigArn": "arn:aws:sagemaker:us-east-2:000000000:endpoint-config/sagemaker-xgboost-endpoint-config", }, ) -async def test_agent(mock_boto_call, mock_secret): +async def test_agent(mock_boto_call): agent = AgentRegistry.get_agent("boto") task_id = Identifier( resource_type=ResourceType.TASK, diff --git a/plugins/flytekit-aws-sagemaker/tests/test_inference_agent.py b/plugins/flytekit-aws-sagemaker/tests/test_inference_agent.py index e4003c0735..5ee8d11f01 100644 --- a/plugins/flytekit-aws-sagemaker/tests/test_inference_agent.py +++ b/plugins/flytekit-aws-sagemaker/tests/test_inference_agent.py @@ -14,10 +14,6 @@ @pytest.mark.asyncio -@mock.patch( - "flytekitplugins.awssagemaker_inference.agent.get_agent_secret", - return_value="mocked_secret", -) @mock.patch( "flytekitplugins.awssagemaker_inference.agent.Boto3AgentMixin._call", return_value={ @@ -59,7 +55,7 @@ }, }, ) -async def test_agent(mock_boto_call, mock_secret): +async def test_agent(mock_boto_call): agent = AgentRegistry.get_agent("sagemaker-endpoint") task_id = Identifier( resource_type=ResourceType.TASK, From d0747861aed548ac7639083a80d67f0ea2755261 Mon Sep 17 00:00:00 2001 From: ByronHsu Date: Fri, 29 Mar 2024 10:39:59 -0700 Subject: [PATCH 27/50] Adapt flytekit ray plugin to kuberay 1.1.0 (#2274) * runtime env yaml Signed-off-by: Pin-Lun Hsu * fix test Signed-off-by: Pin-Lun Hsu * fix fmt Signed-off-by: Pin-Lun Hsu * Update setup.py Signed-off-by: Pin-Lun Hsu --------- Signed-off-by: Pin-Lun Hsu Co-authored-by: Pin-Lun Hsu --- plugins/flytekit-ray/flytekitplugins/ray/models.py | 10 +++++++++- plugins/flytekit-ray/flytekitplugins/ray/task.py | 10 ++++++++-- plugins/flytekit-ray/tests/test_ray.py | 2 ++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/plugins/flytekit-ray/flytekitplugins/ray/models.py b/plugins/flytekit-ray/flytekitplugins/ray/models.py index 06e36af186..81517e2218 100644 --- a/plugins/flytekit-ray/flytekitplugins/ray/models.py +++ b/plugins/flytekit-ray/flytekitplugins/ray/models.py @@ -191,12 +191,14 @@ class RayJob(_common.FlyteIdlEntity): def __init__( self, ray_cluster: RayCluster, - runtime_env: typing.Optional[str], + runtime_env: typing.Optional[str] = None, + runtime_env_yaml: typing.Optional[str] = None, ttl_seconds_after_finished: typing.Optional[int] = None, shutdown_after_job_finishes: bool = False, ): self._ray_cluster = ray_cluster self._runtime_env = runtime_env + self._runtime_env_yaml = runtime_env_yaml self._ttl_seconds_after_finished = ttl_seconds_after_finished self._shutdown_after_job_finishes = shutdown_after_job_finishes @@ -208,6 +210,10 @@ def ray_cluster(self) -> RayCluster: def runtime_env(self) -> typing.Optional[str]: return self._runtime_env + @property + def runtime_env_yaml(self) -> typing.Optional[str]: + return self._runtime_env_yaml + @property def ttl_seconds_after_finished(self) -> typing.Optional[int]: # ttl_seconds_after_finished specifies the number of seconds after which the RayCluster will be deleted after the RayJob finishes. @@ -222,6 +228,7 @@ def to_flyte_idl(self) -> _ray_pb2.RayJob: return _ray_pb2.RayJob( ray_cluster=self.ray_cluster.to_flyte_idl(), runtime_env=self.runtime_env, + runtime_env_yaml=self.runtime_env_yaml, ttl_seconds_after_finished=self.ttl_seconds_after_finished, shutdown_after_job_finishes=self.shutdown_after_job_finishes, ) @@ -231,6 +238,7 @@ def from_flyte_idl(cls, proto: _ray_pb2.RayJob): return cls( ray_cluster=RayCluster.from_flyte_idl(proto.ray_cluster) if proto.ray_cluster else None, runtime_env=proto.runtime_env, + runtime_env_yaml=proto.runtime_env_yaml, ttl_seconds_after_finished=proto.ttl_seconds_after_finished, shutdown_after_job_finishes=proto.shutdown_after_job_finishes, ) diff --git a/plugins/flytekit-ray/flytekitplugins/ray/task.py b/plugins/flytekit-ray/flytekitplugins/ray/task.py index 76688d74cd..49ded26bfd 100644 --- a/plugins/flytekit-ray/flytekitplugins/ray/task.py +++ b/plugins/flytekit-ray/flytekitplugins/ray/task.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, Optional +import yaml from flytekitplugins.ray.models import HeadGroupSpec, RayCluster, RayJob, WorkerGroupSpec from google.protobuf.json_format import MessageToDict @@ -63,6 +64,11 @@ def post_execute(self, user_params: ExecutionParameters, rval: Any) -> Any: def get_custom(self, settings: SerializationSettings) -> Optional[Dict[str, Any]]: cfg = self._task_config + # Deprecated: runtime_env is removed KubeRay >= 1.1.0. It is replaced by runtime_env_yaml + runtime_env = base64.b64encode(json.dumps(cfg.runtime_env).encode()).decode() if cfg.runtime_env else None + + runtime_env_yaml = yaml.dump(cfg.runtime_env) if cfg.runtime_env else None + ray_job = RayJob( ray_cluster=RayCluster( head_group_spec=HeadGroupSpec(cfg.head_node_config.ray_start_params) if cfg.head_node_config else None, @@ -72,8 +78,8 @@ def get_custom(self, settings: SerializationSettings) -> Optional[Dict[str, Any] ], enable_autoscaling=cfg.enable_autoscaling if cfg.enable_autoscaling else False, ), - # Use base64 to encode runtime_env dict and convert it to byte string - runtime_env=base64.b64encode(json.dumps(cfg.runtime_env).encode()).decode(), + runtime_env=runtime_env, + runtime_env_yaml=runtime_env_yaml, ttl_seconds_after_finished=cfg.ttl_seconds_after_finished, shutdown_after_job_finishes=cfg.shutdown_after_job_finishes, ) diff --git a/plugins/flytekit-ray/tests/test_ray.py b/plugins/flytekit-ray/tests/test_ray.py index 0c0ada1944..6fad11dd3e 100644 --- a/plugins/flytekit-ray/tests/test_ray.py +++ b/plugins/flytekit-ray/tests/test_ray.py @@ -2,6 +2,7 @@ import json import ray +import yaml from flytekitplugins.ray.models import RayCluster, RayJob, WorkerGroupSpec from flytekitplugins.ray.task import RayJobConfig, WorkerNodeConfig from google.protobuf.json_format import MessageToDict @@ -42,6 +43,7 @@ def t1(a: int) -> str: ray_job_pb = RayJob( ray_cluster=RayCluster(worker_group_spec=[WorkerGroupSpec("test_group", 3, 0, 10)], enable_autoscaling=True), runtime_env=base64.b64encode(json.dumps({"pip": ["numpy"]}).encode()).decode(), + runtime_env_yaml=yaml.dump({"pip": ["numpy"]}), shutdown_after_job_finishes=True, ttl_seconds_after_finished=20, ).to_flyte_idl() From 9aaf16077a2fb985e83b01711dcfca0ce888ea76 Mon Sep 17 00:00:00 2001 From: Yun Dai Date: Fri, 29 Mar 2024 12:56:55 -0700 Subject: [PATCH 28/50] Remove post-execute hook for ray task (#2305) Signed-off-by: Yun Dai --- plugins/flytekit-ray/flytekitplugins/ray/task.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/flytekit-ray/flytekitplugins/ray/task.py b/plugins/flytekit-ray/flytekitplugins/ray/task.py index 49ded26bfd..e6b3ad8039 100644 --- a/plugins/flytekit-ray/flytekitplugins/ray/task.py +++ b/plugins/flytekit-ray/flytekitplugins/ray/task.py @@ -57,10 +57,6 @@ def pre_execute(self, user_params: ExecutionParameters) -> ExecutionParameters: ray.init(address=self._task_config.address) return user_params - def post_execute(self, user_params: ExecutionParameters, rval: Any) -> Any: - ray.shutdown() - return rval - def get_custom(self, settings: SerializationSettings) -> Optional[Dict[str, Any]]: cfg = self._task_config From acf6bab70ec1c4770b586b20aad6da356c0fbc38 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Fri, 29 Mar 2024 13:00:08 -0700 Subject: [PATCH 29/50] Gz encoding (#2306) * wip, make a sandbox test Signed-off-by: Yee Hing Tong * gzip encoding Signed-off-by: Yee Hing Tong * revert Signed-off-by: Yee Hing Tong * fix test Signed-off-by: Yee Hing Tong * lint Signed-off-by: Yee Hing Tong * test Signed-off-by: Yee Hing Tong --------- Signed-off-by: Yee Hing Tong --- flytekit/types/file/file.py | 11 ++++++-- tests/flytekit/unit/core/test_data.py | 28 +++++++++++++++++++++ tests/flytekit/unit/core/test_flyte_file.py | 5 ++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index de4e49cdaf..73a835a20f 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -446,15 +446,22 @@ def to_literal( # If we're uploading something, that means that the uri should always point to the upload destination. if should_upload: + headers = self.get_additional_headers(source_path) if remote_path is not None: - remote_path = ctx.file_access.put_data(source_path, remote_path, is_multipart=False) + remote_path = ctx.file_access.put_data(source_path, remote_path, is_multipart=False, **headers) else: - remote_path = ctx.file_access.put_raw_data(source_path) + remote_path = ctx.file_access.put_raw_data(source_path, **headers) return Literal(scalar=Scalar(blob=Blob(metadata=meta, uri=remote_path))) # If not uploading, then we can only take the original source path as the uri. else: return Literal(scalar=Scalar(blob=Blob(metadata=meta, uri=source_path))) + @staticmethod + def get_additional_headers(source_path: str | os.PathLike) -> typing.Dict[str, str]: + if str(source_path).endswith(".gz"): + return {"ContentEncoding": "gzip"} + return {} + def to_python_value( self, ctx: FlyteContext, lv: Literal, expected_python_type: typing.Union[typing.Type[FlyteFile], os.PathLike] ) -> FlyteFile: diff --git a/tests/flytekit/unit/core/test_data.py b/tests/flytekit/unit/core/test_data.py index 99963621a7..aa308d7929 100644 --- a/tests/flytekit/unit/core/test_data.py +++ b/tests/flytekit/unit/core/test_data.py @@ -12,7 +12,9 @@ from flytekit.configuration import Config, DataConfig, S3Config from flytekit.core.context_manager import FlyteContextManager from flytekit.core.data_persistence import FileAccessProvider, get_fsspec_storage_options, s3_setup_args +from flytekit.core.type_engine import TypeEngine from flytekit.types.directory.types import FlyteDirectory +from flytekit.types.file import FlyteFile local = fsspec.filesystem("file") root = os.path.abspath(os.sep) @@ -418,3 +420,29 @@ def test_walk_local_copy_to_s3(source_folder): new_crawl = fd.crawl() new_suffixes = [y for x, y in new_crawl] assert len(new_suffixes) == 2 # should have written two files + + +@pytest.mark.sandbox_test +def test_s3_metadata(): + dc = Config.for_sandbox().data_config + random_folder = UUID(int=random.getrandbits(64)).hex + raw_output = f"s3://my-s3-bucket/testing/metadata_test/{random_folder}" + provider = FileAccessProvider(local_sandbox_dir="/tmp/unittest", raw_output_prefix=raw_output, data_config=dc) + _, local_zip = tempfile.mkstemp(suffix=".gz") + with open(local_zip, "w") as f: + f.write("hello world") + + # Test writing file + ff = FlyteFile(path=local_zip) + ff2 = FlyteFile(path=local_zip, remote_path=f"{raw_output}/test.gz") + ctx = FlyteContextManager.current_context() + with FlyteContextManager.with_context(ctx.with_file_access(provider)) as ctx: + lt = TypeEngine.to_literal_type(FlyteFile) + TypeEngine.to_literal(ctx, ff, FlyteFile, lt) + TypeEngine.to_literal(ctx, ff2, FlyteFile, lt) + + fd = FlyteDirectory(path=raw_output) + res = fd.crawl() + res = [(x, y) for x, y in res] + files = [os.path.join(x, y) for x, y in res] + assert len(files) == 2 diff --git a/tests/flytekit/unit/core/test_flyte_file.py b/tests/flytekit/unit/core/test_flyte_file.py index a12c414f35..420279128b 100644 --- a/tests/flytekit/unit/core/test_flyte_file.py +++ b/tests/flytekit/unit/core/test_flyte_file.py @@ -644,3 +644,8 @@ def test_join(): fs = ctx.file_access.get_filesystem("s3") f = ctx.file_access.join("s3://a", "b", "c", fs=fs) assert f == fs.sep.join(["s3://a", "b", "c"]) + + +def test_headers(): + assert FlyteFilePathTransformer.get_additional_headers("xyz") == {} + assert len(FlyteFilePathTransformer.get_additional_headers(".gz")) == 1 From 60c6234306137d1e620df04c76ab31280e48b844 Mon Sep 17 00:00:00 2001 From: novahow <58504997+novahow@users.noreply.github.com> Date: Sat, 30 Mar 2024 05:07:20 +0800 Subject: [PATCH 30/50] Fix limit option (#2292) https://github.com/flyteorg/flyte/issues/5086 1. removed `--limit` option in `RemoteEntityGroup` 2. set `--limit` in `RunLevelParams` to show up instead of hidden. Signed-off-by: novahow --- flytekit/clis/sdk_in_container/run.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/flytekit/clis/sdk_in_container/run.py b/flytekit/clis/sdk_in_container/run.py index e2887ce51a..9f4effe3eb 100644 --- a/flytekit/clis/sdk_in_container/run.py +++ b/flytekit/clis/sdk_in_container/run.py @@ -242,7 +242,6 @@ class RunLevelParams(PyFlyteParams): required=False, type=int, default=50, - hidden=True, show_default=True, help="Use this to limit number of entities to fetch", ) @@ -661,14 +660,6 @@ def __init__(self, command_name: str): super().__init__( name=command_name, help=f"Retrieve {command_name} from a remote flyte instance and execute them.", - params=[ - click.Option( - ["--limit", "limit"], - help=f"Limit the number of {command_name}'s to retrieve.", - default=50, - show_default=True, - ) - ], ) self._command_name = command_name self._entities = [] From aaee58ab44d1255e53ef094dde5200c4bdc5b767 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 29 Mar 2024 14:24:14 -0700 Subject: [PATCH 31/50] Move SageMakerEndpointMetadata to agent.py (#2310) Signed-off-by: Kevin Su --- flytekit/core/base_task.py | 2 +- .../awssagemaker_inference/agent.py | 22 +++++++++++++++++-- .../awssagemaker_inference/task.py | 19 ++++------------ 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index e286e1312b..7411fd635e 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -457,7 +457,7 @@ def __init__( self, task_type: str, name: str, - task_config: Optional[T], + task_config: Optional[T] = None, interface: Optional[Interface] = None, environment: Optional[Dict[str, str]] = None, disable_deck: Optional[bool] = None, diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py index 831c70135c..2fe072fc87 100644 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/agent.py @@ -1,18 +1,36 @@ import json +from dataclasses import dataclass from datetime import datetime -from typing import Optional +from typing import Any, Dict, Optional + +import cloudpickle from flytekit.extend.backend.base_agent import ( AgentRegistry, AsyncAgentBase, Resource, + ResourceMeta, ) from flytekit.extend.backend.utils import convert_to_flyte_phase from flytekit.models.literals import LiteralMap from flytekit.models.task import TaskTemplate from .boto3_mixin import Boto3AgentMixin -from .task import SageMakerEndpointMetadata + + +@dataclass +class SageMakerEndpointMetadata(ResourceMeta): + config: Dict[str, Any] + region: Optional[str] = None + inputs: Optional[LiteralMap] = None + + def encode(self) -> bytes: + return cloudpickle.dumps(self) + + @classmethod + def decode(cls, data: bytes) -> "SageMakerEndpointMetadata": + return cloudpickle.loads(data) + states = { "Creating": "Running", diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/task.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/task.py index 4ed538a410..a381547bf5 100644 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/task.py +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/task.py @@ -1,4 +1,3 @@ -from dataclasses import dataclass from typing import Any, Dict, Optional, Type, Union from flytekit import ImageSpec, kwtypes @@ -6,7 +5,6 @@ from flytekit.core.base_task import PythonTask from flytekit.core.interface import Interface from flytekit.extend.backend.base_agent import AsyncAgentExecutorMixin -from flytekit.models.literals import LiteralMap from .boto3_task import BotoConfig, BotoTask @@ -75,14 +73,7 @@ def __init__( ) -@dataclass -class SageMakerEndpointMetadata(object): - config: Dict[str, Any] - region: Optional[str] = None - inputs: Optional[LiteralMap] = None - - -class SageMakerEndpointTask(AsyncAgentExecutorMixin, PythonTask[SageMakerEndpointMetadata]): +class SageMakerEndpointTask(AsyncAgentExecutorMixin, PythonTask): _TASK_TYPE = "sagemaker-endpoint" def __init__( @@ -103,17 +94,15 @@ def __init__( """ super().__init__( name=name, - task_config=SageMakerEndpointMetadata( - config=config, - region=region, - ), task_type=self._TASK_TYPE, interface=Interface(inputs=inputs, outputs=kwtypes(result=str)), **kwargs, ) + self._config = config + self._region = region def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: - return {"config": self.task_config.config, "region": self.task_config.region} + return {"config": self._config, "region": self._region} class SageMakerDeleteEndpointTask(BotoTask): From d11cea9656adafb0aa57b7f922d6b95c719afc85 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 29 Mar 2024 14:24:24 -0700 Subject: [PATCH 32/50] chore: Add @samhita-alla to CODEOWNERS (#2311) --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index a9aab29ffd..dafddb874b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,3 +1,3 @@ # These owners will be the default owners for everything in # the repo. Unless a later match takes precedence. -* @wild-endeavor @kumare3 @eapolinario @pingsutw @cosmicBboy +* @wild-endeavor @kumare3 @eapolinario @pingsutw @cosmicBboy @samhita-alla From afea2fa583310bdcea34fa1a99ccc043e8f723cb Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Fri, 29 Mar 2024 16:51:56 -0700 Subject: [PATCH 33/50] Add way to create FlyteFile/Directory from remote location (#2312) Add a couple of class methods ``` def from_source(cls, source: str | os.PathLike) -> FlyteFile def from_source(cls, source: str | os.PathLike) -> FlyteDirectory ``` meant to be used with remote paths. This is because if you create a file with an s3/gs link as the path using the normal constructor, it doesn't come with a download function, forcing users to manually call into the data persistence layer manually. Signed-off-by: Yee Hing Tong --- flytekit/types/directory/types.py | 19 +++++++++++ flytekit/types/file/file.py | 17 ++++++++++ .../unit/core/test_flyte_directory.py | 33 ++++++++++++++++++- tests/flytekit/unit/core/test_flyte_file.py | 11 +++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index b2a21c4b50..5c50bab9a5 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -235,6 +235,25 @@ def new_dir(self, name: typing.Optional[str] = None) -> FlyteDirectory: new_path = self.sep.join([str(self.path).rstrip(self.sep), name]) # trim trailing sep if any and join return FlyteDirectory(path=new_path) + @classmethod + def from_source(cls, source: str | os.PathLike) -> FlyteDirectory: + """ + Create a new FlyteDirectory object with the remote source set to the input + """ + ctx = FlyteContextManager.current_context() + lit = Literal( + scalar=Scalar( + blob=Blob( + metadata=BlobMetadata( + type=BlobType(format="", dimensionality=BlobType.BlobDimensionality.MULTIPART) + ), + uri=source, + ) + ) + ) + t = FlyteDirToMultipartBlobTransformer() + return t.to_python_value(ctx, lit, cls) + def download(self) -> str: return self.__fspath__() diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 73a835a20f..5c47bda998 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -157,6 +157,23 @@ def new_remote_file(cls, name: typing.Optional[str] = None) -> FlyteFile: remote_path = ctx.file_access.join(ctx.file_access.raw_output_prefix, r) return cls(path=remote_path) + @classmethod + def from_source(cls, source: str | os.PathLike) -> FlyteFile: + """ + Create a new FlyteFile object with the remote source set to the input + """ + ctx = FlyteContextManager.current_context() + lit = Literal( + scalar=Scalar( + blob=Blob( + metadata=BlobMetadata(type=BlobType(format="", dimensionality=BlobType.BlobDimensionality.SINGLE)), + uri=source, + ) + ) + ) + t = FlyteFilePathTransformer() + return t.to_python_value(ctx, lit, cls) + def __class_getitem__(cls, item: typing.Union[str, typing.Type]) -> typing.Type[FlyteFile]: from flytekit.types.file import FileExt diff --git a/tests/flytekit/unit/core/test_flyte_directory.py b/tests/flytekit/unit/core/test_flyte_directory.py index b11f316bcc..36e7dd6927 100644 --- a/tests/flytekit/unit/core/test_flyte_directory.py +++ b/tests/flytekit/unit/core/test_flyte_directory.py @@ -9,7 +9,7 @@ import pytest import flytekit.configuration -from flytekit.configuration import Image, ImageConfig +from flytekit.configuration import Config, Image, ImageConfig from flytekit.core import context_manager from flytekit.core.context_manager import ExecutionState, FlyteContextManager from flytekit.core.data_persistence import FileAccessProvider @@ -314,3 +314,34 @@ def test_list_dir(mock_get_data, mock_lsdir): with pytest.raises(Exception): open(paths[0], "r") + + +def test_manual_creation(local_dummy_directory): + ff = FlyteDirectory.from_source(source="s3://sample-path/folder") + assert ff.path + assert ff._downloader is not None + assert not ff.downloaded + + if os.name != "nt": + fl = FlyteDirectory.from_source(source=local_dummy_directory) + assert fl.path == local_dummy_directory + + +@pytest.mark.sandbox_test +def test_manual_creation_sandbox(local_dummy_directory): + ctx = FlyteContextManager.current_context() + lt = TypeEngine.to_literal_type(FlyteDirectory) + fd = FlyteDirectory(local_dummy_directory) + + dc = Config.for_sandbox().data_config + with tempfile.TemporaryDirectory() as new_sandbox: + provider = FileAccessProvider( + local_sandbox_dir=new_sandbox, raw_output_prefix="s3://my-s3-bucket/testdata/", data_config=dc + ) + with FlyteContextManager.with_context(ctx.with_file_access(provider)) as ctx: + lit = TypeEngine.to_literal(ctx, fd, FlyteDirectory, lt) + + fd_new = FlyteDirectory.from_source(source=lit.scalar.blob.uri) + fd_new.download() + assert os.path.exists(fd_new.path) + assert os.path.isdir(fd_new.path) diff --git a/tests/flytekit/unit/core/test_flyte_file.py b/tests/flytekit/unit/core/test_flyte_file.py index 420279128b..6e055ca399 100644 --- a/tests/flytekit/unit/core/test_flyte_file.py +++ b/tests/flytekit/unit/core/test_flyte_file.py @@ -593,6 +593,17 @@ def wf(path: str) -> None: wf(path=local_dummy_file) +def test_for_downloading(): + ff = FlyteFile.from_source(source="s3://sample-path/file") + assert ff.path + assert ff._downloader is not None + assert not ff.downloaded + + if os.name != "nt": + fl = FlyteFile.from_source(source=__file__) + assert fl.path == __file__ + + @pytest.mark.sandbox_test def test_file_open_things(): @task From 66a6018f2f0f7bb187c0d23bdca00576401306fd Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Mon, 1 Apr 2024 23:17:31 +0530 Subject: [PATCH 34/50] allow specifying triton version for the image in sagemaker agent (#2313) Signed-off-by: Samhita Alla --- .../flytekitplugins/awssagemaker_inference/__init__.py | 9 ++++++++- .../awssagemaker_inference/boto3_mixin.py | 5 +++-- plugins/flytekit-aws-sagemaker/tests/test_boto3_mixin.py | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py index e907455182..fa58efbb89 100644 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py @@ -19,6 +19,8 @@ delete_sagemaker_deployment """ +from functools import partial + from .agent import SageMakerEndpointAgent from .boto3_agent import BotoAgent from .boto3_task import BotoConfig, BotoTask @@ -33,4 +35,9 @@ ) from .workflow import create_sagemaker_deployment, delete_sagemaker_deployment -triton_image_uri = "{account_id}.dkr.ecr.{region}.{base}/sagemaker-tritonserver:21.08-py3" + +def triton_image_uri(version: str = "23.12"): + return partial( + "{account_id}.dkr.ecr.{region}.{base}/sagemaker-tritonserver:{version}-py3".format, + version=version, + ) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py index 1daa16bc73..56be7dca72 100644 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py @@ -1,3 +1,4 @@ +from functools import partial from typing import Any, Dict, Optional import aioboto3 @@ -151,12 +152,12 @@ async def _call( base = "amazonaws.com.cn" if final_region.startswith("cn-") else "amazonaws.com" images = { image_name: ( - image.format( + image( account_id=account_id_map[final_region], region=final_region, base=base, ) - if isinstance(image, str) and "{region}" in image + if isinstance(image, partial) else image ) for image_name, image in images.items() diff --git a/plugins/flytekit-aws-sagemaker/tests/test_boto3_mixin.py b/plugins/flytekit-aws-sagemaker/tests/test_boto3_mixin.py index 98c5686e2d..c53088cf38 100644 --- a/plugins/flytekit-aws-sagemaker/tests/test_boto3_mixin.py +++ b/plugins/flytekit-aws-sagemaker/tests/test_boto3_mixin.py @@ -105,7 +105,7 @@ async def test_call(mock_session): method="create_model", config=config, inputs=inputs, - images={"image": triton_image_uri}, + images={"image": triton_image_uri(version="21.08")}, ) mock_method.assert_called_with( From 2cbdc99ad03bc3f1371d59d5f4a762271176cd4d Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Mon, 1 Apr 2024 17:14:02 -0700 Subject: [PATCH 35/50] Ignore duplicate handler errors when lazy loading (#2316) If the user registers a custom structured dataset encoder/decoder before the lazy import is run for the first time, the default transformers will fail because they don't run with override. flytekit should swallow those errors. Signed-off-by: Yee Hing Tong --- flytekit/core/type_engine.py | 16 ++++++-- .../types/structured/structured_dataset.py | 2 +- .../test_structured_dataset.py | 37 +++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index df90991b3c..3ad18449d1 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -1044,6 +1044,7 @@ def lazy_import_transformers(cls): register_bigquery_handlers, register_pandas_handlers, ) + from flytekit.types.structured.structured_dataset import DuplicateHandlerError if is_imported("tensorflow"): from flytekit.extras import tensorflow # noqa: F401 @@ -1056,11 +1057,20 @@ def lazy_import_transformers(cls): from flytekit.types.schema.types_pandas import PandasSchemaReader, PandasSchemaWriter # noqa: F401 except ValueError: logger.debug("Transformer for pandas is already registered.") - register_pandas_handlers() + try: + register_pandas_handlers() + except DuplicateHandlerError: + logger.debug("Transformer for pandas is already registered.") if is_imported("pyarrow"): - register_arrow_handlers() + try: + register_arrow_handlers() + except DuplicateHandlerError: + logger.debug("Transformer for arrow is already registered.") if is_imported("google.cloud.bigquery"): - register_bigquery_handlers() + try: + register_bigquery_handlers() + except DuplicateHandlerError: + logger.debug("Transformer for bigquery is already registered.") if is_imported("numpy"): from flytekit.types import numpy # noqa: F401 if is_imported("PIL"): diff --git a/flytekit/types/structured/structured_dataset.py b/flytekit/types/structured/structured_dataset.py index 1d7af31404..8faed9ff45 100644 --- a/flytekit/types/structured/structured_dataset.py +++ b/flytekit/types/structured/structured_dataset.py @@ -177,7 +177,7 @@ def __init__(self, python_type: Type[T], protocol: Optional[str] = None, support is capable of handling. :param supported_format: Arbitrary string representing the format. If not supplied then an empty string will be used. An empty string implies that the encoder works with any format. If the format being asked - for does not exist, the transformer enginer will look for the "" encoder instead and write a warning. + for does not exist, the transformer engine will look for the "" encoder instead and write a warning. """ self._python_type = python_type self._protocol = protocol.replace("://", "") if protocol else None diff --git a/tests/flytekit/unit/types/structured_dataset/test_structured_dataset.py b/tests/flytekit/unit/types/structured_dataset/test_structured_dataset.py index cbe3fc422a..9a5628af0f 100644 --- a/tests/flytekit/unit/types/structured_dataset/test_structured_dataset.py +++ b/tests/flytekit/unit/types/structured_dataset/test_structured_dataset.py @@ -2,6 +2,7 @@ import tempfile import typing +import google.cloud.bigquery import pyarrow as pa import pytest from fsspec.utils import get_protocol @@ -15,6 +16,7 @@ from flytekit.core.task import task from flytekit.core.type_engine import TypeEngine from flytekit.core.workflow import workflow +from flytekit.lazy_import.lazy_module import is_imported from flytekit.models import literals from flytekit.models.literals import StructuredDatasetMetadata from flytekit.models.types import SchemaType, SimpleType, StructuredDatasetType @@ -508,3 +510,38 @@ def test_list_of_annotated(): @task def no_op(data: WineDataset) -> typing.List[WineDataset]: return [data] + + +class PrivatePandasToBQEncodingHandlers(StructuredDatasetEncoder): + def __init__(self): + super().__init__(pd.DataFrame, "bq", supported_format="") + + def encode( + self, + ctx: FlyteContext, + structured_dataset: StructuredDataset, + structured_dataset_type: StructuredDatasetType, + ) -> literals.StructuredDataset: + return literals.StructuredDataset( + uri=typing.cast(str, structured_dataset.uri), metadata=StructuredDatasetMetadata(structured_dataset_type) + ) + + +def test_reregister_encoder(): + # Test that lazy import can run after a user has already registered a custom handler. + # The default handlers don't have override=True (and should not) but the call should not fail. + dir(google.cloud.bigquery) + assert is_imported("google.cloud.bigquery") + + StructuredDatasetTransformerEngine.register( + PrivatePandasToBQEncodingHandlers(), default_format_for_type=False, override=True + ) + TypeEngine.lazy_import_transformers() + + sd = StructuredDataset(dataframe=pd.DataFrame({"a": [1, 2], "b": [3, 4]}), uri="bq://blah", file_format="bq") + + ctx = FlyteContextManager.current_context() + + df_literal_type = TypeEngine.to_literal_type(pd.DataFrame) + + TypeEngine.to_literal(ctx, sd, python_type=pd.DataFrame, expected=df_literal_type) From 55f0b19b2cd9d618d9a1ca8e8e4b2faa02e716d2 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:03:50 -0700 Subject: [PATCH 36/50] Use correct plugin in agent image (#2317) Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- Dockerfile.agent | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.agent b/Dockerfile.agent index 886e4af613..1520b80d7c 100644 --- a/Dockerfile.agent +++ b/Dockerfile.agent @@ -11,7 +11,7 @@ RUN pip install prometheus-client grpcio-health-checking RUN pip install --no-cache-dir -U flytekit==$VERSION \ flytekitplugins-airflow==$VERSION \ flytekitplugins-bigquery==$VERSION \ - flytekitplugins-chatgpt==$VERSION \ + flytekitplugins-openai==$VERSION \ flytekitplugins-snowflake==$VERSION \ flytekitplugins-awssagemaker==$VERSION \ && apt-get clean autoclean \ From eac772a8417d64c23af20bd0cec775b5e52c9ee7 Mon Sep 17 00:00:00 2001 From: Jason Lai Date: Thu, 4 Apr 2024 02:52:07 +0800 Subject: [PATCH 37/50] Add Python Dependency Deck (#2264) * feat: refactor source code rendering in `SourceCodeDeck` class - Add a new class `SourceCodeDeck` to render the source code of a task - Implement the `html` property in the `SourceCodeDeck` class Signed-off-by: jason.lai * refactor: refactor deck rendering for Python dependencies - Change the class name from `SourceCodeDeck` to `PythonDependencyDeck` - Update the deck description to reflect python dependencies instead of source code - Remove the source code rendering functionality and replace it with a table renderer for python dependencies - Add subprocess logic to fetch installed python packages and render them in an HTML table Signed-off-by: jason.lai * refactor: refactor class properties and initialization - Add a property method for `pythondependency_deck` - Initialize `python_dependency_deck` if it is None - Remove a property method for `timeline_deck` Signed-off-by: jason.lai * - feat: consolidate Python dependencies in `flytekit/__init__.py` - Add `PythonDependencyDeck` to `flytekit/__init__.py` Signed-off-by: jason.lai * refactor: refactor `flytekit/deck/deck.py` for `pandas` compatibility - Add `pandas` import to `flytekit/deck/deck.py` - Change the input of `TableRenderer().to_html` to a `pd.DataFrame` in `flytekit/deck/deck.py` Signed-off-by: jason.lai * refactor: refactor return types across multiple files - Update the return type annotation for `pythondependency_deck` in `context_manager.py` Signed-off-by: jason.lai * feat: consolidate Python dependency management in FlyteContext - Add `PythonDependencyDeck` to the TYPE_CHECKING import - Add a method `add_deck` to `FlyteContext` class Signed-off-by: jason.lai * refactor: refactor code to use DataFrame for package handling - Refactor the code to use a DataFrame for installed packages handling Signed-off-by: jason.lai * chore: refactor code for improved naming conventions - Update the name of the PythonDependencyDeck instance from "PythonDependencyDeck" to "Python Dependency" Signed-off-by: jason.lai * style: improve table alignment styling in CSS - Add CSS style to center align the table content Signed-off-by: jason.lai * refactor: refactor method and variable names across files - Rename method `pythondependency_deck` to `python_dependency_deck` - Update variable names in method `python_dependency_deck` - Update method comments in class `FlyteContext` - Update method comments in class `PythonDependencyDeck` Signed-off-by: jason.lai * refactor: refactor imports in flytekit package - Remove import of `PythonDependencyDeck` from `flytekit/__init__.py` Signed-off-by: jason.lai * refactor: consolidate import statements in core/context_manager.py - Remove the import of `PythonDependencyDeck` from `flytekit/core/context_manager.py` - Add an import of `Deck` to `flytekit/core/context_manager.py` Signed-off-by: jason.lai * style: improve code consistency and error checking - Add a condition to check for deck existence before appending - Change single quotes to double quotes for consistency - Update package split delimiter to double quotes Signed-off-by: jason.lai * refactor: refactor Python dependency handling in classes - Remove the `python_dependency_deck` property from the `ExecutionParameters` class in `context_manager.py` - Update the `__init__` method signature in the `PythonDependencyDeck` class in `deck.py` Signed-off-by: jason.lai * chore: optimize imports in deck.py files - Remove unnecessary import of `pandas` in `flytekit/deck/deck.py` - Add import of `pandas` in `flytekit/deck/deck.py` Signed-off-by: jason.lai * test: improve test coverage for PythonDependencyDeck class - Add a new test for the PythonDependencyDeck class in test_deck.py - Add assertions for specific strings in the HTML content in the test_python_dependency_deck() function Signed-off-by: jason.lai * feat: enhance user space deck management - Clear user space decks before adding a new deck - Ensure only one deck is added to user space params Signed-off-by: jason.lai * refactor: refactor deck module and unit tests - Import json and TableRenderer in `flytekit/deck/deck.py` - Change how installed packages are fetched in `flytekit/deck/deck.py` - Remove unused code in `flytekit/deck/deck.py` - Remove assertions for "Library" and "Version" in `tests/flytekit/unit/deck/test_deck.py` Signed-off-by: jason.lai * fix: update subprocess calls to use `sys.executable` - Import `sys` to fix an issue with subprocess execution - Update the subprocess call to use `sys.executable` instead of `pip` Signed-off-by: jason.lai * feat: refactor HTML generation logic and improve user experience - Update the HTML generation logic to include a button for copying the table as requirements.txt. Signed-off-by: jason.lai * feat: enhance table content copying functionality - Add functionality to copy table content as requirements.txt - Improve error handling when copying table content - Display table content as hidden div for copying Signed-off-by: jason.lai * refactor: improve table content copying functionality - Remove unnecessary code for table content copying - Update table content copying functionality to use innerText of requirements_txt element Signed-off-by: jason.lai * refactor: improve package management and error handling - Add logic to generate a `requirements.txt` file from installed packages - Update error logging message in case of subprocess error - Update console log message when accessing clipboard - Update `requirements_txt` div content with actual `requirements_txt` variable Signed-off-by: jason.lai * chore: standardize whitespace in requirements_txt handling - Remove trailing whitespace from `requirements_txt` string - Add a whitespace to the end of the `requirements_txt` variable - Log an error message when fetching installed packages fails Signed-off-by: jason.lai * style: standardize quotation marks for package_info keys - Corrected quotation marks in package_info keys - Changed single quotes to double quotes for consistency Signed-off-by: jason.lai * refactor: simplify requirements_txt generation - Refactor code to simplify the generation of `requirements_txt` - Add the output of `pip freeze` to `requirements_txt` Signed-off-by: jason.lai * docs: fix typos and improve code consistency across files - Fix a typo in the usage of the `pip freeze` command in the PythonDependencyDeck class Signed-off-by: jason.lai * refactor: refactor dependency handling in PythonDependencyDeck class - Update the way `requirements_txt` is populated in the `PythonDependencyDeck` class Signed-off-by: jason.lai * refactor: update default name and test assertion in PythonDependencyDeck - Update the default name in the PythonDependencyDeck constructor from "Python Dependency" to "Python Dependencies" - Update the assertion in the test_python_dependency_deck function to check for the new default name "Python Dependencies" Signed-off-by: jason.lai * refactor: update rendering of Pandas DataFrame using MarkdownRenderer - Import `MarkdownRenderer` from `flytekit.deck` instead of `TableRenderer` - Render the Pandas DataFrame as markdown using `MarkdownRenderer` instead of `TableRenderer` Signed-off-by: jason.lai * refactor: refactor PythonDependencyDeck and related classes - Remove the `add_deck` method from `FlyteContext` - Add imports for `PythonDependencyRenderer` and `PythonDependencyDeck` in `PythonFunctionTask` - Remove the `PythonDependencyDeck` class and related methods from the `deck.py` file - Add the `PythonDependencyRenderer` class in `renderer.py` Signed-off-by: jason.lai * feat: use `TableRenderer` for rendering DataFrames - Import `TableRenderer` from a different module - Replace the `MarkdownRenderer` with `TableRenderer` to render a DataFrame as a table Signed-off-by: jason.lai * test: refactor codebase for improved performance - Remove unused import statements in `flytekit/deck/renderer.py` - Add a new test for `PythonDependencyRenderer` in `tests/flytekit/unit/deck/test_deck.py` Signed-off-by: jason.lai * test: update test_deck.py for python dependency deck testing - Update test_deck.py to include python dependency deck in various test cases - Adjust expected deck counts in test cases to reflect the changes - Add test cases for scenarios involving python dependency deck and input and output decks Signed-off-by: jason.lai * style: standardize import statements for pandas in project files - Remove `import pandas as pd` and replace it with `import pandas as pandas` Signed-off-by: jason.lai * refactor: refactor import statements for `TableRenderer` usage - Import `TableRenderer` from a different location in `flytekit/deck/renderer.py` - Remove `TableRenderer` from the imports in `flytekit-deck-standard/flytekitplugins/deck/__init__.py` Signed-off-by: jason.lai * refactor: update PythonDependencyRenderer description - Update the description of PythonDependencyDeck in PythonDependencyRenderer Signed-off-by: jason.lai * refactor: refactor type hints and assertions across files - Change the type hint for `df` parameter to `pandas.DataFrame` - Add an assertion to check the type of `df` is `pandas.DataFrame` Signed-off-by: jason.lai * feat: refactor rendering classes in deck and plugins - Add TableRenderer to TimeLineDeck class in deck.py - Remove TableRenderer from deck/renderer.py - Add TableRenderer to PythonDependencyRenderer in deck/renderer.py - Add TableRenderer to flytekit-deck-standard plugin - Remove TableRenderer from test_renderer.py Signed-off-by: jason.lai * docs: standardize markdown formatting across files - Update the markdown table format in PythonDependencyRenderer class Signed-off-by: jason.lai * refactor: refactor markdown table rendering in PythonDependencyRenderer - Update the markdown table format in the PythonDependencyRenderer class to use HTML table tags - Modify the markdown_table variable concatenation for installed packages entries in the PythonDependencyRenderer class Signed-off-by: jason.lai * style: improve code consistency in PythonDependencyRenderer - Add a line break after the button element in PythonDependencyRenderer Signed-off-by: jason.lai * style: standardize formatting for better readability - Modify the table headers to have left-aligned text - Add extra line breaks after the button element Signed-off-by: jason.lai * fix: update assertion to check for `Name` and `Version` consistency - Change the assertion to check for `Name` and `Version` instead of `name` and `version` in the result Signed-off-by: jason.lai * test: extend test deadlines across various tests - Increase the test deadlines from 2 to 20 seconds in test_type_conversion_errors.py - Update test deadlines from 2 to 40 seconds in test_type_conversion_errors.py - Adjust test deadlines in test_eager_workflows.py for various tests Signed-off-by: jason.lai * Revert "test: extend test deadlines across various tests" This reverts commit 1885662a0ed963f2a18b4a803ed07c78afa13687. Signed-off-by: Eduardo Apolinario * Only run generate decks in python_function_task if decks are enabled Signed-off-by: Eduardo Apolinario * Remove breakpoint Signed-off-by: Eduardo Apolinario * Fix test_deck.py tests to account for the number of expected decks (now that we're no longer writing decks unnecessarily) Signed-off-by: Eduardo Apolinario * Increase deadline of eager tests Signed-off-by: Eduardo Apolinario * Fix lint error Signed-off-by: Eduardo Apolinario * test: update test assertions in test_deck.py - Add a patch to the `test_python_dependency_renderer` function - Include assertions for `numpy` and `1.21.0` in the test result - Update the `test_deck.py` file in the `tests/flytekit/unit/deck` directory Signed-off-by: jason.lai * refactor: refactor Python code organization - Import `PythonDependencyRenderer` and `SourceCodeRenderer` in `PythonFunctionTask` from separate files - Modify the way `requirements_txt` is generated in `PythonDependencyRenderer` - Refactor the creation of the `table` variable in `PythonDependencyRenderer` to improve readability Signed-off-by: jason.lai * docs: refactor project structure and improve user experience - Change the title of the deck from `Python Dependencies` to `Dependencies` - Update the error message in case of a subprocess error in fetching installed packages - Add a heading `Python Dependencies` above the table output in the HTML Signed-off-by: jason.lai * Separate out tests that require hypothesis Signed-off-by: Eduardo Apolinario --------- Signed-off-by: jason.lai Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- .github/workflows/pythonbuild.yml | 35 +++++++++ Makefile | 5 +- flytekit/core/python_function_task.py | 22 ++++-- flytekit/deck/renderer.py | 74 +++++++++++++++++++ pyproject.toml | 1 + tests/flytekit/unit/conftest.py | 9 +++ tests/flytekit/unit/deck/test_deck.py | 45 ++++++++--- .../unit/experimental/test_eager_workflows.py | 24 +++--- 8 files changed, 185 insertions(+), 30 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index b58b61ac95..378b841d21 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -149,6 +149,41 @@ jobs: fail_ci_if_error: false files: coverage.xml + test-hypothesis: + needs: + - detect-python-versions + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}} + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip + uses: actions/cache@v3 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements files + key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }} + - name: Install dependencies + run: make setup && pip freeze + - name: Test with coverage + env: + FLYTEKIT_HYPOTHESIS_PROFILE: ci + run: | + make unit_test_hypothesis + - name: Codecov + uses: codecov/codecov-action@v3.1.4 + with: + fail_ci_if_error: false + files: coverage.xml + test-serialization: needs: - detect-python-versions diff --git a/Makefile b/Makefile index 859b0aaf44..a8317faa17 100644 --- a/Makefile +++ b/Makefile @@ -62,10 +62,13 @@ unit_test_extras_codecov: unit_test: # Skip all extra tests and run them with the necessary env var set so that a working (albeit slower) # library is used to serialize/deserialize protobufs is used. - $(PYTEST_AND_OPTS) -m "not (serial or sandbox_test)" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/ --ignore=tests/flytekit/unit/models --ignore=tests/flytekit/unit/extend ${CODECOV_OPTS} + $(PYTEST_AND_OPTS) -m "not (serial or sandbox_test or hypothesis)" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/ --ignore=tests/flytekit/unit/models --ignore=tests/flytekit/unit/extend ${CODECOV_OPTS} # Run serial tests without any parallelism $(PYTEST) -m "serial" tests/flytekit/unit/ --ignore=tests/flytekit/unit/extras/ --ignore=tests/flytekit/unit/models --ignore=tests/flytekit/unit/extend ${CODECOV_OPTS} +.PHONY: unit_test_hypothesis +unit_test_hypothesis: + $(PYTEST_AND_OPTS) -m "hypothesis" tests/flytekit/unit/experimental ${CODECOV_OPTS} .PHONY: unit_test_extras unit_test_extras: diff --git a/flytekit/core/python_function_task.py b/flytekit/core/python_function_task.py index 147f15bbb3..f97d96296e 100644 --- a/flytekit/core/python_function_task.py +++ b/flytekit/core/python_function_task.py @@ -349,15 +349,21 @@ def dynamic_execute(self, task_function: Callable, **kwargs) -> Any: raise ValueError(f"Invalid execution provided, execution state: {ctx.execution_state}") def _write_decks(self, native_inputs, native_outputs_as_map, ctx, new_user_params): - # These errors are raised if the source code can not be retrieved - with suppress(OSError, TypeError): - source_code = inspect.getsource(self._task_function) - + if self._disable_deck is False: from flytekit.deck import Deck - from flytekit.deck.renderer import SourceCodeRenderer + from flytekit.deck.renderer import PythonDependencyRenderer + + # These errors are raised if the source code can not be retrieved + with suppress(OSError, TypeError): + source_code = inspect.getsource(self._task_function) + from flytekit.deck.renderer import SourceCodeRenderer + + source_code_deck = Deck("Source Code") + renderer = SourceCodeRenderer() + source_code_deck.append(renderer.to_html(source_code)) - source_code_deck = Deck("Source Code") - renderer = SourceCodeRenderer() - source_code_deck.append(renderer.to_html(source_code)) + python_dependencies_deck = Deck("Dependencies") + renderer = PythonDependencyRenderer() + python_dependencies_deck.append(renderer.to_html()) return super()._write_decks(native_inputs, native_outputs_as_map, ctx, new_user_params) diff --git a/flytekit/deck/renderer.py b/flytekit/deck/renderer.py index 7f4913f6d9..51157dc876 100644 --- a/flytekit/deck/renderer.py +++ b/flytekit/deck/renderer.py @@ -86,3 +86,77 @@ def to_html(self, source_code: str) -> str: css = formatter.get_style_defs(".highlight").replace("#fff0f0", "#ffffff") html = highlight(source_code, PythonLexer(), formatter) return f"{html}" + + +class PythonDependencyRenderer: + """ + PythonDependencyDeck is a deck that contains information about packages installed via pip. + """ + + def __init__(self, title: str = "Dependencies"): + self._title = title + + def to_html(self) -> str: + import json + import subprocess + import sys + + from flytekit.loggers import logger + + try: + installed_packages = json.loads( + subprocess.check_output([sys.executable, "-m", "pip", "list", "--format", "json"]) + ) + requirements_txt = ( + subprocess.check_output([sys.executable, "-m", "pip", "freeze"]) + .decode("utf-8") + .replace("\\n", "\n") + .rstrip() + ) + except subprocess.CalledProcessError as e: + logger.error(f"Error occurred while fetching installed packages: {e}") + return "Error occurred while fetching installed packages." + + table = ( + "\n\n\n\n\n" + ) + + for entry in installed_packages: + table += f"\n\n\n\n" + + table += "
NameVersion
{entry['name']}{entry['version']}
" + + html = f""" + + + + + + Flyte Dependencies + + + + + +

Python Dependencies

+ + {table} + + + + + + """ + return html diff --git a/pyproject.toml b/pyproject.toml index 5678d342b7..45ce9e83a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,6 +99,7 @@ log_cli_level = 20 markers = [ "sandbox_test: fake integration tests", # unit tests that are really integration tests that run on a sandbox environment "serial: tests to avoid using with pytest-xdist", + "hypothesis: tests that use they hypothesis library", ] [tool.coverage.report] diff --git a/tests/flytekit/unit/conftest.py b/tests/flytekit/unit/conftest.py index 7414b5e064..ed9b6ada98 100644 --- a/tests/flytekit/unit/conftest.py +++ b/tests/flytekit/unit/conftest.py @@ -1,4 +1,7 @@ +import os + import pytest +from hypothesis import settings from flytekit.image_spec.image_spec import ImageSpecBuilder @@ -11,3 +14,9 @@ def build_image(self, img): @pytest.fixture() def mock_image_spec_builder(): return MockImageSpecBuilder() + + +settings.register_profile("ci", max_examples=5, deadline=100_000) +settings.register_profile("dev", max_examples=10, deadline=10_000) + +settings.load_profile(os.getenv("FLYTEKIT_HYPOTHESIS_PROFILE", "dev")) diff --git a/tests/flytekit/unit/deck/test_deck.py b/tests/flytekit/unit/deck/test_deck.py index 45056ae283..9cf497bccd 100644 --- a/tests/flytekit/unit/deck/test_deck.py +++ b/tests/flytekit/unit/deck/test_deck.py @@ -3,12 +3,13 @@ import pytest from markdown_it import MarkdownIt -from mock import mock +from mock import mock, patch import flytekit from flytekit import Deck, FlyteContextManager, task from flytekit.deck import MarkdownRenderer, SourceCodeRenderer, TopFrameRenderer from flytekit.deck.deck import _output_deck +from flytekit.deck.renderer import PythonDependencyRenderer @pytest.mark.skipif("pandas" not in sys.modules, reason="Pandas is not installed.") @@ -50,9 +51,9 @@ def test_timeline_deck(): @pytest.mark.parametrize( "disable_deck,expected_decks", [ - (None, 2), # time line deck + source code deck - (False, 4), # time line deck + source code deck + input and output decks - (True, 2), # time line deck + source code deck + (None, 1), # time line deck + (False, 5), # time line deck + source code deck + python dependency deck + input and output decks + (True, 1), # time line deck ], ) def test_deck_for_task(disable_deck, expected_decks): @@ -75,11 +76,21 @@ def t1(a: int) -> str: @pytest.mark.parametrize( "enable_deck,disable_deck, expected_decks, expect_error", [ - (None, None, 3, False), # default deck and time line deck + source code deck - (None, False, 5, False), # default deck and time line deck + source code deck + input and output decks - (None, True, 3, False), # default deck and time line deck + source code deck - (True, None, 5, False), # default deck and time line deck + source code deck + input and output decks - (False, None, 3, False), # default deck and time line deck + source code deck + (None, None, 2, False), # default deck and time line deck + ( + None, + False, + 6, + False, + ), # default deck and time line deck + source code deck + python dependency deck + input and output decks + (None, True, 2, False), # default deck and time line deck + ( + True, + None, + 6, + False, + ), # default deck and time line deck + source code deck + python dependency deck + input and output decks + (False, None, 2, False), # default deck and time line deck (True, True, -1, True), # Set both disable_deck and enable_deck to True and confirm that it fails (False, False, -1, True), # Set both disable_deck and enable_deck to False and confirm that it fails ], @@ -176,3 +187,19 @@ def test_source_code_renderer(): # Assert that the color #ffffff is used instead of #fff0f0 assert "#ffffff" in result assert "#fff0f0" not in result + + +def test_python_dependency_renderer(): + with patch("subprocess.check_output") as mock_check_output: + mock_check_output.return_value = '[{"name": "numpy", "version": "1.21.0"}]'.encode() + renderer = PythonDependencyRenderer() + result = renderer.to_html() + assert "numpy" in result + assert "1.21.0" in result + + # Assert that the result includes parts of the python dependency + assert "Name" in result + assert "Version" in result + + # Assert that the button of copy + assert 'button onclick="copyTable()"' in result diff --git a/tests/flytekit/unit/experimental/test_eager_workflows.py b/tests/flytekit/unit/experimental/test_eager_workflows.py index 9760f8c008..c25e2ae762 100644 --- a/tests/flytekit/unit/experimental/test_eager_workflows.py +++ b/tests/flytekit/unit/experimental/test_eager_workflows.py @@ -6,7 +6,7 @@ import hypothesis.strategies as st import pytest -from hypothesis import given, settings +from hypothesis import given from flytekit import dynamic, task, workflow from flytekit.exceptions.user import FlyteValidationException @@ -15,7 +15,6 @@ from flytekit.types.file import FlyteFile from flytekit.types.structured import StructuredDataset -DEADLINE = 2000 INTEGER_ST = st.integers(min_value=-10_000_000, max_value=10_000_000) @@ -48,7 +47,7 @@ def dynamic_wf(x: int) -> int: @given(x_input=INTEGER_ST) -@settings(deadline=DEADLINE, max_examples=5) +@pytest.mark.hypothesis def test_simple_eager_workflow(x_input: int): """Testing simple eager workflow with just tasks.""" @@ -62,7 +61,7 @@ async def eager_wf(x: int) -> int: @given(x_input=INTEGER_ST) -@settings(deadline=DEADLINE, max_examples=5) +@pytest.mark.hypothesis def test_conditional_eager_workflow(x_input: int): """Test eager workflow with conditional logic.""" @@ -80,7 +79,7 @@ async def eager_wf(x: int) -> int: @given(x_input=INTEGER_ST) -@settings(deadline=DEADLINE, max_examples=5) +@pytest.mark.hypothesis def test_try_except_eager_workflow(x_input: int): """Test eager workflow with try/except logic.""" @@ -99,7 +98,7 @@ async def eager_wf(x: int) -> int: @given(x_input=INTEGER_ST, n_input=st.integers(min_value=1, max_value=20)) -@settings(deadline=DEADLINE, max_examples=5) +@pytest.mark.hypothesis def test_gather_eager_workflow(x_input: int, n_input: int): """Test eager workflow with asyncio gather.""" @@ -113,7 +112,7 @@ async def eager_wf(x: int, n: int) -> typing.List[int]: @given(x_input=INTEGER_ST) -@settings(deadline=DEADLINE, max_examples=5) +@pytest.mark.hypothesis def test_eager_workflow_with_dynamic_exception(x_input: int): """Test eager workflow with dynamic workflow is not supported.""" @@ -131,7 +130,7 @@ async def nested_eager_wf(x: int) -> int: @given(x_input=INTEGER_ST) -@settings(deadline=DEADLINE, max_examples=5) +@pytest.mark.hypothesis def test_nested_eager_workflow(x_input: int): """Testing running nested eager workflows.""" @@ -145,7 +144,7 @@ async def eager_wf(x: int) -> int: @given(x_input=INTEGER_ST) -@settings(deadline=DEADLINE, max_examples=5) +@pytest.mark.hypothesis def test_eager_workflow_within_workflow(x_input: int): """Testing running eager workflow within a static workflow.""" @@ -168,7 +167,7 @@ def subworkflow(x: int) -> int: @given(x_input=INTEGER_ST) -@settings(deadline=DEADLINE, max_examples=5) +@pytest.mark.hypothesis def test_workflow_within_eager_workflow(x_input: int): """Testing running a static workflow within an eager workflow.""" @@ -182,7 +181,7 @@ async def eager_wf(x: int) -> int: @given(x_input=INTEGER_ST) -@settings(deadline=DEADLINE, max_examples=5) +@pytest.mark.hypothesis def test_local_task_eager_workflow_exception(x_input: int): """Testing simple eager workflow with a local function task doesn't work.""" @@ -199,8 +198,8 @@ async def eager_wf_with_local(x: int) -> int: @given(x_input=INTEGER_ST) -@settings(deadline=DEADLINE, max_examples=5) @pytest.mark.filterwarnings("ignore:coroutine 'AsyncEntity.__call__' was never awaited") +@pytest.mark.hypothesis def test_local_workflow_within_eager_workflow_exception(x_input: int): """Cannot call a locally-defined workflow within an eager workflow""" @@ -243,6 +242,7 @@ def create_directory() -> FlyteDirectory: @pytest.mark.skipif("pandas" not in sys.modules, reason="Pandas is not installed.") +@pytest.mark.hypothesis def test_eager_workflow_with_offloaded_types(): """Test eager workflow that eager workflows work with offloaded types.""" import pandas as pd From 6a383cdf56120b8bf61ad4492ffcccb3cf201f21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Apr 2024 11:53:35 -0700 Subject: [PATCH 38/50] Bump pillow (#2321) Bumps [pillow](https://github.com/python-pillow/Pillow) from 10.2.0 to 10.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/10.2.0...10.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../remote/mock_flyte_repo/workflows/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 6d83891785..5fe080f60f 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -221,7 +221,7 @@ packaging==23.1 # matplotlib pandas==1.5.3 # via flytekit -pillow==10.2.0 +pillow==10.3.0 # via matplotlib portalocker==2.7.0 # via msal-extensions From a240b65d726f41dd13c5bd03980fde4a4faeb7ac Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:02:46 -0700 Subject: [PATCH 39/50] Ensure that annotations are dropped in the case of the dataclass and dict type transformers (#2318) The original exception in the `DictTransformer` was put in place to catch the case of `FlyteAnnotation` being set passed to annotated dictionary, e.g. `Annotated[dict, FlyteAnnotation("annotation-1")]` is invalid. A similar argument applies to the dataclass type transformer. Signed-off-by: Eduardo Apolinario --- .github/workflows/pythonbuild.yml | 4 +- flytekit/core/type_engine.py | 30 ++-- tests/flytekit/unit/core/test_dataclass.py | 12 ++ tests/flytekit/unit/core/test_type_engine.py | 168 +++++++++++++++++-- 4 files changed, 184 insertions(+), 30 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 378b841d21..7b308ad634 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -426,10 +426,10 @@ jobs: steps: - name: Fetch the code uses: actions/checkout@v4 - - name: Set up Python 3.8 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.12 - uses: actions/cache@v3 with: path: ~/.cache/pip diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 3ad18449d1..4a5c6bac8d 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -356,6 +356,7 @@ def assert_type(self, expected_type: Type[DataClassJsonMixin], v: T): # However, FooSchema is created by flytekit and it's not equal to the user-defined dataclass (Foo). # Therefore, we should iterate all attributes in the dataclass and check the type of value in dataclass matches the expected_type. + expected_type = get_underlying_type(expected_type) expected_fields_dict = {} for f in dataclasses.fields(expected_type): expected_fields_dict[f.name] = f.type @@ -422,10 +423,16 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: If possible also extracts the JSONSchema for the dataclass. """ if is_annotated(t): - raise ValueError( - "Flytekit does not currently have support for FlyteAnnotations applied to Dataclass." - f"Type {t} cannot be parsed." - ) + args = get_args(t) + for x in args[1:]: + if isinstance(x, FlyteAnnotation): + raise ValueError( + "Flytekit does not currently have support for FlyteAnnotations applied to Dataclass." + f"Type {t} cannot be parsed." + ) + logger.info(f"These annotations will be skipped for dataclasses = {args[1:]}") + # Drop all annotations and handle only the dataclass type passed in. + t = args[0] if not self.is_serializable_class(t): raise AssertionError( @@ -1642,12 +1649,15 @@ def get_dict_types(t: Optional[Type[dict]]) -> typing.Tuple[Optional[type], Opti _origin = get_origin(t) _args = get_args(t) if _origin is not None: - if _origin is Annotated: - raise ValueError( - f"Flytekit does not currently have support \ - for FlyteAnnotations applied to dicts. {t} cannot be \ - parsed." - ) + if _origin is Annotated and _args: + # _args holds the type arguments to the dictionary, in other words: + # >>> get_args(Annotated[dict[int, str], FlyteAnnotation("abc")]) + # (dict[int, str], ) + for x in _args[1:]: + if isinstance(x, FlyteAnnotation): + raise ValueError( + f"Flytekit does not currently have support for FlyteAnnotations applied to dicts. {t} cannot be parsed." + ) if _origin is dict and _args is not None: return _args # type: ignore return None, None diff --git a/tests/flytekit/unit/core/test_dataclass.py b/tests/flytekit/unit/core/test_dataclass.py index b1b6b42761..cdafe24d1a 100644 --- a/tests/flytekit/unit/core/test_dataclass.py +++ b/tests/flytekit/unit/core/test_dataclass.py @@ -4,8 +4,11 @@ import pytest from dataclasses_json import DataClassJsonMixin +from mashumaro.mixins.json import DataClassJSONMixin +from typing_extensions import Annotated from flytekit.core.task import task +from flytekit.core.type_engine import DataclassTransformer from flytekit.core.workflow import workflow @@ -29,3 +32,12 @@ def wf() -> AppParams: res = wf() assert res.region == "us-west-3" + + +def test_dataclass_assert_works_for_annotated(): + @dataclass + class MyDC(DataClassJSONMixin): + my_str: str + + d = Annotated[MyDC, "tag"] + DataclassTransformer().assert_type(d, MyDC(my_str="hi")) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index b94249f804..41bf9e5fb5 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -2118,22 +2118,6 @@ def test_type_alias(): TypeEngine.to_literal_type(t) -def test_unsupported_complex_literals(): - t = typing_extensions.Annotated[typing.Dict[int, str], FlyteAnnotation({"foo": "bar"})] - with pytest.raises(ValueError): - TypeEngine.to_literal_type(t) - - # Enum. - t = typing_extensions.Annotated[Color, FlyteAnnotation({"foo": "bar"})] - with pytest.raises(ValueError): - TypeEngine.to_literal_type(t) - - # Dataclass. - t = typing_extensions.Annotated[Result, FlyteAnnotation({"foo": "bar"})] - with pytest.raises(ValueError): - TypeEngine.to_literal_type(t) - - def test_multiple_annotations(): t = typing_extensions.Annotated[int, FlyteAnnotation({"foo": "bar"}), FlyteAnnotation({"anotha": "one"})] with pytest.raises(Exception): @@ -2155,6 +2139,144 @@ class Result(DataClassJsonMixin): schema: TestSchema # type: ignore +@pytest.mark.parametrize( + "t", + [ + typing_extensions.Annotated[dict, FlyteAnnotation({"foo": "bar"})], + typing_extensions.Annotated[dict[int, str], FlyteAnnotation({"foo": "bar"})], + typing_extensions.Annotated[typing.Dict[int, str], FlyteAnnotation({"foo": "bar"})], + typing_extensions.Annotated[dict[str, str], FlyteAnnotation({"foo": "bar"})], + typing_extensions.Annotated[typing.Dict[str, str], FlyteAnnotation({"foo": "bar"})], + typing_extensions.Annotated[Color, FlyteAnnotation({"foo": "bar"})], + typing_extensions.Annotated[Result, FlyteAnnotation({"foo": "bar"})], + ], +) +def test_unsupported_complex_literals(t): + with pytest.raises(ValueError): + TypeEngine.to_literal_type(t) + + +@dataclass +class DataclassTest(DataClassJsonMixin): + a: int + b: str + + +@dataclass +class AnnotatedDataclassTest(DataClassJsonMixin): + a: int + b: Annotated[str, "str tag"] + + +@pytest.mark.parametrize( + "t,expected_type", + [ + (dict, LiteralType(simple=SimpleType.STRUCT)), + # Annotations are not being copied over to the LiteralType + (typing_extensions.Annotated[dict, "a-tag"], LiteralType(simple=SimpleType.STRUCT)), + (typing.Dict[int, str], LiteralType(simple=SimpleType.STRUCT)), + (typing.Dict[str, int], LiteralType(map_value_type=LiteralType(simple=SimpleType.INTEGER))), + (typing.Dict[str, str], LiteralType(map_value_type=LiteralType(simple=SimpleType.STRING))), + ( + typing.Dict[str, typing.List[int]], + LiteralType(map_value_type=LiteralType(collection_type=LiteralType(simple=SimpleType.INTEGER))), + ), + (typing.Dict[int, typing.List[int]], LiteralType(simple=SimpleType.STRUCT)), + (typing.Dict[int, typing.Dict[int, int]], LiteralType(simple=SimpleType.STRUCT)), + (typing.Dict[str, typing.Dict[int, int]], LiteralType(map_value_type=LiteralType(simple=SimpleType.STRUCT))), + ( + typing.Dict[str, typing.Dict[str, int]], + LiteralType(map_value_type=LiteralType(map_value_type=LiteralType(simple=SimpleType.INTEGER))), + ), + ( + DataclassTest, + LiteralType( + simple=SimpleType.STRUCT, + metadata={ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "DataclasstestSchema": { + "properties": { + "a": {"title": "a", "type": "integer"}, + "b": {"title": "b", "type": "string"}, + }, + "type": "object", + "additionalProperties": False, + } + }, + "$ref": "#/definitions/DataclasstestSchema", + }, + structure=TypeStructure( + tag="", + dataclass_type={ + "a": LiteralType(simple=SimpleType.INTEGER), + "b": LiteralType(simple=SimpleType.STRING), + }, + ), + ), + ), + # Similar to the dict[int, str] case, the annotation is not being copied over to the LiteralType + ( + Annotated[DataclassTest, "another-tag"], + LiteralType( + simple=SimpleType.STRUCT, + metadata={ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "DataclasstestSchema": { + "properties": { + "a": {"title": "a", "type": "integer"}, + "b": {"title": "b", "type": "string"}, + }, + "type": "object", + "additionalProperties": False, + } + }, + "$ref": "#/definitions/DataclasstestSchema", + }, + structure=TypeStructure( + tag="", + dataclass_type={ + "a": LiteralType(simple=SimpleType.INTEGER), + "b": LiteralType(simple=SimpleType.STRING), + }, + ), + ), + ), + # Notice how the annotation in the field is not carried over either + ( + Annotated[AnnotatedDataclassTest, "another-tag"], + LiteralType( + simple=SimpleType.STRUCT, + metadata={ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "AnnotateddataclasstestSchema": { + "properties": { + "a": {"title": "a", "type": "integer"}, + "b": {"title": "b", "type": "string"}, + }, + "type": "object", + "additionalProperties": False, + } + }, + "$ref": "#/definitions/AnnotateddataclasstestSchema", + }, + structure=TypeStructure( + tag="", + dataclass_type={ + "a": LiteralType(simple=SimpleType.INTEGER), + "b": LiteralType(simple=SimpleType.STRING), + }, + ), + ), + ), + ], +) +def test_annotated_dicts(t, expected_type): + assert TypeEngine.to_literal_type(t) == expected_type + + @pytest.mark.skipif("pandas" not in sys.modules, reason="Pandas is not installed.") def test_schema_in_dataclass(): import pandas as pd @@ -2400,8 +2522,18 @@ def test_get_underlying_type(t, expected): assert get_underlying_type(t) == expected -def test_dict_get(): - assert DictTransformer.get_dict_types(None) == (None, None) +@pytest.mark.parametrize( + "t,expected", + [ + (None, (None, None)), + (typing.Dict, ()), + (typing.Dict[str, str], (str, str)), + (Annotated[typing.Dict, "a-tag"], (None, None)), + (typing.Dict[Annotated[str, "a-tag"], int], (Annotated[str, "a-tag"], int)), + ], +) +def test_dict_get(t, expected): + assert DictTransformer.get_dict_types(t) == expected def test_DataclassTransformer_get_literal_type(): From df5dbea5cd12c9515ca250b4038187c440de0c2c Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Wed, 3 Apr 2024 16:41:44 -0700 Subject: [PATCH 40/50] refactor(core): Improve task module extraction logic (#2290) Signed-off-by: Kevin Su --- flytekit/core/tracker.py | 5 ++++- flytekit/remote/remote.py | 18 ++++++++++++++++-- .../tests/test_remote_register.py | 4 ++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/flytekit/core/tracker.py b/flytekit/core/tracker.py index 24ac0ffd06..60ae2fbcff 100644 --- a/flytekit/core/tracker.py +++ b/flytekit/core/tracker.py @@ -330,8 +330,11 @@ def extract_task_module(f: Union[Callable, TrackedInstance]) -> Tuple[str, str, if mod_name == "__main__": if hasattr(f, "task_function"): f = f.task_function + # If the module is __main__, we need to find the actual module name based on the file path inspect_file = inspect.getfile(f) # type: ignore - return name, "", name, os.path.abspath(inspect_file) + file_name, _ = os.path.splitext(os.path.basename(inspect_file)) + mod_name = get_full_module_path(f, file_name) # type: ignore + return name, mod_name, name, os.path.abspath(inspect_file) mod_name = get_full_module_path(mod, mod_name) return f"{mod_name}.{name}", mod_name, name, os.path.abspath(inspect.getfile(mod)) diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 437468a57e..0948de5065 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -40,6 +40,7 @@ from flytekit.core.python_auto_container import PythonAutoContainerTask from flytekit.core.reference_entity import ReferenceSpec from flytekit.core.task import ReferenceTask +from flytekit.core.tracker import extract_task_module from flytekit.core.type_engine import LiteralsResolver, TypeEngine from flytekit.core.workflow import ReferenceWorkflow, WorkflowBase, WorkflowFailurePolicy from flytekit.exceptions import user as user_exceptions @@ -82,7 +83,7 @@ from flytekit.remote.remote_fs import get_flyte_fs from flytekit.tools.fast_registration import fast_package from flytekit.tools.interactive import ipython_check -from flytekit.tools.script_mode import compress_scripts, hash_file +from flytekit.tools.script_mode import _find_project_root, compress_scripts, hash_file from flytekit.tools.translator import ( FlyteControlPlaneEntity, FlyteLocalEntity, @@ -778,7 +779,10 @@ def _serialize_and_register( return ident def register_task( - self, entity: PythonTask, serialization_settings: SerializationSettings, version: typing.Optional[str] = None + self, + entity: PythonTask, + serialization_settings: typing.Optional[SerializationSettings] = None, + version: typing.Optional[str] = None, ) -> FlyteTask: """ Register a qualified task (PythonTask) with Remote @@ -789,6 +793,16 @@ def register_task( :param version: version that will be used to register. If not specified will default to using the serialization settings default :return: """ + # Create a default serialization settings object if not provided + # It makes registration easier for the user + if serialization_settings is None: + _, _, _, module_file = extract_task_module(entity) + project_root = _find_project_root(module_file) + serialization_settings = SerializationSettings( + image_config=ImageConfig.auto_default_image(), + source_root=project_root, + ) + ident = self._serialize_and_register(entity=entity, settings=serialization_settings, version=version) ft = self.fetch_task( ident.project, diff --git a/plugins/flytekit-spark/tests/test_remote_register.py b/plugins/flytekit-spark/tests/test_remote_register.py index 3bb65d09bc..606d15d164 100644 --- a/plugins/flytekit-spark/tests/test_remote_register.py +++ b/plugins/flytekit-spark/tests/test_remote_register.py @@ -44,3 +44,7 @@ def my_python_task(a: str) -> int: # Check if the serialized python task has no mainApplicaitonFile field set by default. assert serialized_spec.template.custom is None + + remote.register_task(my_python_task, version="v1") + serialized_spec = mock_client.create_task.call_args.kwargs["task_spec"] + assert serialized_spec.template.custom is None From af79dddae52701b8426b3a452e9bb306885540ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:46:27 -0700 Subject: [PATCH 41/50] Bump pillow from 10.2.0 to 10.3.0 in /plugins/flytekit-onnx-tensorflow (#2320) Bumps [pillow](https://github.com/python-pillow/Pillow) from 10.2.0 to 10.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/10.2.0...10.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- plugins/flytekit-onnx-tensorflow/dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/flytekit-onnx-tensorflow/dev-requirements.txt b/plugins/flytekit-onnx-tensorflow/dev-requirements.txt index 38a63b116c..57155c699b 100644 --- a/plugins/flytekit-onnx-tensorflow/dev-requirements.txt +++ b/plugins/flytekit-onnx-tensorflow/dev-requirements.txt @@ -18,7 +18,7 @@ onnxruntime==1.16.1 # via -r dev-requirements.in packaging==23.2 # via onnxruntime -pillow==10.2.0 +pillow==10.3.0 # via -r dev-requirements.in protobuf==4.25.0 # via onnxruntime From e62743fc962e28cbee853f83e5081548adad3224 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:46:43 -0700 Subject: [PATCH 42/50] Bump pillow from 10.2.0 to 10.3.0 in /plugins/flytekit-onnx-pytorch (#2319) Bumps [pillow](https://github.com/python-pillow/Pillow) from 10.2.0 to 10.3.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/10.2.0...10.3.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- plugins/flytekit-onnx-pytorch/dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/flytekit-onnx-pytorch/dev-requirements.txt b/plugins/flytekit-onnx-pytorch/dev-requirements.txt index 35a9b49a8f..c3a02670dc 100644 --- a/plugins/flytekit-onnx-pytorch/dev-requirements.txt +++ b/plugins/flytekit-onnx-pytorch/dev-requirements.txt @@ -69,7 +69,7 @@ onnxruntime==1.16.1 # via -r dev-requirements.in packaging==23.2 # via onnxruntime -pillow==10.2.0 +pillow==10.3.0 # via # -r dev-requirements.in # torchvision From 423a1eb004fc18fdb966d559cbcf68cef781caa2 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Wed, 3 Apr 2024 16:51:56 -0700 Subject: [PATCH 43/50] Run ignore query (#2322) When a query is detected, don't do anything, and just say that it'll be resolved. If running locally though, it must be specified. Signed-off-by: Yee Hing Tong --- flytekit/clis/sdk_in_container/run.py | 23 +++++++++- flytekit/core/artifact.py | 41 ++++++++++++++++++ flytekit/interaction/click_types.py | 23 +++++++++- .../unit/interaction/test_click_types.py | 43 +++++++++++++++++++ 4 files changed, 127 insertions(+), 3 deletions(-) diff --git a/flytekit/clis/sdk_in_container/run.py b/flytekit/clis/sdk_in_container/run.py index 9f4effe3eb..256e319ae0 100644 --- a/flytekit/clis/sdk_in_container/run.py +++ b/flytekit/clis/sdk_in_container/run.py @@ -26,6 +26,7 @@ from flytekit.configuration import DefaultImages, FastSerializationSettings, ImageConfig, SerializationSettings from flytekit.configuration.plugin import get_plugin from flytekit.core import context_manager +from flytekit.core.artifact import ArtifactQuery from flytekit.core.base_task import PythonTask from flytekit.core.data_persistence import FileAccessProvider from flytekit.core.type_engine import TypeEngine @@ -376,7 +377,7 @@ def to_click_option( description_extra = "" if literal_var.type.simple == SimpleType.STRUCT: - if default_val: + if default_val and not isinstance(default_val, ArtifactQuery): if type(default_val) == dict or type(default_val) == list: default_val = json.dumps(default_val) else: @@ -384,6 +385,9 @@ def to_click_option( if literal_var.type.metadata: description_extra = f": {json.dumps(literal_var.type.metadata)}" + # If a query has been specified, the input is never strictly required at this layer + required = False if default_val and isinstance(default_val, ArtifactQuery) else required + return click.Option( param_decls=[f"--{input_name}"], type=literal_converter.click_type, @@ -508,7 +512,22 @@ def _run(*args, **kwargs): try: inputs = {} for input_name, _ in entity.python_interface.inputs.items(): - inputs[input_name] = kwargs.get(input_name) + processed_click_value = kwargs.get(input_name) + if isinstance(processed_click_value, ArtifactQuery): + if run_level_params.is_remote: + click.secho( + click.style( + f"Input '{input_name}' not passed, supported backends will query" + f" for {processed_click_value.get_str(**kwargs)}", + bold=True, + ) + ) + continue + else: + raise click.UsageError( + f"Default for '{input_name}' is a query, which must be specified when running locally." + ) + inputs[input_name] = processed_click_value if not run_level_params.is_remote: with FlyteContextManager.with_context(_update_flyte_context(run_level_params)): diff --git a/flytekit/core/artifact.py b/flytekit/core/artifact.py index e9a7909809..77532860cd 100644 --- a/flytekit/core/artifact.py +++ b/flytekit/core/artifact.py @@ -186,6 +186,47 @@ def to_flyte_idl( ) -> art_id.ArtifactQuery: return Serializer.artifact_query_to_idl(self, **kwargs) + def get_time_partition_str(self, **kwargs) -> str: + tp_str = "" + if self.time_partition: + tp = self.time_partition.value + if tp.HasField("time_value"): + tp = tp.time_value.ToDatetime() + tp_str += f" Time partition: {tp}" + elif tp.HasField("input_binding"): + var = tp.input_binding.var + if var not in kwargs: + raise ValueError(f"Time partition input binding {var} not found in kwargs") + else: + tp_str += f" Time partition from input<{var}>," + return tp_str + + def get_partition_str(self, **kwargs) -> str: + p_str = "" + if self.partitions and self.partitions.partitions and len(self.partitions.partitions) > 0: + p_str = " Partitions: " + for k, v in self.partitions.partitions.items(): + if v.value and v.value.HasField("static_value"): + p_str += f"{k}={v.value.static_value}, " + elif v.value and v.value.HasField("input_binding"): + var = v.value.input_binding.var + if var not in kwargs: + raise ValueError(f"Partition input binding {var} not found in kwargs") + else: + p_str += f"{k} from input<{var}>, " + return p_str.rstrip("\n\r, ") + + def get_str(self, **kwargs): + # Detailed string that explains query a bit more, used in running + tp_str = self.get_time_partition_str(**kwargs) + p_str = self.get_partition_str(**kwargs) + + return f"'{self.artifact.name}'...{tp_str}{p_str}" + + def __str__(self): + # Default string used for printing --help + return f"Artifact Query: on {self.artifact.name}" + class TimePartition(object): def __init__( diff --git a/flytekit/interaction/click_types.py b/flytekit/interaction/click_types.py index 6ab9f88a25..439dc3cb73 100644 --- a/flytekit/interaction/click_types.py +++ b/flytekit/interaction/click_types.py @@ -14,6 +14,7 @@ from pytimeparse import parse from flytekit import BlobType, FlyteContext, FlyteContextManager, Literal, LiteralType, StructuredDataset +from flytekit.core.artifact import ArtifactQuery from flytekit.core.data_persistence import FileAccessProvider from flytekit.core.type_engine import TypeEngine from flytekit.models.types import SimpleType @@ -61,6 +62,8 @@ class DirParamType(click.ParamType): def convert( self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context] ) -> typing.Any: + if isinstance(value, ArtifactQuery): + return value p = pathlib.Path(value) # set remote_directory to false if running pyflyte run locally. This makes sure that the original # directory is used and not a random one. @@ -80,6 +83,8 @@ class StructuredDatasetParamType(click.ParamType): def convert( self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context] ) -> typing.Any: + if isinstance(value, ArtifactQuery): + return value if isinstance(value, str): return StructuredDataset(uri=value) elif isinstance(value, StructuredDataset): @@ -93,6 +98,8 @@ class FileParamType(click.ParamType): def convert( self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context] ) -> typing.Any: + if isinstance(value, ArtifactQuery): + return value # set remote_directory to false if running pyflyte run locally. This makes sure that the original # file is used and not a random one. remote_path = None if getattr(ctx.obj, "is_remote", False) else False @@ -109,6 +116,8 @@ class PickleParamType(click.ParamType): def convert( self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context] ) -> typing.Any: + if isinstance(value, ArtifactQuery): + return value # set remote_directory to false if running pyflyte run locally. This makes sure that the original # file is used and not a random one. remote_path = None if getattr(ctx.obj, "is_remote", None) else False @@ -131,6 +140,8 @@ def __init__(self): def convert( self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context] ) -> typing.Any: + if isinstance(value, ArtifactQuery): + return value if value in self._ADDITONAL_FORMATS: if value == self._NOW_FMT: return datetime.datetime.now() @@ -143,6 +154,8 @@ class DurationParamType(click.ParamType): def convert( self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context] ) -> typing.Any: + if isinstance(value, ArtifactQuery): + return value if value is None: raise click.BadParameter("None value cannot be converted to a Duration type.") return datetime.timedelta(seconds=parse(value)) @@ -156,6 +169,8 @@ def __init__(self, enum_type: typing.Type[enum.Enum]): def convert( self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context] ) -> enum.Enum: + if isinstance(value, ArtifactQuery): + return value if isinstance(value, self._enum_type): return value return self._enum_type(super().convert(value, param, ctx)) @@ -191,6 +206,8 @@ def convert( Important to implement NoneType / Optional. Also could we just determine the click types from the python types """ + if isinstance(value, ArtifactQuery): + return value for t in self._types: try: return t.convert(value, param, ctx) @@ -228,6 +245,8 @@ def _parse(self, value: typing.Any, param: typing.Optional[click.Parameter]): def convert( self, value: typing.Any, param: typing.Optional[click.Parameter], ctx: typing.Optional[click.Context] ) -> typing.Any: + if isinstance(value, ArtifactQuery): + return value if value is None: raise click.BadParameter("None value cannot be converted to a Json type.") @@ -274,7 +293,7 @@ def modify_literal_uris(lit: Literal): SimpleType.STRING: click.STRING, SimpleType.BOOLEAN: click.BOOL, SimpleType.DURATION: DurationParamType(), - SimpleType.DATETIME: click.DateTime(), + SimpleType.DATETIME: DateTimeType(), } @@ -353,6 +372,8 @@ def convert( """ Convert the value to a Flyte Literal or a python native type. This is used by click to convert the input. """ + if isinstance(value, ArtifactQuery): + return value try: # If the expected Python type is datetime.date, adjust the value to date if self._python_type is datetime.date: diff --git a/tests/flytekit/unit/interaction/test_click_types.py b/tests/flytekit/unit/interaction/test_click_types.py index b5e94a1ff7..83a191c449 100644 --- a/tests/flytekit/unit/interaction/test_click_types.py +++ b/tests/flytekit/unit/interaction/test_click_types.py @@ -10,13 +10,19 @@ import yaml from flytekit import FlyteContextManager +from flytekit.core.artifact import Artifact from flytekit.core.type_engine import TypeEngine from flytekit.interaction.click_types import ( DateTimeType, + DirParamType, DurationParamType, + EnumParamType, FileParamType, FlyteLiteralConverter, JsonParamType, + PickleParamType, + StructuredDatasetParamType, + UnionParamType, key_value_callback, ) @@ -62,6 +68,23 @@ def test_literal_converter(python_type, python_value): assert lc.convert(click_ctx, dummy_param, python_value) == TypeEngine.to_literal(ctx, python_value, python_type, lt) +def test_literal_converter_query(): + ctx = FlyteContextManager.current_context() + lt = TypeEngine.to_literal_type(int) + + lc = FlyteLiteralConverter( + ctx, + literal_type=lt, + python_type=int, + is_remote=True, + ) + + a = Artifact(name="test-artifact") + query = a.query() + click_ctx = click.Context(click.Command("test_command"), obj={"remote": True}) + assert lc.convert(click_ctx, dummy_param, query) == query + + @pytest.mark.parametrize( "python_type, python_str_value, python_value", [ @@ -163,3 +186,23 @@ def test_key_value_callback(): key_value_callback(ctx, "a", ["a=b", "c=d", "e"]) with pytest.raises(click.BadParameter): key_value_callback(ctx, "a", ["a=b", "c=d", "e=f", "g"]) + + +@pytest.mark.parametrize( + "param_type", + [ + (DateTimeType()), + (DurationParamType()), + (JsonParamType(typing.Dict[str, str])), + (UnionParamType([click.FLOAT, click.INT])), + (EnumParamType(Color)), + (PickleParamType()), + (StructuredDatasetParamType()), + (DirParamType()), + ], +) +def test_query_passing(param_type: click.ParamType): + a = Artifact(name="test-artifact") + query = a.query() + + assert param_type.convert(value=query, param=None, ctx=None) is query From f888a5ccfc816eb65209896f90d36bbe300e6d41 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Thu, 4 Apr 2024 07:55:31 +0800 Subject: [PATCH 44/50] Stop requiring users to import `dataclasses_json` or `DataClassJSONMixin` for dataclass (#2279) Signed-off-by: Future-Outlier Co-authored-by: Thomas J. Fan Co-authored-by: Eduardo Apolinario --- flytekit/core/type_engine.py | 68 +++++++++------- pyproject.toml | 2 +- tests/flytekit/unit/core/test_type_engine.py | 84 +++++++++++++++++--- tests/flytekit/unit/core/test_type_hints.py | 12 --- 4 files changed, 114 insertions(+), 52 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 4a5c6bac8d..cf0cccf0f1 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -25,6 +25,7 @@ from google.protobuf.message import Message from google.protobuf.struct_pb2 import Struct from marshmallow_enum import EnumField, LoadDumpOptions +from mashumaro.codecs.json import JSONDecoder, JSONEncoder from mashumaro.mixins.json import DataClassJSONMixin from typing_extensions import Annotated, get_args, get_origin from typing_inspect import is_union_type @@ -328,13 +329,8 @@ class Test(DataClassJsonMixin): def __init__(self): super().__init__("Object-Dataclass-Transformer", object) - self._serializable_classes = [DataClassJSONMixin, DataClassJsonMixin] - try: - from mashumaro.mixins.orjson import DataClassORJSONMixin - - self._serializable_classes.append(DataClassORJSONMixin) - except ModuleNotFoundError: - pass + self._encoder: Dict[Type, JSONEncoder] = {} + self._decoder: Dict[Type, JSONDecoder] = {} def assert_type(self, expected_type: Type[DataClassJsonMixin], v: T): # Skip iterating all attributes in the dataclass if the type of v already matches the expected_type @@ -434,11 +430,6 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: # Drop all annotations and handle only the dataclass type passed in. t = args[0] - if not self.is_serializable_class(t): - raise AssertionError( - f"Dataclass {t} should be decorated with @dataclass_json or mixin with DataClassJSONMixin to be " - f"serialized correctly" - ) schema = None try: if issubclass(t, DataClassJsonMixin): @@ -482,9 +473,6 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: return _type_models.LiteralType(simple=_type_models.SimpleType.STRUCT, metadata=schema, structure=ts) - def is_serializable_class(self, class_: Type[T]) -> bool: - return any(issubclass(class_, serializable_class) for serializable_class in self._serializable_classes) - def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: if isinstance(python_val, dict): json_str = json.dumps(python_val) @@ -495,14 +483,23 @@ def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], exp f"{type(python_val)} is not of type @dataclass, only Dataclasses are supported for " f"user defined datatypes in Flytekit" ) - if not self.is_serializable_class(type(python_val)): - raise TypeTransformerFailedError( - f"Dataclass {python_type} should be decorated with @dataclass_json or inherit DataClassJSONMixin to be " - f"serialized correctly" - ) + self._serialize_flyte_type(python_val, python_type) - json_str = python_val.to_json() # type: ignore + # The `to_json` function is integrated through either the `dataclasses_json` decorator or by inheriting from `DataClassJsonMixin`. + # It serializes a data class into a JSON string. + if hasattr(python_val, "to_json"): + json_str = python_val.to_json() + else: + # The function looks up or creates a JSONEncoder specifically designed for the object's type. + # This encoder is then used to convert a data class into a JSON string. + try: + encoder = self._encoder[python_type] + except KeyError: + encoder = JSONEncoder(python_type) + self._encoder[python_type] = encoder + + json_str = encoder.encode(python_val) return Literal(scalar=Scalar(generic=_json_format.Parse(json_str, _struct.Struct()))) # type: ignore @@ -729,7 +726,7 @@ def _fix_val_int(self, t: typing.Type, val: typing.Any) -> typing.Any: return val - def _fix_dataclass_int(self, dc_type: Type[DataClassJsonMixin], dc: DataClassJsonMixin) -> DataClassJsonMixin: + def _fix_dataclass_int(self, dc_type: Type[dataclasses.dataclass], dc: typing.Any) -> typing.Any: # type: ignore """ This is a performance penalty to convert to the right types, but this is expected by the user and hence needs to be done @@ -738,8 +735,9 @@ def _fix_dataclass_int(self, dc_type: Type[DataClassJsonMixin], dc: DataClassJso # https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Value # Thus we will have to walk the given dataclass and typecast values to int, where expected. for f in dataclasses.fields(dc_type): - val = dc.__getattribute__(f.name) - dc.__setattr__(f.name, self._fix_val_int(f.type, val)) + val = getattr(dc, f.name) + setattr(dc, f.name, self._fix_val_int(f.type, val)) + return dc def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> T: @@ -748,13 +746,23 @@ def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: f"{expected_python_type} is not of type @dataclass, only Dataclasses are supported for " "user defined datatypes in Flytekit" ) - if not self.is_serializable_class(expected_python_type): - raise TypeTransformerFailedError( - f"Dataclass {expected_python_type} should be decorated with @dataclass_json or mixin with DataClassJSONMixin to be " - f"serialized correctly" - ) + json_str = _json_format.MessageToJson(lv.scalar.generic) - dc = expected_python_type.from_json(json_str) # type: ignore + + # The `from_json` function is integrated through either the `dataclasses_json` decorator or by inheriting from `DataClassJsonMixin`. + # It deserializes a JSON string into a data class. + if hasattr(expected_python_type, "from_json"): + dc = expected_python_type.from_json(json_str) # type: ignore + else: + # The function looks up or creates a JSONDecoder specifically designed for the object's type. + # This decoder is then used to convert a JSON string into a data class. + try: + decoder = self._decoder[expected_python_type] + except KeyError: + decoder = JSONDecoder(expected_python_type) + self._decoder[expected_python_type] = decoder + + dc = decoder.decode(json_str) dc = self._fix_structured_dataset_type(expected_python_type, dc) return self._fix_dataclass_int(expected_python_type, self._deserialize_flyte_type(dc, expected_python_type)) diff --git a/pyproject.toml b/pyproject.toml index 45ce9e83a7..f6aa192f7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "markdown-it-py", "marshmallow-enum", "marshmallow-jsonschema>=0.12.0", - "mashumaro>=3.9.1", + "mashumaro>=3.11", "protobuf!=4.25.0", "pyarrow", "pygments", diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 41bf9e5fb5..4df877a84a 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -9,7 +9,7 @@ from dataclasses import asdict, dataclass, field from datetime import timedelta from enum import Enum, auto -from typing import Optional, Type +from typing import List, Optional, Type import mock import pyarrow as pa @@ -25,13 +25,11 @@ from mashumaro.mixins.orjson import DataClassORJSONMixin from typing_extensions import Annotated, get_args, get_origin -from flytekit import kwtypes +from flytekit import dynamic, kwtypes, task, workflow from flytekit.core.annotation import FlyteAnnotation from flytekit.core.context_manager import FlyteContext, FlyteContextManager from flytekit.core.data_persistence import flyte_tmp_dir -from flytekit.core.dynamic_workflow_task import dynamic from flytekit.core.hash import HashMethod -from flytekit.core.task import task from flytekit.core.type_engine import ( DataclassTransformer, DictTransformer, @@ -2648,16 +2646,46 @@ class DatumMashumaro(DataClassJSONMixin): @dataclass_json @dataclass - class Datum(DataClassJSONMixin): + class DatumDataclassJson(DataClassJSONMixin): x: int y: Color - transformer = DataclassTransformer() + @dataclass + class DatumDataclass: + x: int + y: Color + + @dataclass + class DatumDataUnion: + data: typing.Union[str, float] + + transformer = TypeEngine.get_transformer(DatumDataUnion) ctx = FlyteContext.current_context() - lt = TypeEngine.to_literal_type(Datum) - datum = Datum(5, Color.RED) - lv = transformer.to_literal(ctx, datum, Datum, lt) + lt = TypeEngine.to_literal_type(DatumDataUnion) + datum_dataunion = DatumDataUnion(data="s3://my-file") + lv = transformer.to_literal(ctx, datum_dataunion, DatumDataUnion, lt) + gt = transformer.guess_python_type(lt) + pv = transformer.to_python_value(ctx, lv, expected_python_type=DatumDataUnion) + assert datum_dataunion.data == pv.data + + datum_dataunion = DatumDataUnion(data="0.123") + lv = transformer.to_literal(ctx, datum_dataunion, DatumDataUnion, lt) + gt = transformer.guess_python_type(lt) + pv = transformer.to_python_value(ctx, lv, expected_python_type=gt) + assert datum_dataunion.data == pv.data + + lt = TypeEngine.to_literal_type(DatumDataclass) + datum_dataclass = DatumDataclass(5, Color.RED) + lv = transformer.to_literal(ctx, datum_dataclass, DatumDataclass, lt) + gt = transformer.guess_python_type(lt) + pv = transformer.to_python_value(ctx, lv, expected_python_type=gt) + assert datum_dataclass.x == pv.x + assert datum_dataclass.y.value == pv.y + + lt = TypeEngine.to_literal_type(DatumDataclassJson) + datum = DatumDataclassJson(5, Color.RED) + lv = transformer.to_literal(ctx, datum, DatumDataclassJson, lt) gt = transformer.guess_python_type(lt) pv = transformer.to_python_value(ctx, lv, expected_python_type=gt) assert datum.x == pv.x @@ -2682,6 +2710,44 @@ class Datum(DataClassJSONMixin): assert datum_mashumaro_orjson.z.isoformat() == pv.z +def test_dataclass_encoder_and_decoder_registry(): + iterations = 10 + + @dataclass + class Datum: + x: int + y: str + z: dict[int, int] + w: List[int] + + @task + def create_dataclasses() -> List[Datum]: + return [Datum(x=1, y="1", z={1: 1}, w=[1, 1, 1, 1])] + + @task + def concat_dataclasses(x: List[Datum], y: List[Datum]) -> List[Datum]: + return x + y + + @dynamic + def dynamic_wf() -> List[Datum]: + all_dataclasses: List[Datum] = [] + for _ in range(iterations): + data = create_dataclasses() + all_dataclasses = concat_dataclasses(x=all_dataclasses, y=data) + return all_dataclasses + + @workflow + def wf() -> List[Datum]: + return dynamic_wf() + + datum_list = wf() + assert len(datum_list) == iterations + + transformer = TypeEngine.get_transformer(Datum) + assert transformer._encoder.get(Datum) + assert transformer._decoder.get(Datum) + + def test_ListTransformer_get_sub_type(): assert ListTransformer.get_sub_type_or_none(typing.List[str]) is str diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index 3c4f76316d..00410f74d6 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -1102,18 +1102,6 @@ def wf(): wf() -def test_wf_custom_types_missing_dataclass_json(): - with pytest.raises(AssertionError): - - @dataclass - class MyCustomType(object): - pass - - @task - def t1(a: int) -> MyCustomType: - return MyCustomType() - - def test_wf_custom_types(): @dataclass class MyCustomType(DataClassJsonMixin): From a478635f820cdafc599c6e3264048f805e4d7faa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Apr 2024 19:51:24 -0700 Subject: [PATCH 45/50] Bump cryptography (#2206) Bumps [cryptography](https://github.com/pyca/cryptography) from 42.0.2 to 42.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/42.0.2...42.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../remote/mock_flyte_repo/workflows/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 5fe080f60f..63b2f9899b 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -68,7 +68,7 @@ cookiecutter==2.2.3 # via flytekit croniter==1.4.1 # via flytekit -cryptography==42.0.2 +cryptography==42.0.4 # via # azure-identity # azure-storage-blob From e121d35009b6b91112c1ba1a007edcce2e18d7d7 Mon Sep 17 00:00:00 2001 From: pbrogan12 Date: Wed, 3 Apr 2024 23:25:39 -0400 Subject: [PATCH 46/50] fix databricks job request serialization (#2286) Signed-off-by: Patrick Brogan --- flytekit/models/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/models/task.py b/flytekit/models/task.py index 198adf2859..5072f03757 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -939,7 +939,7 @@ def from_flyte_idl(cls, pb2_object): return cls( image=pb2_object.image, command=pb2_object.command, - args=pb2_object.args, + args=[arg for arg in pb2_object.args], resources=Resources.from_flyte_idl(pb2_object.resources), env={kv.key: kv.value for kv in pb2_object.env}, config={kv.key: kv.value for kv in pb2_object.config}, From ce0d2cbdb163e3f333b107b1fcd9c56cfe75a377 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Thu, 4 Apr 2024 12:50:52 -0700 Subject: [PATCH 47/50] Add name to union param (#2329) Signed-off-by: Yee Hing Tong --- flytekit/interaction/click_types.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flytekit/interaction/click_types.py b/flytekit/interaction/click_types.py index 439dc3cb73..94e9f1a88a 100644 --- a/flytekit/interaction/click_types.py +++ b/flytekit/interaction/click_types.py @@ -185,6 +185,10 @@ def __init__(self, types: typing.List[click.ParamType]): super().__init__() self._types = self._sort_precedence(types) + @property + def name(self) -> str: + return "|".join([t.name for t in self._types]) + @staticmethod def _sort_precedence(tp: typing.List[click.ParamType]) -> typing.List[click.ParamType]: unprocessed = [] From 1d6636ed357121e6eb2bf21606c6f6681f9b579b Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Thu, 4 Apr 2024 16:19:17 -0400 Subject: [PATCH 48/50] Remove typing-inspect dependency (#2327) * Remove typing-inspect dependencies Signed-off-by: Thomas J. Fan * DOC Adds docstring Signed-off-by: Thomas J. Fan --------- Signed-off-by: Thomas J. Fan --- flytekit/core/type_engine.py | 17 +++++++++++++++-- pyproject.toml | 1 - 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index cf0cccf0f1..7f0ddd445a 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -28,7 +28,6 @@ from mashumaro.codecs.json import JSONDecoder, JSONEncoder from mashumaro.mixins.json import DataClassJSONMixin from typing_extensions import Annotated, get_args, get_origin -from typing_inspect import is_union_type from flytekit.core.annotation import FlyteAnnotation from flytekit.core.context_manager import FlyteContext @@ -1516,6 +1515,19 @@ def _are_types_castable(upstream: LiteralType, downstream: LiteralType) -> bool: return False +def _is_union_type(t): + """Returns True if t is a Union type.""" + + if sys.version_info >= (3, 10): + import types + + UnionType = types.UnionType + else: + UnionType = None + + return t is typing.Union or get_origin(t) is Union or UnionType and isinstance(t, UnionType) + + class UnionTransformer(TypeTransformer[T]): """ Transformer that handles a typing.Union[T1, T2, ...] @@ -1526,7 +1538,8 @@ def __init__(self): @staticmethod def is_optional_type(t: Type[T]) -> bool: - return is_union_type(t) and type(None) in get_args(t) + """Return True if `t` is a Union or Optional type.""" + return _is_union_type(t) or type(None) in get_args(t) @staticmethod def get_sub_type_in_optional(t: Type[T]) -> Type[T]: diff --git a/pyproject.toml b/pyproject.toml index f6aa192f7e..0373db6f67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,6 @@ dependencies = [ "s3fs>=2023.3.0,!=2024.3.1", "statsd>=3.0.0,<4.0.0", "typing_extensions", - "typing-inspect", "urllib3>=1.22,<2.0.0", ] classifiers = [ From 0a7a72aabc83f648f6979517a4f61bb5240149b7 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 5 Apr 2024 01:49:57 +0530 Subject: [PATCH 49/50] don't return triton image as a functools.partial (#2326) * move triton image code from mixin to task Signed-off-by: Samhita Alla * don't return triton image partial Signed-off-by: Samhita Alla --------- Signed-off-by: Samhita Alla --- .../flytekitplugins/awssagemaker_inference/__init__.py | 8 ++------ .../flytekitplugins/awssagemaker_inference/boto3_mixin.py | 5 ++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py index fa58efbb89..c00952f540 100644 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/__init__.py @@ -19,8 +19,6 @@ delete_sagemaker_deployment """ -from functools import partial - from .agent import SageMakerEndpointAgent from .boto3_agent import BotoAgent from .boto3_task import BotoConfig, BotoTask @@ -37,7 +35,5 @@ def triton_image_uri(version: str = "23.12"): - return partial( - "{account_id}.dkr.ecr.{region}.{base}/sagemaker-tritonserver:{version}-py3".format, - version=version, - ) + image = "{account_id}.dkr.ecr.{region}.{base}/sagemaker-tritonserver:{version}-py3" + return image.replace("{version}", version) diff --git a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py index 56be7dca72..98b1c513f1 100644 --- a/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py +++ b/plugins/flytekit-aws-sagemaker/flytekitplugins/awssagemaker_inference/boto3_mixin.py @@ -1,4 +1,3 @@ -from functools import partial from typing import Any, Dict, Optional import aioboto3 @@ -152,12 +151,12 @@ async def _call( base = "amazonaws.com.cn" if final_region.startswith("cn-") else "amazonaws.com" images = { image_name: ( - image( + image.format( account_id=account_id_map[final_region], region=final_region, base=base, ) - if isinstance(image, partial) + if isinstance(image, str) and "sagemaker-tritonserver" in image else image ) for image_name, image in images.items() From f4d894af2ebb66f4492506ebb121509ef8cb8aa3 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Date: Thu, 4 Apr 2024 13:20:12 -0700 Subject: [PATCH 50/50] Set a minimum version to mlflow in flytekit plugin (#2324) * Set `mlflow>=2.10.0` Signed-off-by: Eduardo Apolinario * Run mlflow tests on python 3.11. Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario --- .github/workflows/pythonbuild.yml | 4 ---- plugins/flytekit-mlflow/setup.py | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 7b308ad634..c2e5d96544 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -350,10 +350,6 @@ jobs: plugin-names: "flytekit-onnx-scikitlearn" - python-version: 3.11 plugin-names: "flytekit-onnx-tensorflow" - # numba, a dependency of mlflow, doesn't support python 3.11 - # https://github.com/numba/numba/issues/8304 - - python-version: 3.11 - plugin-names: "flytekit-mlflow" # vaex currently doesn't support python 3.11 - python-version: 3.11 plugin-names: "flytekit-vaex" diff --git a/plugins/flytekit-mlflow/setup.py b/plugins/flytekit-mlflow/setup.py index 666aff4316..8074f4ed06 100644 --- a/plugins/flytekit-mlflow/setup.py +++ b/plugins/flytekit-mlflow/setup.py @@ -4,8 +4,7 @@ microlib_name = f"flytekitplugins-{PLUGIN_NAME}" -# TODO: support mlflow 2.0+ -plugin_requires = ["flytekit>=1.1.0,<2.0.0", "plotly", "mlflow<2.0.0", "pandas"] +plugin_requires = ["flytekit>=1.1.0,<2.0.0", "plotly", "mlflow>=2.10.0", "pandas"] __version__ = "0.0.0+develop" @@ -27,6 +26,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development",