Merge branch 'master' into dict-bug

flytekit/configuration/__init__.py

@@ -53,6 +53,7 @@
    ~Image
    ~ImageConfig
    ~SerializationSettings
+   ~FastSerializationSettings
 
 .. _configuration-execution-time-settings:
 

plugins/flytekit-deck-standard/requirements.txt

@@ -6,10 +6,18 @@
 #
 -e file:.#egg=flytekitplugins-deck-standard
     # via -r requirements.in
+appnope==0.1.3
+    # via
+    #   ipykernel
+    #   ipython
 arrow==1.2.3
     # via jinja2-time
+asttokens==2.2.1
+    # via stack-data
 attrs==22.1.0
     # via visions
+backcall==0.2.0
+    # via ipython
 binaryornot==0.4.4
     # via cookiecutter
 certifi==2022.12.7
@@ -26,22 +34,26 @@ click==8.1.3
     #   flytekit
 cloudpickle==2.2.0
     # via flytekit
+comm==0.1.2
+    # via ipykernel
 contourpy==1.0.6
     # via matplotlib
 cookiecutter==2.1.1
     # via flytekit
 croniter==1.3.8
     # via flytekit
 cryptography==38.0.4
-    # via
-    #   pyopenssl
-    #   secretstorage
+    # via pyopenssl
 cycler==0.11.0
     # via matplotlib
 dataclasses-json==0.5.7
     # via flytekit
+debugpy==1.6.4
+    # via ipykernel
 decorator==5.1.1
-    # via retry
+    # via
+    #   ipython
+    #   retry
 deprecated==1.2.13
     # via flytekit
 diskcache==5.4.0
@@ -52,6 +64,10 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.15
     # via flytekit
+entrypoints==0.4
+    # via jupyter-client
+executing==1.2.0
+    # via stack-data
 flyteidl==1.3.0
     # via flytekit
 flytekit==1.3.0b2
@@ -79,12 +95,18 @@ importlib-metadata==5.1.0
     #   flytekit
     #   keyring
     #   markdown
+ipykernel==6.19.4
+    # via ipywidgets
+ipython==8.7.0
+    # via
+    #   ipykernel
+    #   ipywidgets
+ipywidgets==8.0.4
+    # via flytekitplugins-deck-standard
 jaraco-classes==3.2.3
     # via keyring
-jeepney==0.8.0
-    # via
-    #   keyring
-    #   secretstorage
+jedi==0.18.2
+    # via ipython
 jinja2==3.1.2
     # via
     #   cookiecutter
@@ -96,6 +118,12 @@ joblib==1.2.0
     # via
     #   flytekit
     #   phik
+jupyter-client==7.4.8
+    # via ipykernel
+jupyter-core==5.1.1
+    # via jupyter-client
+jupyterlab-widgets==3.0.5
+    # via ipywidgets
 keyring==23.11.0
     # via flytekit
 kiwisolver==1.4.4
@@ -118,6 +146,10 @@ matplotlib==3.6.2
     #   pandas-profiling
     #   phik
     #   seaborn
+matplotlib-inline==0.1.6
+    # via
+    #   ipykernel
+    #   ipython
 more-itertools==9.0.0
     # via jaraco-classes
 multimethod==1.9
@@ -128,6 +160,10 @@ mypy-extensions==0.4.3
     # via typing-inspect
 natsort==8.2.0
     # via flytekit
+nest-asyncio==1.5.6
+    # via
+    #   ipykernel
+    #   jupyter-client
 networkx==2.8.8
     # via visions
 numpy==1.23.5
@@ -148,6 +184,7 @@ numpy==1.23.5
 packaging==22.0
     # via
     #   docker
+    #   ipykernel
     #   marshmallow
     #   matplotlib
     #   statsmodels
@@ -161,17 +198,27 @@ pandas==1.5.2
     #   visions
 pandas-profiling==3.5.0
     # via flytekitplugins-deck-standard
+parso==0.8.3
+    # via jedi
 patsy==0.5.3
     # via statsmodels
+pexpect==4.8.0
+    # via ipython
 phik==0.12.3
     # via pandas-profiling
+pickleshare==0.7.5
+    # via ipython
 pillow==9.3.0
     # via
     #   imagehash
     #   matplotlib
     #   visions
+platformdirs==2.6.0
+    # via jupyter-core
 plotly==5.11.0
     # via flytekitplugins-deck-standard
+prompt-toolkit==3.0.36
+    # via ipython
 protobuf==4.21.11
     # via
     #   flyteidl
@@ -180,6 +227,12 @@ protobuf==4.21.11
     #   protoc-gen-swagger
 protoc-gen-swagger==0.1.0
     # via flyteidl
+psutil==5.9.4
+    # via ipykernel
+ptyprocess==0.7.0
+    # via pexpect
+pure-eval==0.2.2
+    # via stack-data
 py==1.11.0
     # via retry
 pyarrow==10.0.1
@@ -188,6 +241,8 @@ pycparser==2.21
     # via cffi
 pydantic==1.10.2
     # via pandas-profiling
+pygments==2.13.0
+    # via ipython
 pyopenssl==22.1.0
     # via flytekit
 pyparsing==3.0.9
@@ -197,6 +252,7 @@ python-dateutil==2.8.2
     #   arrow
     #   croniter
     #   flytekit
+    #   jupyter-client
     #   matplotlib
     #   pandas
 python-json-logger==2.0.4
@@ -216,6 +272,10 @@ pyyaml==6.0
     #   cookiecutter
     #   flytekit
     #   pandas-profiling
+pyzmq==24.0.1
+    # via
+    #   ipykernel
+    #   jupyter-client
 regex==2022.10.31
     # via docker-image-py
 requests==2.28.1
@@ -237,14 +297,15 @@ scipy==1.9.3
     #   statsmodels
 seaborn==0.12.1
     # via pandas-profiling
-secretstorage==3.3.3
-    # via keyring
 six==1.16.0
     # via
+    #   asttokens
     #   patsy
     #   python-dateutil
 sortedcontainers==2.4.0
     # via flytekit
+stack-data==0.6.2
+    # via ipython
 statsd==3.3.0
     # via flytekit
 statsmodels==0.13.5
@@ -257,8 +318,21 @@ text-unidecode==1.3
     # via python-slugify
 toml==0.10.2
     # via responses
+tornado==6.2
+    # via
+    #   ipykernel
+    #   jupyter-client
 tqdm==4.64.1
     # via pandas-profiling
+traitlets==5.8.0
+    # via
+    #   comm
+    #   ipykernel
+    #   ipython
+    #   ipywidgets
+    #   jupyter-client
+    #   jupyter-core
+    #   matplotlib-inline
 typeguard==2.13.3
     # via pandas-profiling
 types-toml==0.10.8.1
@@ -278,10 +352,14 @@ urllib3==1.26.13
     #   responses
 visions[type_image_path]==0.7.5
     # via pandas-profiling
+wcwidth==0.2.5
+    # via prompt-toolkit
 websocket-client==1.4.2
     # via docker
 wheel==0.38.4
     # via flytekit
+widgetsnbextension==4.0.5
+    # via ipywidgets
 wrapt==1.14.1
     # via
     #   deprecated

plugins/flytekit-deck-standard/setup.py

@@ -4,7 +4,7 @@
 
 microlib_name = f"flytekitplugins-{PLUGIN_NAME}-standard"
 
-plugin_requires = ["flytekit>=1.3.0b2,<2.0.0", "markdown", "plotly", "pandas_profiling"]
+plugin_requires = ["flytekit>=1.3.0b2,<2.0.0", "markdown", "plotly", "pandas_profiling", "ipywidgets"]
 
 __version__ = "0.0.0+develop"
 

plugins/flytekit-k8s-pod/flytekitplugins/pod/task.py

@@ -13,6 +13,7 @@
 from flytekit.models import task as _task_models
 
 _PRIMARY_CONTAINER_NAME_FIELD = "primary_container_name"
+PRIMARY_CONTAINER_DEFAULT_NAME = "primary"
 
 
 def _sanitize_resource_name(resource: _task_models.Resources.ResourceEntry) -> str:
@@ -26,17 +27,19 @@ class Pod(object):
     This plugin helps expose a fully modifiable Kubernetes pod spec to customize the task execution runtime.
     To use pod tasks: (1) Define a pod spec, and (2) Specify the primary container name.
     :param V1PodSpec pod_spec: Kubernetes pod spec. https://kubernetes.io/docs/concepts/workloads/pods
-    :param str primary_container_name: the primary container name
+    :param str primary_container_name: the primary container name. If provided the pod-spec can contain a container whose name matches the primary_container_name. This will force Flyte to give up control of the primary
+             container and will expect users to control setting up the container. If you expect your python function to run as is, simply create containers that do not match the default primary-container-name and Flyte will auto-inject a
+             container for the python function based on the default image provided during serialization.
     :param Optional[Dict[str, str]] labels: Labels are key/value pairs that are attached to pod spec
     :param Optional[Dict[str, str]] annotations: Annotations are key/value pairs that are attached to arbitrary non-identifying metadata to pod spec.
     """
 
     pod_spec: V1PodSpec
-    primary_container_name: str = _PRIMARY_CONTAINER_NAME_FIELD
+    primary_container_name: str = PRIMARY_CONTAINER_DEFAULT_NAME
     labels: Optional[Dict[str, str]] = None
     annotations: Optional[Dict[str, str]] = None
 
-    def __post_init_(self):
+    def __post_init__(self):
         if not self.pod_spec:
             raise _user_exceptions.FlyteValidationException("A pod spec cannot be undefined")
         if not self.primary_container_name:

plugins/flytekit-k8s-pod/tests/test_pod.py

@@ -12,6 +12,7 @@
 from flytekit.configuration import FastSerializationSettings, Image, ImageConfig, SerializationSettings
 from flytekit.core import context_manager
 from flytekit.core.type_engine import TypeEngine
+from flytekit.exceptions import user
 from flytekit.extend import ExecutionState
 from flytekit.tools.translator import get_serializable
 
@@ -473,3 +474,32 @@ def dynamic_task_with_pod_subtask(dummy_input: str) -> str:
             assert dynamic_job_spec.tasks[0].k8s_pod.pod_spec["containers"][0]["resources"]["requests"]["gpu"] == "1"
 
     assert context_manager.FlyteContextManager.size() == 1
+
+
+def test_pod_config():
+    with pytest.raises(user.FlyteValidationException):
+        Pod(pod_spec=None)
+
+    with pytest.raises(user.FlyteValidationException):
+        Pod(pod_spec=V1PodSpec(containers=[]), primary_container_name=None)
+
+    selector = {"node_group": "memory"}
+
+    @task(
+        task_config=Pod(
+            pod_spec=V1PodSpec(
+                containers=[],
+                node_selector=selector,
+            ),
+        ),
+        requests=Resources(
+            mem="1G",
+        ),
+    )
+    def my_pod_task():
+        print("hello world")
+        time.sleep(30000)
+
+    assert my_pod_task.task_config
+    assert isinstance(my_pod_task.task_config, Pod)
+    assert my_pod_task.task_config.pod_spec.node_selector == selector