Update Flyte components (#2773)

Signed-off-by: Flyte-Bot <[email protected]>

CHANGELOG/CHANGELOG-v1.2.0-b2.md

@@ -0,0 +1,5 @@
+# Flyte v1.2.0-b2 Changelog
+
+[Add support Ray task](https://github.com/flyteorg/flyte/issues/2641)
+
+Others will be filled in later

boilerplate/flyte/end2end/Makefile

@@ -5,8 +5,7 @@
 
 .PHONY: end2end_execute
 end2end_execute:
-	# TODO: These arguments could come from environment variables
-	./boilerplate/flyte/end2end/end2end.sh ./boilerplate/flyte/end2end/functional-test.config --return_non_zero_on_failure
+	./boilerplate/flyte/end2end/end2end.sh ./boilerplate/flyte/end2end/functional-test-config.yaml --return_non_zero_on_failure
 
 .PHONY: k8s_integration_execute
 k8s_integration_execute:

boilerplate/flyte/end2end/end2end.sh

@@ -9,8 +9,9 @@ set -e
 CONFIG_FILE=$1; shift
 EXTRA_FLAGS=( "$@" )
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+# By default only execute `core` tests
+PRIORITIES="${PRIORITIES:-P0}"
 
 LATEST_VERSION=$(curl --silent "https://api.github.com/repos/flyteorg/flytesnacks/releases/latest" | jq -r .tag_name)
 
-python ./boilerplate/flyte/end2end/run-tests.py $LATEST_VERSION P0,P1 $CONFIG_FILE ${EXTRA_FLAGS[@]}
+python ./boilerplate/flyte/end2end/run-tests.py $LATEST_VERSION $PRIORITIES $CONFIG_FILE ${EXTRA_FLAGS[@]}

boilerplate/flyte/end2end/functional-test-config.yaml

@@ -0,0 +1,5 @@
+admin:
+  # For GRPC endpoints you might want to use dns:///flyte.myexample.com
+  endpoint: localhost:30081
+  authType: Pkce
+  insecure: true

boilerplate/flyte/end2end/run-tests.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import click
+import datetime
 import json
 import sys
 import time
@@ -10,17 +11,18 @@
 from flytekit.remote import FlyteRemote
 from flytekit.models.core.execution import WorkflowExecutionPhase
 from flytekit.configuration import Config, ImageConfig, SerializationSettings
+from flytekit.remote.executions import FlyteWorkflowExecution
 
 
 WAIT_TIME = 10
-MAX_ATTEMPTS = 60
+MAX_ATTEMPTS = 200
 
 # This dictionary maps the names found in the flytesnacks manifest to a list of workflow names and
 # inputs. This is so we can progressively cover all priorities in the original flytesnacks manifest,
 # starting with "core".
 FLYTESNACKS_WORKFLOW_GROUPS: Mapping[str, List[Tuple[str, dict]]] = {
     "core": [
-        ("core.control_flow.chain_tasks.chain_tasks_wf", {}),
+        ("core.control_flow.chain_entities.chain_workflows_wf", {}),
         ("core.control_flow.dynamics.wf", {"s1": "Pear", "s2": "Earth"}),
         ("core.control_flow.map_task.my_map_workflow", {"a": [1, 2, 3, 4, 5]}),
         # Workflows that use nested executions cannot be launched via flyteremote.
@@ -50,70 +52,116 @@
         # ("core.type_system.enums.enum_wf", {"c": "red"}),
         ("core.type_system.schema.df_wf", {"a": 42}),
         ("core.type_system.typed_schema.wf", {}),
-        ("my.imperative.workflow.example", {"in1": "hello", "in2": "foo"}),
+        #("my.imperative.workflow.example", {"in1": "hello", "in2": "foo"}),
+    ],
+    "integrations-k8s-spark": [
+        ("k8s_spark.pyspark_pi.my_spark", {"triggered_date": datetime.datetime.now()}),
+    ],
+    "integrations-kfpytorch": [
+        ("kfpytorch.pytorch_mnist.pytorch_training_wf", {}),
+    ],
+    "integrations-kftensorflow": [
+        ("kftensorflow.tf_mnist.mnist_tensorflow_workflow", {}),
+    ],
+    # "integrations-pod": [
+    #     ("pod.pod.pod_workflow", {}),
+    # ],
+    "integrations-pandera_examples": [
+        ("pandera_examples.basic_schema_example.process_data", {}),
+        # TODO: investigate type mismatch float -> numpy.float64
+        # ("pandera_examples.validating_and_testing_ml_pipelines.pipeline", {"data_random_state": 42, "model_random_state": 99}),
+    ],
+    "integrations-modin_examples": [
+        ("modin_examples.knn_classifier.pipeline", {}),
+    ],
+    "integrations-papermilltasks": [
+        ("papermilltasks.simple.nb_to_python_wf", {"f": 3.1415926535}),
+    ],
+    "integrations-greatexpectations": [
+        ("greatexpectations.task_example.simple_wf", {}),
+        ("greatexpectations.task_example.file_wf", {}),
+        ("greatexpectations.task_example.schema_wf", {}),
+        ("greatexpectations.task_example.runtime_wf", {}),
     ],
 }
 
 
-def run_launch_plan(remote, version, workflow_name, inputs):
+def execute_workflow(remote, version, workflow_name, inputs):
     print(f"Fetching workflow={workflow_name} and version={version}")
-    lp = remote.fetch_workflow(name=workflow_name, version=version)
-    return remote.execute(lp, inputs=inputs, wait=False)
+    wf = remote.fetch_workflow(name=workflow_name, version=version)
+    return remote.execute(wf, inputs=inputs, wait=False)
+
+def executions_finished(executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]]) -> bool:
+    for executions in executions_by_wfgroup.values():
+        if not all([execution.is_done for execution in executions]):
+            return False
+    return True
+
+def sync_executions(remote: FlyteRemote, executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]]):
+    try:
+        for executions in executions_by_wfgroup.values():
+            for execution in executions:
+                print(f"About to sync execution_id={execution.id.name}")
+                remote.sync(execution)
+    except:
+        print("GOT TO THE EXCEPT")
+        print("COUNT THIS!")
 
 
-def schedule_workflow_group(
+def report_executions(executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]]):
+    for executions in executions_by_wfgroup.values():
+        for execution in executions:
+            print(execution)
+
+def schedule_workflow_groups(
     tag: str,
-    workflow_group: str,
+    workflow_groups: List[str],
     remote: FlyteRemote,
     terminate_workflow_on_failure: bool,
-) -> bool:
+) -> Dict[str, bool]:
     """
-    Schedule all workflows executions and return True if all executions succeed, otherwise
+    Schedule workflows executions for all workflow gropus and return True if all executions succeed, otherwise
     return False.
     """
-    workflows = FLYTESNACKS_WORKFLOW_GROUPS.get(workflow_group, [])
-
-    launch_plans = [
-        run_launch_plan(remote, tag, workflow[0], workflow[1]) for workflow in workflows
-    ]
+    executions_by_wfgroup = {}
+    # Schedule executions for each workflow group,
+    for wf_group in workflow_groups:
+        workflows = FLYTESNACKS_WORKFLOW_GROUPS.get(wf_group, [])
+        executions_by_wfgroup[wf_group] = [
+            execute_workflow(remote, tag, workflow[0], workflow[1]) for workflow in workflows
+        ]
 
-    # Wait for all launch plans to finish
+    # Wait for all executions to finish
     attempt = 0
     while attempt == 0 or (
-        not all([lp.is_done for lp in launch_plans]) and attempt < MAX_ATTEMPTS
+        not executions_finished(executions_by_wfgroup) and attempt < MAX_ATTEMPTS
     ):
         attempt += 1
         print(
             f"Not all executions finished yet. Sleeping for some time, will check again in {WAIT_TIME}s"
         )
         time.sleep(WAIT_TIME)
-        # Need to sync to refresh status of executions
-        for lp in launch_plans:
-            print(f"About to sync execution_id={lp.id.name}")
-            remote.sync(lp)
-
-    # Report result of each launch plan
-    for lp in launch_plans:
-        print(lp)
-
-    # Collect all failing launch plans
-    non_succeeded_lps = [
-        lp
-        for lp in launch_plans
-        if lp.closure.phase != WorkflowExecutionPhase.SUCCEEDED
-    ]
+        sync_executions(remote, executions_by_wfgroup)
 
-    if len(non_succeeded_lps) == 0:
-        print("All executions succeeded.")
-        return True
 
-    print("Failed executions:")
-    # Report failing cases
-    for lp in non_succeeded_lps:
-        print(f"    workflow={lp.spec.launch_plan.name}, execution_id={lp.id.name}")
-        if terminate_workflow_on_failure:
-            remote.terminate(lp, "aborting execution scheduled in functional test")
-    return False
+    report_executions(executions_by_wfgroup)
+
+    results = {}
+    for wf_group, executions in executions_by_wfgroup.items():
+        non_succeeded_executions = []
+        for execution in executions:
+            if execution.closure.phase != WorkflowExecutionPhase.SUCCEEDED:
+                non_succeeded_executions.append(execution)
+        # Report failing cases
+        if len(non_succeeded_executions) != 0:
+            print(f"Failed executions for {wf_group}:")
+            for execution in non_succeeded_executions:
+                print(f"    workflow={execution.spec.launch_plan.name}, execution_id={execution.id.name}")
+                if terminate_workflow_on_failure:
+                    remote.terminate(execution, "aborting execution scheduled in functional test")
+        # A workflow group succeeds iff all of its executions succeed
+        results[wf_group] = len(non_succeeded_executions) == 0
+    return results
 
 
 def valid(workflow_group):
@@ -147,6 +195,7 @@ def run(
         group["name"] for group in parsed_manifest if group["priority"] in priorities
     ]
     results = []
+    valid_workgroups = []
     for workflow_group in workflow_groups:
         if not valid(workflow_group):
             results.append(
@@ -157,19 +206,17 @@ def run(
                 }
             )
             continue
+        valid_workgroups.append(workflow_group)
 
-        try:
-            workflows_succeeded = schedule_workflow_group(
-                flytesnacks_release_tag,
-                workflow_group,
-                remote,
-                terminate_workflow_on_failure,
-            )
-        except Exception:
-            print(traceback.format_exc())
-            workflows_succeeded = False
+    results_by_wfgroup = schedule_workflow_groups(
+        flytesnacks_release_tag,
+        valid_workgroups,
+        remote,
+        terminate_workflow_on_failure
+    )
 
-        if workflows_succeeded:
+    for workflow_group, succeeded in results_by_wfgroup.items():
+        if succeeded:
             background_color = "green"
             status = "passing"
         else:

boilerplate/update.cfg

@@ -1,3 +1,2 @@
 flyte/end2end
-flyte/welcome_bot
 flyte/code_of_conduct

charts/flyte-core/README.md

@@ -139,7 +139,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.extraArgs | object | `{}` | Appends extra command line arguments to the serve command |
 | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment |
-| flyteadmin.image.tag | string | `"v1.1.29-hotfix"` |  |
+| flyteadmin.image.tag | string | `"v1.1.34"` |  |
 | flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create |
 | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment |
 | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods |
@@ -161,7 +161,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteconsole.ga.tracking_id | string | `"G-0QW4DJWJ20"` |  |
 | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment |
-| flyteconsole.image.tag | string | `"v1.1.6"` |  |
+| flyteconsole.image.tag | string | `"v1.2.4"` |  |
 | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment |
 | flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods |
 | flyteconsole.priorityClassName | string | `""` | Sets priorityClassName for flyte console pod(s). |
@@ -178,7 +178,7 @@ helm install gateway bitnami/contour -n flyte
 | flytepropeller.extraArgs | object | `{}` | Appends extra command line arguments to the main command |
 | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment |
-| flytepropeller.image.tag | string | `"v1.1.15"` |  |
+| flytepropeller.image.tag | string | `"v1.1.26"` |  |
 | flytepropeller.manager | bool | `false` |  |
 | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment |
 | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods |
@@ -194,7 +194,7 @@ helm install gateway bitnami/contour -n flyte
 | flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files |
 | flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment |
-| flytescheduler.image.tag | string | `"v1.1.30"` | Docker image tag |
+| flytescheduler.image.tag | string | `"v1.1.34"` | Docker image tag |
 | flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment |
 | flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods |
 | flytescheduler.priorityClassName | string | `""` | Sets priorityClassName for flyte scheduler pod(s). |

charts/flyte-core/values.yaml

@@ -16,7 +16,7 @@ flyteadmin:
   image:
     # -- Docker image for Flyteadmin deployment
     repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE
-    tag: v1.1.29-hotfix # FLYTEADMIN_TAG
+    tag: v1.1.34 # FLYTEADMIN_TAG
     pullPolicy: IfNotPresent
   # -- Additional flyteadmin container environment variables
   #
@@ -93,7 +93,7 @@ flytescheduler:
     # -- Docker image for Flytescheduler deployment
     repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE
     # -- Docker image tag
-    tag: v1.1.30 # FLYTESCHEDULER_TAG
+    tag: v1.1.34 # FLYTESCHEDULER_TAG
     # -- Docker image pull policy
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flytescheduler deployment
@@ -196,7 +196,7 @@ flytepropeller:
   image:
     # -- Docker image for Flytepropeller deployment
     repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE
-    tag: v1.1.15 # FLYTEPROPELLER_TAG
+    tag: v1.1.26 # FLYTEPROPELLER_TAG
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flytepropeller deployment
   resources:
@@ -246,7 +246,7 @@ flyteconsole:
   image:
     # -- Docker image for Flyteconsole deployment
     repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE
-    tag: v1.1.6 # FLYTECONSOLE_TAG
+    tag: v1.2.4 # FLYTECONSOLE_TAG
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flyteconsole deployment
   resources: