From 2e17adc17769fd54989417668a020be12a872277 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Tue, 2 May 2023 18:33:20 -0500 Subject: [PATCH 1/3] added environment variables to ExecutionConfig and propagated to taskExecutionMetadata Signed-off-by: Daniel Rammer --- .../flyteworkflow/v1alpha1/execution_config.go | 2 ++ pkg/controller/nodes/task/taskexec_context.go | 14 ++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/apis/flyteworkflow/v1alpha1/execution_config.go b/pkg/apis/flyteworkflow/v1alpha1/execution_config.go index 878df4768..0068b7a63 100644 --- a/pkg/apis/flyteworkflow/v1alpha1/execution_config.go +++ b/pkg/apis/flyteworkflow/v1alpha1/execution_config.go @@ -32,6 +32,8 @@ type ExecutionConfig struct { Interruptible *bool // Defines whether a workflow should skip all its cached results and re-compute its output, overwriting any already stored data. OverwriteCache bool + // Defines a map of environment varable name / value pairs that are applied to all tasks. + EnvironmentVariables map[string]string } type TaskPluginOverride struct { diff --git a/pkg/controller/nodes/task/taskexec_context.go b/pkg/controller/nodes/task/taskexec_context.go index 9e8296b3c..7abfd2599 100644 --- a/pkg/controller/nodes/task/taskexec_context.go +++ b/pkg/controller/nodes/task/taskexec_context.go @@ -59,10 +59,11 @@ func (te taskExecutionID) GetGeneratedNameWith(minLength, maxLength int) (string type taskExecutionMetadata struct { handler.NodeExecutionMetadata - taskExecID taskExecutionID - o pluginCore.TaskOverrides - maxAttempts uint32 - platformResources *v1.ResourceRequirements + taskExecID taskExecutionID + o pluginCore.TaskOverrides + maxAttempts uint32 + platformResources *v1.ResourceRequirements + environmentVariables map[string]string } func (t taskExecutionMetadata) GetTaskExecutionID() pluginCore.TaskExecutionID { @@ -81,6 +82,10 @@ func (t taskExecutionMetadata) GetPlatformResources() *v1.ResourceRequirements { return t.platformResources } +func (t taskExecutionMetadata) GetEnvironmentVariables() map[string]string { + return t.environmentVariables +} + type taskExecutionContext struct { handler.NodeExecutionContext tm taskExecutionMetadata @@ -289,6 +294,7 @@ func (t *Handler) newTaskExecutionContext(ctx context.Context, nCtx handler.Node o: nCtx.Node(), maxAttempts: maxAttempts, platformResources: convertTaskResourcesToRequirements(nCtx.ExecutionContext().GetExecutionConfig().TaskResources), + environmentVariables: nCtx.ExecutionContext().GetExecutionConfig().EnvironmentVariables, }, rm: resourcemanager.GetTaskResourceManager( t.resourceManager, resourceNamespacePrefix, id), From 57bfc0d68f72c41db9f9c8d64d9a61de3d3aa2d1 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 3 May 2023 13:42:07 -0500 Subject: [PATCH 2/3] fixed unit tests Signed-off-by: Daniel Rammer --- ...002_core.containerization.multi_images.my_workflow_2_wf.yaml | 1 + .../k8s/010_core.containerization.raw_container.wf_2_wf.yaml | 1 + ...re.containerization.use_secrets.my_secret_workflow_2_wf.yaml | 1 + .../019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml | 1 + .../k8s/022_core.control_flow.checkpoint.example_2_wf.yaml | 1 + .../k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml | 1 + .../k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml | 1 + .../k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml | 1 + .../035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml | 1 + .../037_core.control_flow.conditions.bool_input_wf_2_wf.yaml | 1 + ...039_core.control_flow.conditions.nested_conditions_2_wf.yaml | 1 + .../042_core.control_flow.conditions.consume_outputs_2_wf.yaml | 1 + .../snacks-core/k8s/048_core.control_flow.dynamics.wf_2_wf.yaml | 1 + .../053_core.control_flow.map_task.my_map_workflow_2_wf.yaml | 1 + .../k8s/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml | 1 + .../k8s/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml | 1 + .../069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml | 1 + ...7_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml | 1 + .../k8s/081_core.extend_flyte.custom_types.wf_2_wf.yaml | 1 + .../k8s/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml | 1 + .../k8s/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml | 1 + .../k8s/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml | 1 + ....flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml | 1 + ...e.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml | 1 + ....flyte_basics.documented_workflow.google_docstring_2_wf.yaml | 1 + .../105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml | 1 + ...te_basics.folders.download_and_normalize_csv_files_2_wf.yaml | 1 + .../k8s/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml | 1 + .../k8s/117_my.imperative.workflow.example_2_wf.yaml | 1 + .../snacks-core/k8s/120_core.flyte_basics.lp.my_wf_2_wf.yaml | 1 + .../snacks-core/k8s/125_core.flyte_basics.lp.go_greet_2_wf.yaml | 1 + .../k8s/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml | 1 + .../k8s/140_core.flyte_basics.shell_task.wf_2_wf.yaml | 1 + ...7_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml | 1 + ...scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml | 1 + ..._core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml | 1 + .../k8s/162_core.type_system.custom_objects.wf_2_wf.yaml | 1 + .../k8s/166_core.type_system.enums.enum_wf_2_wf.yaml | 1 + .../k8s/169_core.type_system.flyte_pickle.welcome_2_wf.yaml | 1 + .../snacks-core/k8s/173_core.type_system.schema.df_wf_2_wf.yaml | 1 + ..._system.structured_dataset.pandas_compatibility_wf_2_wf.yaml | 1 + ..._system.structured_dataset.schema_compatibility_wf_2_wf.yaml | 1 + .../k8s/185_core.type_system.typed_schema.wf_2_wf.yaml | 1 + pkg/controller/nodes/dynamic/dynamic_workflow_test.go | 2 +- 44 files changed, 44 insertions(+), 1 deletion(-) diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/002_core.containerization.multi_images.my_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/002_core.containerization.multi_images.my_workflow_2_wf.yaml index 98ea1f620..12b97cfd0 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/002_core.containerization.multi_images.my_workflow_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/002_core.containerization.multi_images.my_workflow_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/010_core.containerization.raw_container.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/010_core.containerization.raw_container.wf_2_wf.yaml index 9a00ffdff..d73c31601 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/010_core.containerization.raw_container.wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/010_core.containerization.raw_container.wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml index 0d33b2aad..8c7cfb7c8 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml index 7efbcf697..ad22eceb2 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/022_core.control_flow.checkpoint.example_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/022_core.control_flow.checkpoint.example_2_wf.yaml index 36abd7d14..0fc0290aa 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/022_core.control_flow.checkpoint.example_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/022_core.control_flow.checkpoint.example_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml index af54d19a9..74116d674 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml index 57b143920..b24ad357b 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml index 27dd0868b..37e8b5f33 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml index 45e222160..3856ff37f 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml index cfc31d71e..03ad38cd9 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/039_core.control_flow.conditions.nested_conditions_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/039_core.control_flow.conditions.nested_conditions_2_wf.yaml index bf85763b9..7c13f51de 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/039_core.control_flow.conditions.nested_conditions_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/039_core.control_flow.conditions.nested_conditions_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/042_core.control_flow.conditions.consume_outputs_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/042_core.control_flow.conditions.consume_outputs_2_wf.yaml index 36db07065..68f437726 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/042_core.control_flow.conditions.consume_outputs_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/042_core.control_flow.conditions.consume_outputs_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/048_core.control_flow.dynamics.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/048_core.control_flow.dynamics.wf_2_wf.yaml index f7477b6e5..c799c28ac 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/048_core.control_flow.dynamics.wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/048_core.control_flow.dynamics.wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml index 46c7b0ccd..4c5a3f70d 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml index e340f7bd2..62ca3c514 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml index 4c2b3b822..c50e0ed0b 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml index b3342ba32..3c652334a 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml index 407584f70..46106f72b 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/081_core.extend_flyte.custom_types.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/081_core.extend_flyte.custom_types.wf_2_wf.yaml index fdccbb84b..61f35fdf3 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/081_core.extend_flyte.custom_types.wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/081_core.extend_flyte.custom_types.wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml index c2b156942..1f214840b 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml index baedd2afd..265d00d75 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml index 15f19ddb1..64c2d33bd 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml index 56324a47c..bea325ce8 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml index 89dd7dc55..5bb361883 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml index 76407b7ae..d7cf5ac8b 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml index bb99dd7b6..b16532944 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml index 5d943ca0d..791dc12b8 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml index d2d30b315..9cea7205b 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/117_my.imperative.workflow.example_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/117_my.imperative.workflow.example_2_wf.yaml index 8c27274b0..7ef5b34aa 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/117_my.imperative.workflow.example_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/117_my.imperative.workflow.example_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/120_core.flyte_basics.lp.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/120_core.flyte_basics.lp.my_wf_2_wf.yaml index eb91ab4cb..4d461d672 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/120_core.flyte_basics.lp.my_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/120_core.flyte_basics.lp.my_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/125_core.flyte_basics.lp.go_greet_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/125_core.flyte_basics.lp.go_greet_2_wf.yaml index 2fa067626..0f6aa74dd 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/125_core.flyte_basics.lp.go_greet_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/125_core.flyte_basics.lp.go_greet_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml index 2ad1ef7ee..86638b623 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/140_core.flyte_basics.shell_task.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/140_core.flyte_basics.shell_task.wf_2_wf.yaml index 15b186204..3e8cbbce6 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/140_core.flyte_basics.shell_task.wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/140_core.flyte_basics.shell_task.wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml index 44d2f9562..765b2dd80 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml index 06e20a2a9..dfeda53b8 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml index 064338460..92c721784 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/162_core.type_system.custom_objects.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/162_core.type_system.custom_objects.wf_2_wf.yaml index 570453d1a..b11427079 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/162_core.type_system.custom_objects.wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/162_core.type_system.custom_objects.wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/166_core.type_system.enums.enum_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/166_core.type_system.enums.enum_wf_2_wf.yaml index 1eaf99845..5eb063803 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/166_core.type_system.enums.enum_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/166_core.type_system.enums.enum_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/169_core.type_system.flyte_pickle.welcome_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/169_core.type_system.flyte_pickle.welcome_2_wf.yaml index f4466f9d5..0bcc757aa 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/169_core.type_system.flyte_pickle.welcome_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/169_core.type_system.flyte_pickle.welcome_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/173_core.type_system.schema.df_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/173_core.type_system.schema.df_wf_2_wf.yaml index e6f0239d6..cf16bac0b 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/173_core.type_system.schema.df_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/173_core.type_system.schema.df_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml index 4577e374a..12b278919 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml index 4340ffef5..bde2f2892 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/compiler/test/testdata/snacks-core/k8s/185_core.type_system.typed_schema.wf_2_wf.yaml b/pkg/compiler/test/testdata/snacks-core/k8s/185_core.type_system.typed_schema.wf_2_wf.yaml index d04c5cc46..bec4de68e 100755 --- a/pkg/compiler/test/testdata/snacks-core/k8s/185_core.type_system.typed_schema.wf_2_wf.yaml +++ b/pkg/compiler/test/testdata/snacks-core/k8s/185_core.type_system.typed_schema.wf_2_wf.yaml @@ -1,5 +1,6 @@ apiVersion: flyte.lyft.com/v1alpha1 executionConfig: + EnvironmentVariables: null Interruptible: null MaxParallelism: 0 OverwriteCache: false diff --git a/pkg/controller/nodes/dynamic/dynamic_workflow_test.go b/pkg/controller/nodes/dynamic/dynamic_workflow_test.go index 63359352f..2e539a00f 100644 --- a/pkg/controller/nodes/dynamic/dynamic_workflow_test.go +++ b/pkg/controller/nodes/dynamic/dynamic_workflow_test.go @@ -498,7 +498,7 @@ func Test_dynamicNodeHandler_buildContextualDynamicWorkflow_withLaunchPlans(t *t composedPBStore.OnWriteRawMatch( mock.MatchedBy(func(ctx context.Context) bool { return true }), storage.DataReference("s3://my-s3-bucket/foo/bar/futures_compiled.pb"), - int64(1473), + int64(1501), storage.Options{}, mock.MatchedBy(func(rdr *bytes.Reader) bool { return true })).Return(errors.New("foo")) From d4998b147b649e2b288b52e7fad63c678f7a96f3 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Wed, 3 May 2023 14:56:44 -0500 Subject: [PATCH 3/3] updated flyteplugins Signed-off-by: Daniel Rammer --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d0d6abb60..886a2021b 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295 github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1 github.com/fatih/color v1.13.0 - github.com/flyteorg/flyteidl v1.3.14 - github.com/flyteorg/flyteplugins v1.0.52 + github.com/flyteorg/flyteidl v1.3.19 + github.com/flyteorg/flyteplugins v1.0.56 github.com/flyteorg/flytestdlib v1.0.15 github.com/ghodss/yaml v1.0.0 github.com/go-redis/redis v6.15.7+incompatible diff --git a/go.sum b/go.sum index 97d3b363c..5a45b73b8 100644 --- a/go.sum +++ b/go.sum @@ -260,10 +260,10 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flyteorg/flyteidl v1.3.14 h1:o5M0g/r6pXTPu5PEurbYxbQmuOu3hqqsaI2M6uvK0N8= -github.com/flyteorg/flyteidl v1.3.14/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM= -github.com/flyteorg/flyteplugins v1.0.52 h1:AWNrRYgm0bCzOws+bIfJDfPBZqBmTdABxW78r8q3kP4= -github.com/flyteorg/flyteplugins v1.0.52/go.mod h1:ztsonku5fKwyxcIg1k69PTiBVjRI6d3nK5DnC+iwx08= +github.com/flyteorg/flyteidl v1.3.19 h1:i79Dh7UoP8Z4LEJ2ox6jlfZVJtFZ+r4g84CJj1gh22Y= +github.com/flyteorg/flyteidl v1.3.19/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM= +github.com/flyteorg/flyteplugins v1.0.56 h1:kBTDgTpdSi7wcptk4cMwz5vfh1MU82VaUMMboe1InXw= +github.com/flyteorg/flyteplugins v1.0.56/go.mod h1:aFCKSn8TPzxSAILIiogHtUnHlUCN9+y6Vf+r9R4KZDU= github.com/flyteorg/flytestdlib v1.0.15 h1:kv9jDQmytbE84caY+pkZN8trJU2ouSAmESzpTEhfTt0= github.com/flyteorg/flytestdlib v1.0.15/go.mod h1:ghw/cjY0sEWIIbyCtcJnL/Gt7ZS7gf9SUi0CCPhbz3s= github.com/flyteorg/stow v0.3.6 h1:jt50ciM14qhKBaIrB+ppXXY+SXB59FNREFgTJqCyqIk=