Added support for EnvironmentVariables on ExecutionConfig

go.mod

@@ -6,8 +6,8 @@ require (
 	github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295
 	github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1
 	github.com/fatih/color v1.13.0
-	github.com/flyteorg/flyteidl v1.3.14
-	github.com/flyteorg/flyteplugins v1.0.52
+	github.com/flyteorg/flyteidl v1.3.19
+	github.com/flyteorg/flyteplugins v1.0.56
 	github.com/flyteorg/flytestdlib v1.0.15
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-redis/redis v6.15.7+incompatible

go.sum

@@ -260,10 +260,10 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/flyteorg/flyteidl v1.3.14 h1:o5M0g/r6pXTPu5PEurbYxbQmuOu3hqqsaI2M6uvK0N8=
-github.com/flyteorg/flyteidl v1.3.14/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM=
-github.com/flyteorg/flyteplugins v1.0.52 h1:AWNrRYgm0bCzOws+bIfJDfPBZqBmTdABxW78r8q3kP4=
-github.com/flyteorg/flyteplugins v1.0.52/go.mod h1:ztsonku5fKwyxcIg1k69PTiBVjRI6d3nK5DnC+iwx08=
+github.com/flyteorg/flyteidl v1.3.19 h1:i79Dh7UoP8Z4LEJ2ox6jlfZVJtFZ+r4g84CJj1gh22Y=
+github.com/flyteorg/flyteidl v1.3.19/go.mod h1:Pkt2skI1LiHs/2ZoekBnyPhuGOFMiuul6HHcKGZBsbM=
+github.com/flyteorg/flyteplugins v1.0.56 h1:kBTDgTpdSi7wcptk4cMwz5vfh1MU82VaUMMboe1InXw=
+github.com/flyteorg/flyteplugins v1.0.56/go.mod h1:aFCKSn8TPzxSAILIiogHtUnHlUCN9+y6Vf+r9R4KZDU=
 github.com/flyteorg/flytestdlib v1.0.15 h1:kv9jDQmytbE84caY+pkZN8trJU2ouSAmESzpTEhfTt0=
 github.com/flyteorg/flytestdlib v1.0.15/go.mod h1:ghw/cjY0sEWIIbyCtcJnL/Gt7ZS7gf9SUi0CCPhbz3s=
 github.com/flyteorg/stow v0.3.6 h1:jt50ciM14qhKBaIrB+ppXXY+SXB59FNREFgTJqCyqIk=

pkg/apis/flyteworkflow/v1alpha1/execution_config.go

@@ -32,6 +32,8 @@ type ExecutionConfig struct {
 	Interruptible *bool
 	// Defines whether a workflow should skip all its cached results and re-compute its output, overwriting any already stored data.
 	OverwriteCache bool
+	// Defines a map of environment varable name / value pairs that are applied to all tasks.
+	EnvironmentVariables map[string]string
 }
 
 type TaskPluginOverride struct {

pkg/compiler/test/testdata/snacks-core/k8s/002_core.containerization.multi_images.my_workflow_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/010_core.containerization.raw_container.wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/015_core.containerization.use_secrets.my_secret_workflow_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/019_core.control_flow.chain_tasks.chain_tasks_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/022_core.control_flow.checkpoint.example_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/026_core.control_flow.conditions.multiplier_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/028_core.control_flow.conditions.multiplier_2_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/030_core.control_flow.conditions.multiplier_3_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/035_core.control_flow.conditions.basic_boolean_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/037_core.control_flow.conditions.bool_input_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/039_core.control_flow.conditions.nested_conditions_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/042_core.control_flow.conditions.consume_outputs_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/048_core.control_flow.dynamics.wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/053_core.control_flow.map_task.my_map_workflow_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/059_core.control_flow.merge_sort.merge_sort_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/062_core.control_flow.subworkflows.my_subwf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/069_core.control_flow.subworkflows.ext_workflow_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/077_core.extend_flyte.custom_task_plugin.my_workflow_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/081_core.extend_flyte.custom_types.wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/085_core.flyte_basics.basic_workflow.my_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/089_core.flyte_basics.decorating_tasks.wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/095_core.flyte_basics.decorating_workflows.wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/098_core.flyte_basics.documented_workflow.sphinx_docstring_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/100_core.flyte_basics.documented_workflow.numpy_docstring_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/102_core.flyte_basics.documented_workflow.google_docstring_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/105_core.flyte_basics.files.normalize_csv_file_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/109_core.flyte_basics.folders.download_and_normalize_csv_files_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/112_core.flyte_basics.hello_world.my_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/117_my.imperative.workflow.example_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/120_core.flyte_basics.lp.my_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/125_core.flyte_basics.lp.go_greet_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/129_core.flyte_basics.named_outputs.my_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/140_core.flyte_basics.shell_task.wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/147_core.flyte_basics.task_cache.cached_dataframe_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/151_core.scheduled_workflows.lp_schedules.date_formatter_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/155_core.scheduled_workflows.lp_schedules.positive_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/162_core.type_system.custom_objects.wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/166_core.type_system.enums.enum_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/169_core.type_system.flyte_pickle.welcome_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/173_core.type_system.schema.df_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/179_core.type_system.structured_dataset.pandas_compatibility_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/181_core.type_system.structured_dataset.schema_compatibility_wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/compiler/test/testdata/snacks-core/k8s/185_core.type_system.typed_schema.wf_2_wf.yaml

@@ -1,5 +1,6 @@
 apiVersion: flyte.lyft.com/v1alpha1
 executionConfig:
+  EnvironmentVariables: null
   Interruptible: null
   MaxParallelism: 0
   OverwriteCache: false

pkg/controller/nodes/dynamic/dynamic_workflow_test.go

@@ -498,7 +498,7 @@ func Test_dynamicNodeHandler_buildContextualDynamicWorkflow_withLaunchPlans(t *t
 		composedPBStore.OnWriteRawMatch(
 			mock.MatchedBy(func(ctx context.Context) bool { return true }),
 			storage.DataReference("s3://my-s3-bucket/foo/bar/futures_compiled.pb"),
-			int64(1473),
+			int64(1501),
 			storage.Options{},
 			mock.MatchedBy(func(rdr *bytes.Reader) bool { return true })).Return(errors.New("foo"))
 

pkg/controller/nodes/task/taskexec_context.go

@@ -59,10 +59,11 @@ func (te taskExecutionID) GetGeneratedNameWith(minLength, maxLength int) (string
 
 type taskExecutionMetadata struct {
 	handler.NodeExecutionMetadata
-	taskExecID        taskExecutionID
-	o                 pluginCore.TaskOverrides
-	maxAttempts       uint32
-	platformResources *v1.ResourceRequirements
+	taskExecID           taskExecutionID
+	o                    pluginCore.TaskOverrides
+	maxAttempts          uint32
+	platformResources    *v1.ResourceRequirements
+	environmentVariables map[string]string
 }
 
 func (t taskExecutionMetadata) GetTaskExecutionID() pluginCore.TaskExecutionID {
@@ -81,6 +82,10 @@ func (t taskExecutionMetadata) GetPlatformResources() *v1.ResourceRequirements {
 	return t.platformResources
 }
 
+func (t taskExecutionMetadata) GetEnvironmentVariables() map[string]string {
+	return t.environmentVariables
+}
+
 type taskExecutionContext struct {
 	handler.NodeExecutionContext
 	tm  taskExecutionMetadata
@@ -289,6 +294,7 @@ func (t *Handler) newTaskExecutionContext(ctx context.Context, nCtx handler.Node
 			o:                     nCtx.Node(),
 			maxAttempts:           maxAttempts,
 			platformResources:     convertTaskResourcesToRequirements(nCtx.ExecutionContext().GetExecutionConfig().TaskResources),
+			environmentVariables:  nCtx.ExecutionContext().GetExecutionConfig().EnvironmentVariables,
 		},
 		rm: resourcemanager.GetTaskResourceManager(
 			t.resourceManager, resourceNamespacePrefix, id),