From dc5bbcad720345fdee21d178c68701f05d1c783d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bu=C4=9Fra=20Gedik?= Date: Tue, 20 Aug 2024 16:19:17 -0700 Subject: [PATCH] Add FLYTE_INTERNAL_POD_NAME environment variable that holds the pod name (#5616) * Add environment variable for pod name Signed-off-by: Bugra Gedik * [flyteadmin] Refactor panic recovery into middleware (#5546) * Refactor panic handling to middleware Signed-off-by: Jason Parraga * Remove registration of old panicCounter Signed-off-by: Jason Parraga * Add test coverage Signed-off-by: Jason Parraga --------- Signed-off-by: Jason Parraga Signed-off-by: Bugra Gedik * Snowflake agent Doc (#5620) * TEST build Signed-off-by: Future-Outlier * remove emphasize-lines Signed-off-by: Future-Outlier * test build Signed-off-by: Future-Outlier * revert Signed-off-by: Future-Outlier --------- Signed-off-by: Future-Outlier Signed-off-by: Bugra Gedik * [flytepropeller][compiler] Error Handling when Type is not found (#5612) * FlytePropeller Compiler Avoid Crash when Type not found Signed-off-by: Future-Outlier * Update pingsu's error message advices Signed-off-by: Future-Outlier Co-authored-by: pingsutw * fix lint Signed-off-by: Future-Outlier * Trigger CI Signed-off-by: Future-Outlier * Trigger CI Signed-off-by: Future-Outlier --------- Signed-off-by: Future-Outlier Co-authored-by: pingsutw Signed-off-by: Bugra Gedik * Fix nil pointer when task plugin load returns error (#5622) Signed-off-by: Bugra Gedik * Log stack trace when refresh cache sync recovers from panic (#5623) Signed-off-by: Bugra Gedik * use private-key (#5626) Signed-off-by: Bugra Gedik * Explain how Agent Secret Works (#5625) * first version Signed-off-by: Future-Outlier * update Signed-off-by: Future-Outlier --------- Signed-off-by: Future-Outlier Signed-off-by: Bugra Gedik * Fix typo in execution manager (#5619) Signed-off-by: ddl-rliu <140021987+ddl-rliu@users.noreply.github.com> Signed-off-by: Bugra Gedik * Amend Admin to use grpc message size (#5628) * add send arg Signed-off-by: Yee Hing Tong * Add acction to remove cache in gh runner Signed-off-by: Eduardo Apolinario * Use correct checked out path Signed-off-by: Eduardo Apolinario * Path in strings Signed-off-by: Eduardo Apolinario * Checkout repo in root Signed-off-by: Eduardo Apolinario * Use the correct path to new action Signed-off-by: Eduardo Apolinario * Do not use gh var in path to clear-action-cache Signed-off-by: Eduardo Apolinario * Remove wrong invocation of clear-action-cache Signed-off-by: Eduardo Apolinario * GITHUB_WORKSPACE is implicit in the checkout action Signed-off-by: Eduardo Apolinario * Refer to local `flyte` directory Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Yee Hing Tong Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario Signed-off-by: Bugra Gedik * document the process of setting ttl for a ray cluster (#5636) Signed-off-by: Kevin Su Signed-off-by: Bugra Gedik * Add CustomHeaderMatcher to pass additional headers (#5563) Signed-off-by: Andrew Dye Signed-off-by: Bugra Gedik * Turn flyteidl and flytectl releases into manual gh workflows (#5635) * Make flyteidl releases go through a manual gh workflow Signed-off-by: Eduardo Apolinario * Make flytectl releases go through a manual gh workflow Signed-off-by: Eduardo Apolinario * Rewrite the documentation for `version` and clarify wording in RELEASE.md Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario Signed-off-by: Bugra Gedik * docs: fix typo (#5643) * fix CHANGELOG-v0.2.0.md Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> * fix CHANGELOG-v1.0.2-b1.md Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> * fix CHANGELOG-v1.1.0.md Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> * fix CHANGELOG-v1.3.0.md Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> --------- Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> Signed-off-by: Bugra Gedik * Use enable_deck=True in docs (#5645) Signed-off-by: Bugra Gedik * Fix flyteidl release checkout all tags (#5646) * Fetch all tags in flyteidl-release.yml Signed-off-by: Eduardo Apolinario * Fix sed expression for npm job Signed-off-by: Eduardo Apolinario --------- Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario Signed-off-by: Bugra Gedik * Install pyarrow in sandbox functional tests (#5647) Signed-off-by: Eduardo Apolinario Co-authored-by: Eduardo Apolinario Signed-off-by: Bugra Gedik * docs: add documentation for configuring notifications in GCP (#5545) * update Signed-off-by: Desi Hsu * dco Signed-off-by: Desi Hsu * dco Signed-off-by: Desi Hsu * typo Signed-off-by: Desi Hsu --------- Signed-off-by: Desi Hsu Signed-off-by: Bugra Gedik * Correct "sucessfile" to "successfile" (#5652) Signed-off-by: Bugra Gedik * Fix ordering for custom template values in cluster resource controller (#5648) Signed-off-by: Katrina Rogan Signed-off-by: Bugra Gedik * Don't error when attempting to trigger schedules for inactive projects (#5649) * Don't error when attempting to trigger schedules for inactive projects Signed-off-by: Katrina Rogan * regen Signed-off-by: Katrina Rogan --------- Signed-off-by: Katrina Rogan Signed-off-by: Bugra Gedik * fix tests Signed-off-by: Bugra Gedik * change to shorter names Signed-off-by: Bugra Gedik * change to shorter names Signed-off-by: Bugra Gedik * change to shorter names Signed-off-by: Bugra Gedik * change to shorter names Signed-off-by: Bugra Gedik * change to shorter names Signed-off-by: Bugra Gedik * Fix comment symbol Signed-off-by: Eduardo Apolinario * fix one more test Signed-off-by: Bugra Gedik --------- Signed-off-by: Bugra Gedik Signed-off-by: Jason Parraga Signed-off-by: Future-Outlier Signed-off-by: ddl-rliu <140021987+ddl-rliu@users.noreply.github.com> Signed-off-by: Yee Hing Tong Signed-off-by: Eduardo Apolinario Signed-off-by: Kevin Su Signed-off-by: Andrew Dye Signed-off-by: Christina <156356273+cratiu222@users.noreply.github.com> Signed-off-by: Desi Hsu Signed-off-by: Katrina Rogan Co-authored-by: Jason Parraga Co-authored-by: Future-Outlier Co-authored-by: pingsutw Co-authored-by: ddl-rliu <140021987+ddl-rliu@users.noreply.github.com> Co-authored-by: Yee Hing Tong Co-authored-by: Eduardo Apolinario Co-authored-by: Andrew Dye Co-authored-by: Eduardo Apolinario <653394+eapolinario@users.noreply.github.com> Co-authored-by: Christina <156356273+cratiu222@users.noreply.github.com> Co-authored-by: Thomas J. Fan Co-authored-by: desihsu <43691987+desihsu@users.noreply.github.com> Co-authored-by: ShengYu Co-authored-by: Katrina Rogan Signed-off-by: pmahindrakar-oss --- .../pluginmachinery/flytek8s/container_helper_test.go | 2 +- .../pluginmachinery/flytek8s/k8s_resource_adds.go | 9 +++++++++ .../pluginmachinery/flytek8s/k8s_resource_adds_test.go | 10 +++++----- flyteplugins/go/tasks/plugins/k8s/spark/spark_test.go | 4 ++-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/container_helper_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/container_helper_test.go index 4e609c72b2e..3b7aa88aeb0 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/container_helper_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/container_helper_test.go @@ -525,7 +525,7 @@ func TestAddFlyteCustomizationsToContainer(t *testing.T) { assert.EqualValues(t, container.Command, []string{"s3://input/path"}) assert.Len(t, container.Resources.Limits, 3) assert.Len(t, container.Resources.Requests, 3) - assert.Len(t, container.Env, 12) + assert.Len(t, container.Env, 13) } func TestAddFlyteCustomizationsToContainer_Resources(t *testing.T) { diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds.go index 34e13adfa83..b77615120ab 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds.go @@ -60,6 +60,15 @@ func GetExecutionEnvVars(id pluginsCore.TaskExecutionID, consoleURL string) []v1 Name: "FLYTE_INTERNAL_EXECUTION_DOMAIN", Value: nodeExecutionID.Domain, }, + { + // FLYTE_INTERNAL_POD_NAME + Name: "_F_PN", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, { Name: "FLYTE_ATTEMPT_NUMBER", Value: attemptNumber, diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go index 4015a8d9b8d..fd4828fbbd2 100644 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/k8s_resource_adds_test.go @@ -27,13 +27,13 @@ func TestGetExecutionEnvVars(t *testing.T) { }{ { "no-console-url", - 12, + 13, "", nil, }, { "with-console-url", - 13, + 14, "scheme://host/path", &v12.EnvVar{ Name: "FLYTE_EXECUTION_URL", @@ -42,7 +42,7 @@ func TestGetExecutionEnvVars(t *testing.T) { }, { "with-console-url-ending-in-single-slash", - 13, + 14, "scheme://host/path/", &v12.EnvVar{ Name: "FLYTE_EXECUTION_URL", @@ -51,7 +51,7 @@ func TestGetExecutionEnvVars(t *testing.T) { }, { "with-console-url-ending-in-multiple-slashes", - 13, + 14, "scheme://host/path////", &v12.EnvVar{ Name: "FLYTE_EXECUTION_URL", @@ -63,7 +63,7 @@ func TestGetExecutionEnvVars(t *testing.T) { envVars := GetExecutionEnvVars(mock, tt.consoleURL) assert.Len(t, envVars, tt.expectedEnvVars) if tt.expectedEnvVar != nil { - assert.True(t, proto.Equal(&envVars[4], tt.expectedEnvVar)) + assert.True(t, proto.Equal(&envVars[5], tt.expectedEnvVar)) } } } diff --git a/flyteplugins/go/tasks/plugins/k8s/spark/spark_test.go b/flyteplugins/go/tasks/plugins/k8s/spark/spark_test.go index 7ea6c42be2a..d657d4c273c 100644 --- a/flyteplugins/go/tasks/plugins/k8s/spark/spark_test.go +++ b/flyteplugins/go/tasks/plugins/k8s/spark/spark_test.go @@ -853,7 +853,7 @@ func TestBuildResourcePodTemplate(t *testing.T) { assert.Equal(t, defaultConfig.DefaultEnvVars["foo"], findEnvVarByName(sparkApp.Spec.Driver.Env, "foo").Value) assert.Equal(t, defaultConfig.DefaultEnvVars["fooEnv"], findEnvVarByName(sparkApp.Spec.Driver.Env, "fooEnv").Value) assert.Equal(t, findEnvVarByName(dummyEnvVarsWithSecretRef, "SECRET"), findEnvVarByName(sparkApp.Spec.Driver.Env, "SECRET")) - assert.Equal(t, 9, len(sparkApp.Spec.Driver.Env)) + assert.Equal(t, 10, len(sparkApp.Spec.Driver.Env)) assert.Equal(t, testImage, *sparkApp.Spec.Driver.Image) assert.Equal(t, flytek8s.GetServiceAccountNameFromTaskExecutionMetadata(taskCtx.TaskExecutionMetadata()), *sparkApp.Spec.Driver.ServiceAccount) assert.Equal(t, defaultConfig.DefaultPodSecurityContext, sparkApp.Spec.Driver.SecurityContenxt) @@ -890,7 +890,7 @@ func TestBuildResourcePodTemplate(t *testing.T) { assert.Equal(t, defaultConfig.DefaultEnvVars["foo"], findEnvVarByName(sparkApp.Spec.Executor.Env, "foo").Value) assert.Equal(t, defaultConfig.DefaultEnvVars["fooEnv"], findEnvVarByName(sparkApp.Spec.Executor.Env, "fooEnv").Value) assert.Equal(t, findEnvVarByName(dummyEnvVarsWithSecretRef, "SECRET"), findEnvVarByName(sparkApp.Spec.Executor.Env, "SECRET")) - assert.Equal(t, 9, len(sparkApp.Spec.Executor.Env)) + assert.Equal(t, 10, len(sparkApp.Spec.Executor.Env)) assert.Equal(t, testImage, *sparkApp.Spec.Executor.Image) assert.Equal(t, defaultConfig.DefaultPodSecurityContext, sparkApp.Spec.Executor.SecurityContenxt) assert.Equal(t, defaultConfig.DefaultPodDNSConfig, sparkApp.Spec.Executor.DNSConfig)