From c499a48ecd725278c1b99f0d8d7bed32eb06c1dc Mon Sep 17 00:00:00 2001 From: bstadlbauer <11799671+bstadlbauer@users.noreply.github.com> Date: Fri, 26 May 2023 23:25:09 +0200 Subject: [PATCH] Improve demystifying GKE spot node preemtion #patch (#354) * Improve demystifying GKE spot node preemtion Signed-off-by: Bernhard Stadlbauer <11799671+bstadlbauer@users.noreply.github.com> * Add test Signed-off-by: Bernhard Stadlbauer <11799671+bstadlbauer@users.noreply.github.com> --------- Signed-off-by: Bernhard Stadlbauer <11799671+bstadlbauer@users.noreply.github.com> --- go/tasks/pluginmachinery/flytek8s/pod_helper.go | 4 +++- go/tasks/pluginmachinery/flytek8s/pod_helper_test.go | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/go/tasks/pluginmachinery/flytek8s/pod_helper.go index df75b92a9..9c4fe6efa 100755 --- a/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -622,7 +622,9 @@ func DemystifyFailure(status v1.PodStatus, info pluginsCore.TaskInfo) (pluginsCo // startTime: "2022-01-30T14:24:07Z" // } // } - if code == "Shutdown" { + // + // In some versions of GKE the reason can also be "Terminated" + if code == "Shutdown" || code == "Terminated" { return pluginsCore.PhaseInfoSystemRetryableFailure(Interrupted, message, &info), nil } diff --git a/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go b/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go index f469a2227..2f011f43a 100755 --- a/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go +++ b/go/tasks/pluginmachinery/flytek8s/pod_helper_test.go @@ -900,6 +900,17 @@ func TestDemystifyFailure(t *testing.T) { assert.Equal(t, "Interrupted", phaseInfo.Err().Code) assert.Equal(t, core.ExecutionError_SYSTEM, phaseInfo.Err().Kind) }) + + t.Run("GKE kubelet graceful node shutdown", func(t *testing.T) { + phaseInfo, err := DemystifyFailure(v1.PodStatus{ + Message: "Foobar", + Reason: "Terminated", + }, pluginsCore.TaskInfo{}) + assert.Nil(t, err) + assert.Equal(t, pluginsCore.PhaseRetryableFailure, phaseInfo.Phase()) + assert.Equal(t, "Interrupted", phaseInfo.Err().Code) + assert.Equal(t, core.ExecutionError_SYSTEM, phaseInfo.Err().Kind) + }) } func TestDemystifyPending_testcases(t *testing.T) {