Skip to content

Commit

Permalink
ImagePull failure is now reported on the UI (flyteorg#77)
Browse files Browse the repository at this point in the history
  • Loading branch information
Ketan Umare authored Apr 6, 2020
1 parent 8aba473 commit a6dc2b0
Show file tree
Hide file tree
Showing 3 changed files with 444 additions and 3 deletions.
8 changes: 5 additions & 3 deletions go/tasks/pluginmachinery/flytek8s/pod_helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,8 @@ func DemystifyPending(status v1.PodStatus) (pluginsCore.PhaseInfo, error) {
// There are a variety of reasons that can cause a pod to be in this waiting state.
// Waiting state may be legitimate when the container is being downloaded, started or init containers are running
reason := containerStatus.State.Waiting.Reason
finalReason := fmt.Sprintf("%s|%s", c.Reason, reason)
finalMessage := fmt.Sprintf("%s|%s", c.Message, containerStatus.State.Waiting.Message)
switch reason {
case "ErrImagePull", "ContainerCreating", "PodInitializing":
// But, there are only two "reasons" when a pod is successfully being created and hence it is in
Expand All @@ -149,12 +151,12 @@ func DemystifyPending(status v1.PodStatus) (pluginsCore.PhaseInfo, error) {
// ErrImagePull -> Transitionary phase to ImagePullBackOff
// ContainerCreating -> Image is being downloaded
// PodInitializing -> Init containers are running
return pluginsCore.PhaseInfoInitializing(c.LastTransitionTime.Time, pluginsCore.DefaultPhaseVersion, fmt.Sprintf("%s:%s", c.Reason, c.Message)), nil
return pluginsCore.PhaseInfoInitializing(c.LastTransitionTime.Time, pluginsCore.DefaultPhaseVersion, fmt.Sprintf("[%s]: %s", finalReason, finalMessage)), nil

case "CreateContainerError":
// This happens if for instance the command to the container is incorrect, ie doesn't run
t := c.LastTransitionTime.Time
return pluginsCore.PhaseInfoFailure(c.Reason, c.Message, &pluginsCore.TaskInfo{
return pluginsCore.PhaseInfoFailure(finalReason, finalMessage, &pluginsCore.TaskInfo{
OccurredAt: &t,
}), nil

Expand All @@ -168,7 +170,7 @@ func DemystifyPending(status v1.PodStatus) (pluginsCore.PhaseInfo, error) {
// So be default if the container is not waiting with the PodInitializing/ContainerCreating
// reasons, then we will assume a failure reason, and fail instantly
t := c.LastTransitionTime.Time
return pluginsCore.PhaseInfoSystemRetryableFailure(c.Reason, c.Message, &pluginsCore.TaskInfo{
return pluginsCore.PhaseInfoSystemRetryableFailure(finalReason, finalMessage, &pluginsCore.TaskInfo{
OccurredAt: &t,
}), nil
}
Expand Down
39 changes: 39 additions & 0 deletions go/tasks/pluginmachinery/flytek8s/pod_helper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ package flytek8s

import (
"context"
"encoding/json"
"io/ioutil"
"path/filepath"
"testing"

config1 "github.com/lyft/flytestdlib/config"
Expand Down Expand Up @@ -443,3 +446,39 @@ func TestConvertPodFailureToError(t *testing.T) {
assert.Equal(t, code, "OOMKilled")
})
}

func TestDemystifyPending_testcases(t *testing.T) {

tests := []struct {
name string
filename string
isErr bool
errCode string
message string
}{
{"ImagePullBackOff", "imagepull-failurepod.json", false, "ContainersNotReady|ImagePullBackOff", "containers with unready status: [fdf98e4ed2b524dc3bf7-get-flyte-id-task-0]|Back-off pulling image \"image\""},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
testFile := filepath.Join("testdata", tt.filename)
data, err := ioutil.ReadFile(testFile)
assert.NoError(t, err, "failed to read file %s", testFile)
pod := &v1.Pod{}
if assert.NoError(t, json.Unmarshal(data, pod), "failed to unmarshal json in %s. Expected of type v1.Pod", testFile) {
p, err := DemystifyPending(pod.Status)
if tt.isErr {
assert.Error(t, err, "Error expected from method")
} else {
assert.NoError(t, err, "Error not expected")
assert.NotNil(t, p)
assert.Equal(t, p.Phase(), pluginsCore.PhaseRetryableFailure)
if assert.NotNil(t, p.Err()) {
assert.Equal(t, p.Err().Code, tt.errCode)
assert.Equal(t, p.Err().Message, tt.message)
}
}
}
})
}
}
Loading

0 comments on commit a6dc2b0

Please sign in to comment.