Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Mark taskResult completed if wait container terminated not gracefully. Fixes #13373 #13491

Merged
merged 1 commit into from
Aug 29, 2024
Merged

fix: Mark taskResult completed if wait container terminated not gracefully. Fixes #13373 #13491

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 23 additions & 3 deletions workflow/controller/operator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1496,14 +1496,34 @@ func (woc *wfOperationCtx) assessNodeStatus(ctx context.Context, pod *apiv1.Pod,
}
}

waitContainerCleanedUp := true
// We cannot fail the node if the wait container is still running because it may be busy saving outputs, and these
// would not get captured successfully.
for _, c := range pod.Status.ContainerStatuses {
if c.Name == common.WaitContainerName && c.State.Running != nil && new.Phase.Completed() {
woc.log.WithField("new.phase", new.Phase).Info("leaving phase un-changed: wait container is not yet terminated ")
new.Phase = old.Phase
if c.Name == common.WaitContainerName {
waitContainerCleanedUp = false
switch {
case c.State.Running != nil && new.Phase.Completed():
woc.log.WithField("new.phase", new.Phase).Info("leaving phase un-changed: wait container is not yet terminated ")
new.Phase = old.Phase
case c.State.Terminated != nil && c.State.Terminated.ExitCode != 0:
// Mark its taskResult as completed directly since wait container did not exit normally,
// and it will never have a chance to report taskResult correctly.
nodeID := woc.nodeID(pod)
woc.log.WithFields(log.Fields{"nodeID": nodeID, "exitCode": c.State.Terminated.ExitCode, "reason": c.State.Terminated.Reason}).
Warn("marking its taskResult as completed since wait container did not exit normally")
woc.wf.Status.MarkTaskResultComplete(nodeID)
}
}
}
if pod.Status.Phase == apiv1.PodFailed && pod.Status.Reason == "Evicted" && waitContainerCleanedUp {
// Mark its taskResult as completed directly since wait container has been cleaned up because of pod evicted,
// and it will never have a chance to report taskResult correctly.
nodeID := woc.nodeID(pod)
woc.log.WithFields(log.Fields{"nodeID": nodeID}).
Warn("marking its taskResult as completed since wait container has been cleaned up.")
woc.wf.Status.MarkTaskResultComplete(nodeID)
}

// if we are transitioning from Pending to a different state, clear out unchanged message
if old.Phase == wfv1.NodePending && new.Phase != wfv1.NodePending && old.Message == new.Message {
Expand Down
Loading