From 2528c2a7923d73849c6ab9cb527348eece67b016 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Mon, 8 Aug 2022 10:06:56 -0500 Subject: [PATCH 1/5] using 'failing' state to handle subworkflow aborts Signed-off-by: Daniel Rammer --- pkg/controller/nodes/subworkflow/subworkflow.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/controller/nodes/subworkflow/subworkflow.go b/pkg/controller/nodes/subworkflow/subworkflow.go index 3e688e60d..638086767 100644 --- a/pkg/controller/nodes/subworkflow/subworkflow.go +++ b/pkg/controller/nodes/subworkflow/subworkflow.go @@ -83,11 +83,12 @@ func (s *subworkflowHandler) handleSubWorkflow(ctx context.Context, nCtx handler } err = nCtx.NodeStateWriter().PutWorkflowNodeState(workflowNodeState) - if subworkflow.GetOnFailureNode() != nil { + /*if subworkflow.GetOnFailureNode() != nil { return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoFailingErr(state.Err, nil)), err } - return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoFailureErr(state.Err, nil)), err + return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoFailureErr(state.Err, nil)), err*/ + return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoFailingErr(state.Err, nil)), err } if state.IsComplete() { @@ -190,6 +191,11 @@ func (s *subworkflowHandler) HandleFailingSubWorkflow(ctx context.Context, nCtx return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoFailure(core.ExecutionError_SYSTEM, errors.SubWorkflowExecutionFailed, err.Error(), nil)), nil } + if err := s.HandleAbort(ctx, nCtx, "subworkflow node failed"); err != nil { + logger.Warnf(ctx, "failed to abort failing subworkflow with err '%v'", err) + return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoUndefined), err + } + status := nCtx.NodeStatus() status.GetWorkflowNodeStatus() if subWorkflow.GetOnFailureNode() == nil { From 4eb0853bd5a92a0ee4cb8b7c413827d0ff8d4ebc Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Mon, 8 Aug 2022 19:28:49 -0500 Subject: [PATCH 2/5] propogating node failure is subworkflow to subworkflow failure message in ui Signed-off-by: Daniel Rammer --- pkg/controller/nodes/node_state_manager.go | 1 + pkg/controller/nodes/subworkflow/subworkflow.go | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/controller/nodes/node_state_manager.go b/pkg/controller/nodes/node_state_manager.go index 871fecd36..bf4d270a2 100644 --- a/pkg/controller/nodes/node_state_manager.go +++ b/pkg/controller/nodes/node_state_manager.go @@ -79,6 +79,7 @@ func (n nodeStateManager) GetWorkflowNodeState() handler.WorkflowNodeState { ws := handler.WorkflowNodeState{} if wn != nil { ws.Phase = wn.GetWorkflowNodePhase() + ws.Error = wn.GetExecutionError() } return ws } diff --git a/pkg/controller/nodes/subworkflow/subworkflow.go b/pkg/controller/nodes/subworkflow/subworkflow.go index 638086767..c2787bb5e 100644 --- a/pkg/controller/nodes/subworkflow/subworkflow.go +++ b/pkg/controller/nodes/subworkflow/subworkflow.go @@ -83,12 +83,12 @@ func (s *subworkflowHandler) handleSubWorkflow(ctx context.Context, nCtx handler } err = nCtx.NodeStateWriter().PutWorkflowNodeState(workflowNodeState) - /*if subworkflow.GetOnFailureNode() != nil { - return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoFailingErr(state.Err, nil)), err + if err != nil { + logger.Warnf(ctx, "failed to store failing subworkflow state with err '%s'", err) + return handler.UnknownTransition, err } - return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoFailureErr(state.Err, nil)), err*/ - return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoFailingErr(state.Err, nil)), err + return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoRunning(nil)), nil } if state.IsComplete() { @@ -191,7 +191,7 @@ func (s *subworkflowHandler) HandleFailingSubWorkflow(ctx context.Context, nCtx return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoFailure(core.ExecutionError_SYSTEM, errors.SubWorkflowExecutionFailed, err.Error(), nil)), nil } - if err := s.HandleAbort(ctx, nCtx, "subworkflow node failed"); err != nil { + if err := s.HandleAbort(ctx, nCtx, "subworkflow failed"); err != nil { logger.Warnf(ctx, "failed to abort failing subworkflow with err '%v'", err) return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoUndefined), err } From ecf12cab4198e90acf3c09a172e47a8dbb43757e Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Mon, 8 Aug 2022 19:36:54 -0500 Subject: [PATCH 3/5] working with other failure scenarios Signed-off-by: Daniel Rammer --- pkg/controller/nodes/subworkflow/subworkflow.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/controller/nodes/subworkflow/subworkflow.go b/pkg/controller/nodes/subworkflow/subworkflow.go index c2787bb5e..ddde583ab 100644 --- a/pkg/controller/nodes/subworkflow/subworkflow.go +++ b/pkg/controller/nodes/subworkflow/subworkflow.go @@ -196,14 +196,19 @@ func (s *subworkflowHandler) HandleFailingSubWorkflow(ctx context.Context, nCtx return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoUndefined), err } - status := nCtx.NodeStatus() - status.GetWorkflowNodeStatus() if subWorkflow.GetOnFailureNode() == nil { logger.Infof(ctx, "Subworkflow has no failure nodes, failing immediately.") - return handler.DoTransition(handler.TransitionTypeEphemeral, - handler.PhaseInfoFailureErr(nCtx.NodeStateReader().GetWorkflowNodeState().Error, nil)), err + state := nCtx.NodeStateReader().GetWorkflowNodeState() + if state.Error != nil { + return handler.DoTransition(handler.TransitionTypeEphemeral, + handler.PhaseInfoFailureErr(state.Error, nil)), nil + } else { + return handler.DoTransition(handler.TransitionTypeEphemeral, + handler.PhaseInfoFailure(core.ExecutionError_UNKNOWN, "SubworkflowNodeFailing", "", nil)), nil + } } + status := nCtx.NodeStatus() nodeLookup := executors.NewNodeLookup(subWorkflow, status) return s.HandleFailureNodeOfSubWorkflow(ctx, nCtx, subWorkflow, nodeLookup) } From 102d8759487109988b1dfa9bc04d93db43c9d281 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Mon, 8 Aug 2022 19:43:17 -0500 Subject: [PATCH 4/5] fixed lint issue Signed-off-by: Daniel Rammer --- pkg/controller/nodes/subworkflow/subworkflow.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controller/nodes/subworkflow/subworkflow.go b/pkg/controller/nodes/subworkflow/subworkflow.go index ddde583ab..0b25106f5 100644 --- a/pkg/controller/nodes/subworkflow/subworkflow.go +++ b/pkg/controller/nodes/subworkflow/subworkflow.go @@ -202,10 +202,10 @@ func (s *subworkflowHandler) HandleFailingSubWorkflow(ctx context.Context, nCtx if state.Error != nil { return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoFailureErr(state.Error, nil)), nil - } else { - return handler.DoTransition(handler.TransitionTypeEphemeral, - handler.PhaseInfoFailure(core.ExecutionError_UNKNOWN, "SubworkflowNodeFailing", "", nil)), nil } + + return handler.DoTransition(handler.TransitionTypeEphemeral, + handler.PhaseInfoFailure(core.ExecutionError_UNKNOWN, "SubworkflowNodeFailing", "", nil)), nil } status := nCtx.NodeStatus() From 9f3aae7a2634a70e2a29dc65034d3b563254d2a8 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Tue, 9 Aug 2022 06:21:45 -0500 Subject: [PATCH 5/5] updated error message to match Signed-off-by: Daniel Rammer --- pkg/controller/nodes/subworkflow/subworkflow.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/nodes/subworkflow/subworkflow.go b/pkg/controller/nodes/subworkflow/subworkflow.go index 0b25106f5..24d74473f 100644 --- a/pkg/controller/nodes/subworkflow/subworkflow.go +++ b/pkg/controller/nodes/subworkflow/subworkflow.go @@ -84,7 +84,7 @@ func (s *subworkflowHandler) handleSubWorkflow(ctx context.Context, nCtx handler err = nCtx.NodeStateWriter().PutWorkflowNodeState(workflowNodeState) if err != nil { - logger.Warnf(ctx, "failed to store failing subworkflow state with err '%s'", err) + logger.Warnf(ctx, "failed to store failing subworkflow state with err: [%v]", err) return handler.UnknownTransition, err } @@ -192,7 +192,7 @@ func (s *subworkflowHandler) HandleFailingSubWorkflow(ctx context.Context, nCtx } if err := s.HandleAbort(ctx, nCtx, "subworkflow failed"); err != nil { - logger.Warnf(ctx, "failed to abort failing subworkflow with err '%v'", err) + logger.Warnf(ctx, "failed to abort failing subworkflow with err: [%v]", err) return handler.DoTransition(handler.TransitionTypeEphemeral, handler.PhaseInfoUndefined), err }