Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(api): debug constraint fk_worker_workflow_node_run_job #6678

Merged
merged 1 commit into from
Nov 9, 2023
Merged

fix(api): debug constraint fk_worker_workflow_node_run_job #6678

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 18 additions & 18 deletions engine/api/workflow/execute_node_run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,28 +307,28 @@ func executeNodeRun(ctx context.Context, db gorpmapper.SqlExecutorWithTx, store
report.Merge(ctx, r1)
}

// Delete the line in workflow_node_run_job
if err := DeleteNodeJobRuns(db, workflowNodeRun.ID); err != nil {
// Checking the error:
// pq: update or delete on table "workflow_node_run_job" violates foreign key constraint "fk_worker_workflow_node_run_job" on table "worker")
type WorkerInfo struct {
WorkerID string `db:"worker_id"`
WorkerName string `db:"worker_name"`
WorkflowNodeRunJobID string `db:"workflow_node_run_job_id"`
}

var workers []WorkerInfo
if _, errSelect := db.Select(&workers, `
// Temporary debug:
// Listing worker for node run that block the job deletion
type WorkerInfo struct {
WorkerID string `db:"worker_id"`
WorkerName string `db:"worker_name"`
WorkflowNodeRunJobID string `db:"workflow_node_run_job_id"`
}
var blockingWorkers []WorkerInfo
if _, err := db.Select(&blockingWorkers, `
SELECT worker.id as worker_id, worker.name as worker_name, workflow_node_run_job.id as workflow_node_run_job_id FROM worker
JOIN workflow_node_run_job ON workflow_node_run_job.worker_id = worker.id
WHERE workflow_node_run_job.workflow_node_run_id = $1
`, workflowNodeRun.ID); errSelect != nil {
log.ErrorWithStackTrace(ctx, sdk.WrapError(errSelect, "unable to get worker list for node run with id %d", workflowNodeRun.ID))
} else {
buf, _ := json.Marshal(workers)
log.Error(ctx, "list of workers for node run %d that block jobs deletion (len:%d): %q", workflowNodeRun.ID, len(workers), string(buf))
}
`, workflowNodeRun.ID); err != nil {
log.ErrorWithStackTrace(ctx, sdk.WrapError(err, "unable to get worker list for node run with id %d", workflowNodeRun.ID))
} else if len(blockingWorkers) > 0 {
buf, _ := json.Marshal(blockingWorkers)
log.Info(ctx, "list of workers for node run %d that could block jobs deletion (len:%d): %q", workflowNodeRun.ID, len(blockingWorkers), string(buf))
}
// End of temporary debug

// Delete the line in workflow_node_run_job
if err := DeleteNodeJobRuns(db, workflowNodeRun.ID); err != nil {
return nil, sdk.WrapError(err, "unable to delete node %d job runs", workflowNodeRun.ID)
}

Expand Down
12 changes: 8 additions & 4 deletions engine/api/workflow_queue.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -596,11 +596,17 @@ func (api *API) postJobResult(ctx context.Context, tx gorpmapper.SqlExecutorWith
}
// ^ build variables are now updated on job run and on node

var reloadedWorker *sdk.Worker
if wr != nil {
//Update worker status
if err := worker.SetStatus(ctx, tx, wr.ID, sdk.StatusWaiting); err != nil {
return nil, sdk.WrapError(err, "cannot update worker %s status", wr.ID)
}
var err error
reloadedWorker, err = worker.LoadByID(ctx, tx, wr.ID)
if err != nil {
log.Error(ctx, "unable to reload worker %q: %v", wr.ID, err)
}
}

// Update action status
Expand All @@ -611,11 +617,9 @@ func (api *API) postJobResult(ctx context.Context, tx gorpmapper.SqlExecutorWith
// pq: update or delete on table "workflow_node_run_job" violates foreign key constraint "fk_worker_workflow_node_run_job" on table "worker")
if wr != nil && strings.Contains(err.Error(), "fk_worker_workflow_node_run_job") {
log.ErrorWithStackTrace(ctx, err)
statusFromDB, err := tx.SelectNullStr("select status from worker where id = $1", wr.ID)
if err != nil {
log.ErrorWithStackTrace(ctx, sdk.WrapError(err, "unable to get worker %q status in db with id %q", wr.Name, wr.ID))
if reloadedWorker != nil {
log.Error(ctx, "reloaded worker %s (%s) status is %q", reloadedWorker.Name, reloadedWorker.ID, reloadedWorker.Status)
}
log.Error(ctx, "worker %q status is %q (%q in DB)", wr.Name, wr.Status, statusFromDB)
}
return nil, sdk.WrapError(err, "cannot update NodeJobRun %d status", job.ID)
}
Expand Down