fix(api): debug constraint fk_worker_workflow_node_run_job (#6678)

ovh · Nov 9, 2023 · 46ea8c6 · 46ea8c6
1 parent e9a28d1
commit 46ea8c6
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 22 deletions.
diff --git a/engine/api/workflow/execute_node_run.go b/engine/api/workflow/execute_node_run.go
@@ -307,28 +307,28 @@ func executeNodeRun(ctx context.Context, db gorpmapper.SqlExecutorWithTx, store
 			report.Merge(ctx, r1)
 		}
 
-		// Delete the line in workflow_node_run_job
-		if err := DeleteNodeJobRuns(db, workflowNodeRun.ID); err != nil {
-			// Checking the error:
-			// pq: update or delete on table "workflow_node_run_job" violates foreign key constraint "fk_worker_workflow_node_run_job" on table "worker")
-			type WorkerInfo struct {
-				WorkerID             string `db:"worker_id"`
-				WorkerName           string `db:"worker_name"`
-				WorkflowNodeRunJobID string `db:"workflow_node_run_job_id"`
-			}
-
-			var workers []WorkerInfo
-			if _, errSelect := db.Select(&workers, `
+		// Temporary debug:
+		// Listing worker for node run that block the job deletion
+		type WorkerInfo struct {
+			WorkerID             string `db:"worker_id"`
+			WorkerName           string `db:"worker_name"`
+			WorkflowNodeRunJobID string `db:"workflow_node_run_job_id"`
+		}
+		var blockingWorkers []WorkerInfo
+		if _, err := db.Select(&blockingWorkers, `
         SELECT worker.id as worker_id, worker.name as worker_name, workflow_node_run_job.id as workflow_node_run_job_id FROM worker
         JOIN workflow_node_run_job ON workflow_node_run_job.worker_id = worker.id
         WHERE workflow_node_run_job.workflow_node_run_id = $1
-      `, workflowNodeRun.ID); errSelect != nil {
-				log.ErrorWithStackTrace(ctx, sdk.WrapError(errSelect, "unable to get worker list for node run with id %d", workflowNodeRun.ID))
-			} else {
-				buf, _ := json.Marshal(workers)
-				log.Error(ctx, "list of workers for node run %d that block jobs deletion (len:%d): %q", workflowNodeRun.ID, len(workers), string(buf))
-			}
+      `, workflowNodeRun.ID); err != nil {
+			log.ErrorWithStackTrace(ctx, sdk.WrapError(err, "unable to get worker list for node run with id %d", workflowNodeRun.ID))
+		} else if len(blockingWorkers) > 0 {
+			buf, _ := json.Marshal(blockingWorkers)
+			log.Info(ctx, "list of workers for node run %d that could block jobs deletion (len:%d): %q", workflowNodeRun.ID, len(blockingWorkers), string(buf))
+		}
+		// End of temporary debug
 
+		// Delete the line in workflow_node_run_job
+		if err := DeleteNodeJobRuns(db, workflowNodeRun.ID); err != nil {
 			return nil, sdk.WrapError(err, "unable to delete node %d job runs", workflowNodeRun.ID)
 		}
 

diff --git a/engine/api/workflow_queue.go b/engine/api/workflow_queue.go
@@ -596,11 +596,17 @@ func (api *API) postJobResult(ctx context.Context, tx gorpmapper.SqlExecutorWith
 	}
 	// ^ build variables are now updated on job run and on node
 
+	var reloadedWorker *sdk.Worker
 	if wr != nil {
 		//Update worker status
 		if err := worker.SetStatus(ctx, tx, wr.ID, sdk.StatusWaiting); err != nil {
 			return nil, sdk.WrapError(err, "cannot update worker %s status", wr.ID)
 		}
+		var err error
+		reloadedWorker, err = worker.LoadByID(ctx, tx, wr.ID)
+		if err != nil {
+			log.Error(ctx, "unable to reload worker %q: %v", wr.ID, err)
+		}
 	}
 
 	// Update action status
@@ -611,11 +617,9 @@ func (api *API) postJobResult(ctx context.Context, tx gorpmapper.SqlExecutorWith
 		// pq: update or delete on table "workflow_node_run_job" violates foreign key constraint "fk_worker_workflow_node_run_job" on table "worker")
 		if wr != nil && strings.Contains(err.Error(), "fk_worker_workflow_node_run_job") {
 			log.ErrorWithStackTrace(ctx, err)
-			statusFromDB, err := tx.SelectNullStr("select status from worker where id = $1", wr.ID)
-			if err != nil {
-				log.ErrorWithStackTrace(ctx, sdk.WrapError(err, "unable to get worker %q status in db with id %q", wr.Name, wr.ID))
+			if reloadedWorker != nil {
+				log.Error(ctx, "reloaded worker %s (%s) status is %q", reloadedWorker.Name, reloadedWorker.ID, reloadedWorker.Status)
 			}
-			log.Error(ctx, "worker %q status is %q (%q in DB)", wr.Name, wr.Status, statusFromDB)
 		}
 		return nil, sdk.WrapError(err, "cannot update NodeJobRun %d status", job.ID)
 	}