Recreate pod on TaskRun's pod deletion

A TaskRun's pod may be deleted either manually by the user or due to system constraints (e.g. node recreation). This change adds modifies the TaskRun reconciliation logic to recreate pods which are not found.
tektoncd · Apr 13, 2019 · d1c46b0 · d1c46b0
1 parent 40c9b4a
commit d1c46b0
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 5 deletions.
diff --git a/pkg/reconciler/v1alpha1/taskrun/taskrun.go b/pkg/reconciler/v1alpha1/taskrun/taskrun.go
@@ -279,12 +279,18 @@ func (c *Reconciler) reconcile(ctx context.Context, tr *v1alpha1.TaskRun) error
 	if tr.Status.PodName != "" {
 		pod, err = c.KubeClientSet.CoreV1().Pods(tr.Namespace).Get(tr.Status.PodName, metav1.GetOptions{})
 		if err != nil {
-			c.Logger.Errorf("Error getting pod %q: %v", tr.Status.PodName, err)
-			return err
+			if errors.IsNotFound(err) {
+				// Reset the start time + timeout
+				tr.Status.StartTime = nil
+				c.timeoutHandler.Release(tr)
+			} else {
+				c.Logger.Errorf("Error getting pod %q: %v", tr.Status.PodName, err)
+				return err
+			}
 		}
-	} else {
+	}
+	if pod == nil {
 		// Pod is not present, create pod.
-		go c.timeoutHandler.WaitTaskRun(tr)
 		pod, err = c.createPod(tr, rtr.TaskSpec, rtr.TaskName)
 		if err != nil {
 			// This Run has failed, so we need to mark it as failed and stop reconciling it

diff --git a/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go b/pkg/reconciler/v1alpha1/taskrun/taskrun_test.go
@@ -273,11 +273,16 @@ func TestReconcile(t *testing.T) {
 		),
 	)
 
+	taskRunWithPod := tb.TaskRun("test-taskrun-with-pod", "foo",
+		tb.TaskRunSpec(tb.TaskRunTaskRef(simpleTask.Name)),
+		tb.TaskRunStatus(tb.PodName("some-pod-that-no-longer-exists")),
+	)
+
 	taskruns := []*v1alpha1.TaskRun{
 		taskRunSuccess, taskRunWithSaSuccess,
 		taskRunTemplating, taskRunInputOutput,
 		taskRunWithTaskSpec, taskRunWithClusterTask, taskRunWithResourceSpecAndTaskSpec,
-		taskRunWithLabels,
+		taskRunWithLabels, taskRunWithPod,
 	}
 
 	d := test.Data{
@@ -652,6 +657,36 @@ func TestReconcile(t *testing.T) {
 				),
 			),
 		),
+	}, {
+		name:    "taskrun-with-pod",
+		taskRun: taskRunWithPod,
+		wantPod: tb.Pod("test-taskrun-with-pod-pod-123456", "foo",
+			tb.PodAnnotation("sidecar.istio.io/inject", "false"),
+			tb.PodLabel(taskNameLabelKey, "test-task"),
+			tb.PodLabel(taskRunNameLabelKey, "test-taskrun-with-pod"),
+			tb.PodOwnerReference("TaskRun", "test-taskrun-with-pod",
+				tb.OwnerReferenceAPIVersion(currentApiVersion)),
+			tb.PodSpec(
+				tb.PodVolumes(toolsVolume, workspaceVolume, homeVolume),
+				tb.PodRestartPolicy(corev1.RestartPolicyNever),
+				getCredentialsInitContainer("9l9zj"),
+				placeToolsInitContainer,
+				tb.PodContainer("build-step-simple-step", "foo",
+					tb.Command(entrypointLocation),
+					tb.Args("-wait_file", "", "-post_file", "/builder/tools/0", "-entrypoint", "/mycmd", "--"),
+					tb.WorkingDir(workspaceDir),
+					tb.EnvVar("HOME", "/builder/home"),
+					tb.VolumeMount("tools", "/builder/tools"),
+					tb.VolumeMount("workspace", workspaceDir),
+					tb.VolumeMount("home", "/builder/home"),
+				),
+				tb.PodContainer("nop", "override-with-nop:latest",
+					tb.Command("/builder/tools/entrypoint"),
+					tb.Args("-wait_file", "/builder/tools/0", "-post_file", "/builder/tools/1", "-entrypoint", "/ko-app/nop", "--"),
+					tb.VolumeMount(entrypoint.MountName, entrypoint.MountPoint),
+				),
+			),
+		),
 	}} {
 		t.Run(tc.name, func(t *testing.T) {
 			names.TestingSeed()