Recreate pod on TaskRun's pod deletion

A TaskRun's pod may be deleted either manually by the user or due to system constraints (e.g. node recreation). This change adds modifies the TaskRun reconciliation logic to recreate pods which are not found.

pkg/reconciler/v1alpha1/taskrun/resources/pod.go

@@ -29,6 +29,7 @@ import (
 
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -160,6 +161,23 @@ func makeCredentialInitializer(serviceAccountName, namespace string, kubeclient
 	}, volumes, nil
 }
 
+// GetPod returns the Pod for the given pod name
+type GetPod func(string, metav1.GetOptions) (*corev1.Pod, error)
+
+// TryGetPod fetches the TaskRun's pod, returning nil if it has not been created or it does not exist.
+func TryGetPod(taskRunStatus v1alpha1.TaskRunStatus, gp GetPod) (*corev1.Pod, error) {
+	if taskRunStatus.PodName == "" {
+		return nil, nil
+	}
+
+	pod, err := gp(taskRunStatus.PodName, metav1.GetOptions{})
+	if err == nil || errors.IsNotFound(err) {
+		return pod, nil
+	}
+
+	return nil, err
+}
+
 // MakePod converts TaskRun and TaskSpec objects to a Pod which implements the taskrun specified
 // by the supplied CRD.
 func MakePod(taskRun *v1alpha1.TaskRun, taskSpec v1alpha1.TaskSpec, kubeclient kubernetes.Interface, cache *entrypoint.Cache, logger *zap.SugaredLogger) (*corev1.Pod, error) {

pkg/reconciler/v1alpha1/taskrun/resources/pod_test.go

@@ -18,13 +18,16 @@ package resources
 
 import (
 	"crypto/rand"
+	"fmt"
 	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	fakek8s "k8s.io/client-go/kubernetes/fake"
 
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
@@ -48,6 +51,68 @@ var (
 	}
 )
 
+func TestTryGetPod(t *testing.T) {
+	err := fmt.Errorf("something went wrong")
+	for _, c := range []struct {
+		desc    string
+		trs     v1alpha1.TaskRunStatus
+		gp      GetPod
+		wantNil bool
+		wantErr error
+	}{{
+		desc: "no-pod",
+		trs:  v1alpha1.TaskRunStatus{},
+		gp: func(string, metav1.GetOptions) (*corev1.Pod, error) {
+			t.Errorf("Did not expect pod to be fetched")
+			return nil, nil
+		},
+		wantNil: true,
+		wantErr: nil,
+	}, {
+		desc: "non-existent-pod",
+		trs: v1alpha1.TaskRunStatus{
+			PodName: "no-longer-exist",
+		},
+		gp: func(name string, opts metav1.GetOptions) (*corev1.Pod, error) {
+			return nil, errors.NewNotFound(schema.GroupResource{}, name)
+		},
+		wantNil: true,
+		wantErr: nil,
+	}, {
+		desc: "existing-pod",
+		trs: v1alpha1.TaskRunStatus{
+			PodName: "exists",
+		},
+		gp: func(name string, opts metav1.GetOptions) (*corev1.Pod, error) {
+			return &corev1.Pod{}, nil
+		},
+		wantNil: false,
+		wantErr: nil,
+	}, {
+		desc: "pod-fetch-error",
+		trs: v1alpha1.TaskRunStatus{
+			PodName: "something-went-wrong",
+		},
+		gp: func(name string, opts metav1.GetOptions) (*corev1.Pod, error) {
+			return nil, err
+		},
+		wantNil: true,
+		wantErr: err,
+	}} {
+		t.Run(c.desc, func(t *testing.T) {
+			pod, err := TryGetPod(c.trs, c.gp)
+			if err != c.wantErr {
+				t.Fatalf("TryGetPod: %v", err)
+			}
+
+			wasNil := pod == nil
+			if wasNil != c.wantNil {
+				t.Errorf("Pod got %v, want %v", wasNil, c.wantNil)
+			}
+		})
+	}
+}
+
 func TestMakePod(t *testing.T) {
 	names.TestingSeed()
 	subPath := "subpath"

pkg/reconciler/v1alpha1/taskrun/taskrun.go

@@ -275,14 +275,12 @@ func (c *Reconciler) reconcile(ctx context.Context, tr *v1alpha1.TaskRun) error
 	}
 
 	// Get the TaskRun's Pod if it should have one. Otherwise, create the Pod.
-	var pod *corev1.Pod
-	if tr.Status.PodName != "" {
-		pod, err = c.KubeClientSet.CoreV1().Pods(tr.Namespace).Get(tr.Status.PodName, metav1.GetOptions{})
-		if err != nil {
-			c.Logger.Errorf("Error getting pod %q: %v", tr.Status.PodName, err)
-			return err
-		}
-	} else {
+	pod, err := resources.TryGetPod(tr.Status, c.KubeClientSet.CoreV1().Pods(tr.Namespace).Get)
+	if err != nil {
+		c.Logger.Errorf("Error getting pod %q: %v", tr.Status.PodName, err)
+		return err
+	}
+	if pod == nil {
 		// Pod is not present, create pod.
 		pod, err = c.createPod(tr, rtr.TaskSpec, rtr.TaskName)
 		if err != nil {

pkg/reconciler/v1alpha1/taskrun/taskrun_test.go

@@ -333,11 +333,16 @@ func TestReconcile(t *testing.T) {
 		),
 	)
 
+	taskRunWithPod := tb.TaskRun("test-taskrun-with-pod", "foo",
+		tb.TaskRunSpec(tb.TaskRunTaskRef(simpleTask.Name)),
+		tb.TaskRunStatus(tb.PodName("some-pod-that-no-longer-exists")),
+	)
+
 	taskruns := []*v1alpha1.TaskRun{
 		taskRunSuccess, taskRunWithSaSuccess,
 		taskRunTemplating, taskRunInputOutput,
 		taskRunWithTaskSpec, taskRunWithClusterTask, taskRunWithResourceSpecAndTaskSpec,
-		taskRunWithLabels, taskRunWithResourceRequests, taskRunTaskEnv,
+		taskRunWithLabels, taskRunWithResourceRequests, taskRunTaskEnv, taskRunWithPod,
 	}
 
 	d := test.Data{
@@ -911,6 +916,41 @@ func TestReconcile(t *testing.T) {
 				),
 			),
 		),
+	}, {
+		name:    "taskrun-with-pod",
+		taskRun: taskRunWithPod,
+		wantPod: tb.Pod("test-taskrun-with-pod-pod-123456", "foo",
+			tb.PodAnnotation("sidecar.istio.io/inject", "false"),
+			tb.PodLabel(taskNameLabelKey, "test-task"),
+			tb.PodLabel(taskRunNameLabelKey, "test-taskrun-with-pod"),
+			tb.PodOwnerReference("TaskRun", "test-taskrun-with-pod",
+				tb.OwnerReferenceAPIVersion(currentApiVersion)),
+			tb.PodSpec(
+				tb.PodVolumes(toolsVolume, workspaceVolume, homeVolume),
+				tb.PodRestartPolicy(corev1.RestartPolicyNever),
+				getCredentialsInitContainer("9l9zj"),
+				getPlaceToolsInitContainer(),
+				tb.PodContainer("build-step-simple-step", "foo",
+					tb.Command(entrypointLocation),
+					tb.Args("-wait_file", "", "-post_file", "/builder/tools/0", "-entrypoint", "/mycmd", "--"),
+					tb.WorkingDir(workspaceDir),
+					tb.EnvVar("HOME", "/builder/home"),
+					tb.VolumeMount("tools", "/builder/tools"),
+					tb.VolumeMount("workspace", workspaceDir),
+					tb.VolumeMount("home", "/builder/home"),
+					tb.Resources(tb.Requests(
+						tb.CPU("0"),
+						tb.Memory("0"),
+						tb.EphemeralStorage("0"),
+					)),
+				),
+				tb.PodContainer("nop", "override-with-nop:latest",
+					tb.Command("/builder/tools/entrypoint"),
+					tb.Args("-wait_file", "/builder/tools/0", "-post_file", "/builder/tools/1", "-entrypoint", "/ko-app/nop", "--"),
+					tb.VolumeMount(entrypoint.MountName, entrypoint.MountPoint),
+				),
+			),
+		),
 	}} {
 		t.Run(tc.name, func(t *testing.T) {
 			names.TestingSeed()