feat: report error when common PVC cleanup job hangs

Fix devfile#551

Signed-off-by: Andrew Obuchowicz <[email protected]>

pkg/provision/storage/cleanup.go

@@ -28,7 +28,9 @@ import (
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/types"
+	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	"github.com/devfile/devworkspace-operator/internal/images"
@@ -91,6 +93,14 @@ func runCommonPVCCleanupJob(workspace *dw.DevWorkspace, clusterAPI sync.ClusterA
 			}
 		}
 	}
+
+	if msg, err := checkCommonPVCCleanupJobEvents(clusterJob, workspace.Status.DevWorkspaceId, clusterAPI); err != nil || msg != "" {
+		return &ProvisioningError{
+			Err:     err,
+			Message: fmt.Sprintf("DevWorkspace PVC cleanup job failed: see logs for job %q for details. Additional information: %s", clusterJob.Name, msg),
+		}
+	}
+
 	// Requeue at least each 10 seconds to check if PVC is not removed by someone else
 	return &NotReadyError{
 		Message:      "Cleanup job is not in completed state",
@@ -203,3 +213,25 @@ func commonPVCExists(workspace *dw.DevWorkspace, clusterAPI sync.ClusterAPI) (bo
 	}
 	return true, nil
 }
+
+func checkCommonPVCCleanupJobEvents(job *batchv1.Job, workspaceID string, clusterAPI sync.ClusterAPI) (msg string, err error) {
+	evs := &corev1.EventList{}
+	selector, err := fields.ParseSelector(fmt.Sprintf("involvedObject.name=%s", job.Name))
+	if err != nil {
+		return "", fmt.Errorf("failed to parse field selector: %s", err)
+	}
+	if err := clusterAPI.Client.List(clusterAPI.Ctx, evs, k8sclient.InNamespace(job.Namespace), k8sclient.MatchingFieldsSelector{Selector: selector}); err != nil {
+		return "", fmt.Errorf("failed to list events in namespace %s: %w", job.Namespace, err)
+	}
+	for _, ev := range evs.Items {
+		if ev.InvolvedObject.Kind != "Job" {
+			continue
+		}
+
+		if ev.Reason == "FailedScheduling" {
+			msg = fmt.Sprintf("Detected unrecoverable event %s: %s", ev.Reason, ev.Message)
+			return msg, nil
+		}
+	}
+	return "", nil
+}