feat: force delete work in the cluster namespace (#269)

pkg/controllers/membercluster/membercluster_controller.go

@@ -25,7 +25,9 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1"
 
 	"go.goms.io/fleet/apis"
 	fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1"
@@ -60,12 +62,26 @@ type Reconciler struct {
 
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	klog.V(3).InfoS("Reconcile", "memberCluster", req.NamespacedName)
-	var mc fleetv1alpha1.MemberCluster
-	if err := r.Client.Get(ctx, req.NamespacedName, &mc); err != nil {
+	var oldMC fleetv1alpha1.MemberCluster
+	if err := r.Client.Get(ctx, req.NamespacedName, &oldMC); err != nil {
 		klog.ErrorS(err, "failed to get member cluster", "memberCluster", req.Name)
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
+	// Handle deleting member cluster, garbage collect all the resources in the cluster namespace
+	if !oldMC.DeletionTimestamp.IsZero() {
+		klog.V(2).InfoS("the member cluster is in the process of being deleted", "memberCluster", klog.KObj(&oldMC))
+		return r.garbageCollectWork(ctx, &oldMC)
+	}
+
+	mc := oldMC.DeepCopy()
+	mcObjRef := klog.KObj(mc)
+	// Add the finalizer to the member cluster
+	if err := r.ensureFinalizer(ctx, mc); err != nil {
+		klog.ErrorS(err, "failed to add the finalizer to member cluster", "memberCluster", mcObjRef)
+		return ctrl.Result{}, err
+	}
+
 	// Get current internal member cluster.
 	namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, mc.Name)
 	imcNamespacedName := types.NamespacedName{Namespace: namespaceName, Name: mc.Name}
@@ -82,32 +98,76 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 
 	switch mc.Spec.State {
 	case fleetv1alpha1.ClusterStateJoin:
-		if err := r.join(ctx, &mc, currentImc); err != nil {
-			klog.ErrorS(err, "failed to join", "memberCluster", klog.KObj(&mc))
+		if err := r.join(ctx, mc, currentImc); err != nil {
+			klog.ErrorS(err, "failed to join", "memberCluster", mcObjRef)
 			return ctrl.Result{}, err
 		}
 
 	case fleetv1alpha1.ClusterStateLeave:
-		if err := r.leave(ctx, &mc, currentImc); err != nil {
-			klog.ErrorS(err, "failed to leave", "memberCluster", klog.KObj(&mc))
+		if err := r.leave(ctx, mc, currentImc); err != nil {
+			klog.ErrorS(err, "failed to leave", "memberCluster", mcObjRef)
 			return ctrl.Result{}, err
 		}
 
 	default:
-		klog.Errorf("encountered a fatal error. unknown state %v in MemberCluster: %s", mc.Spec.State, klog.KObj(&mc))
+		klog.Errorf("encountered a fatal error. unknown state %v in MemberCluster: %s", mc.Spec.State, mcObjRef)
 		return ctrl.Result{}, nil
 	}
 
 	// Copy status from InternalMemberCluster to MemberCluster.
-	r.syncInternalMemberClusterStatus(currentImc, &mc)
-	if err := r.updateMemberClusterStatus(ctx, &mc); err != nil {
-		klog.ErrorS(err, "failed to update status for", klog.KObj(&mc))
+	r.syncInternalMemberClusterStatus(currentImc, mc)
+	if err := r.updateMemberClusterStatus(ctx, mc); err != nil {
+		klog.ErrorS(err, "failed to update status for", klog.KObj(mc))
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
 	return ctrl.Result{}, nil
 }
 
+// garbageCollectWork remove all the finalizers on the work that are in the cluster namespace
+func (r *Reconciler) garbageCollectWork(ctx context.Context, mc *fleetv1alpha1.MemberCluster) (ctrl.Result, error) {
+	var works workv1alpha1.WorkList
+	var clusterNS corev1.Namespace
+	// check if the namespace still exist
+	namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, mc.Name)
+	if err := r.Client.Get(ctx, types.NamespacedName{Name: namespaceName}, &clusterNS); apierrors.IsNotFound(err) {
+		klog.V(2).InfoS("the member cluster namespace is successfully deleted", "memberCluster", klog.KObj(mc))
+		return ctrl.Result{}, nil
+	}
+	// list all the work object we created in the member cluster namespace
+	listOpts := []client.ListOption{
+		client.MatchingLabels{utils.LabelFleetObj: utils.LabelFleetObjValue},
+		client.InNamespace(namespaceName),
+	}
+	if err := r.Client.List(ctx, &works, listOpts...); err != nil {
+		klog.ErrorS(err, "failed to list all the work object", "memberCluster", klog.KObj(mc))
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+	for _, work := range works.Items {
+		staleWork := work.DeepCopy()
+		staleWork.SetFinalizers(nil)
+		if updateErr := r.Update(ctx, staleWork, &client.UpdateOptions{}); updateErr != nil {
+			klog.ErrorS(updateErr, "failed to remove the finalizer from the work",
+				"memberCluster", klog.KObj(mc), "work", klog.KObj(staleWork))
+			return ctrl.Result{}, updateErr
+		}
+	}
+	klog.V(2).InfoS("successfully removed all the work finalizers in the cluster namespace",
+		"memberCluster", klog.KObj(mc), "number of work", len(works.Items))
+	controllerutil.RemoveFinalizer(mc, utils.MemberClusterFinalizer)
+	return ctrl.Result{}, r.Update(ctx, mc, &client.UpdateOptions{})
+}
+
+// ensureFinalizer makes sure that the member cluster CR has a finalizer on it
+func (r *Reconciler) ensureFinalizer(ctx context.Context, mc *fleetv1alpha1.MemberCluster) error {
+	if controllerutil.ContainsFinalizer(mc, utils.MemberClusterFinalizer) {
+		return nil
+	}
+	klog.InfoS("add the member cluster finalizer", "memberCluster", klog.KObj(mc))
+	controllerutil.AddFinalizer(mc, utils.MemberClusterFinalizer)
+	return r.Update(ctx, mc, client.FieldOwner(utils.MCControllerFieldManagerName))
+}
+
 // join takes the actions to make hub cluster ready for member cluster to join, including:
 // - Create namespace for member cluster
 // - Create role & role bindings for member cluster to access hub cluster

pkg/resourcewatcher/event_handlers.go

@@ -168,12 +168,13 @@ func (d *ChangeDetector) onResourceUpdated(oldObj, newObj interface{}) {
 		return
 	}
 	if oldObjMeta.GetResourceVersion() != newObjMeta.GetResourceVersion() {
-		klog.V(5).InfoS("A resource is updated", "obj", klog.KObj(oldObjMeta),
-			"gvk", runtimeObject.GetObjectKind().GroupVersionKind().String())
+		klog.V(5).InfoS("A resource is updated", "obj", oldObjMeta.GetName(),
+			"namespace", oldObjMeta.GetNamespace(), "gvk", runtimeObject.GetObjectKind().GroupVersionKind().String())
 		d.ResourceChangeController.Enqueue(newObj)
 		return
 	}
-	klog.V(5).InfoS("Received a resource updated event with no change", "obj", klog.KObj(oldObjMeta))
+	klog.V(5).InfoS("Received a resource updated event with no change", "obj", oldObjMeta.GetName(),
+		"namespace", oldObjMeta.GetNamespace(), "gvk", runtimeObject.GetObjectKind().GroupVersionKind().String())
 }
 
 // onResourceDeleted handles object delete event and push the deleted object to the resource queue.

pkg/utils/common.go

@@ -57,8 +57,8 @@ const (
 	// This label aims to enable different work objects to be managed by different placement.
 	LabelWorkPlacementName = "work.fleet.azure.com/placement-name"
 
-	// PlacementFinalizer is used to make sure that we handle gc of placement resources.
-	PlacementFinalizer = "work.fleet.azure.com/placement-protection"
+	// MemberClusterFinalizer is used to make sure that we handle gc of all the member cluster resources on the hub cluster
+	MemberClusterFinalizer = "work.fleet.azure.com/membercluster-finalizer"
 )
 
 var (

pkg/utils/test_util.go

@@ -85,3 +85,26 @@ func (matcher NotFoundMatcher) FailureMessage(actual interface{}) (message strin
 func (matcher NotFoundMatcher) NegatedFailureMessage(actual interface{}) (message string) {
 	return format.Message(actual, "to be found")
 }
+
+// AlreadyExistMatcher matches the error to be already exist
+type AlreadyExistMatcher struct {
+}
+
+// Match matches error.
+func (matcher AlreadyExistMatcher) Match(actual interface{}) (success bool, err error) {
+	if actual == nil {
+		return false, nil
+	}
+	actualError := actual.(error)
+	return apierrors.IsAlreadyExists(actualError), nil
+}
+
+// FailureMessage builds an error message.
+func (matcher AlreadyExistMatcher) FailureMessage(actual interface{}) (message string) {
+	return format.Message(actual, "to be already exist")
+}
+
+// NegatedFailureMessage builds an error message.
+func (matcher AlreadyExistMatcher) NegatedFailureMessage(actual interface{}) (message string) {
+	return format.Message(actual, "not to be already exist")
+}

test/e2e/README.md

@@ -18,21 +18,32 @@ make install-helm
 ```
 
 3. Run the test
+
+run the e2e test suite
  ```shell
 make run-e2e
 ```
+or test manually
+```shell
+kubectl --context=kind-hub-testing apply -f examples/fleet_v1alpha1_membercluster.yaml
+kubectl --context=kind-hub-testing apply -f test/integration/manifests/resources
+kubectl --context=kind-hub-testing apply -f test/integration/manifests/resources
+kubectl --context=kind-hub-testing apply -f test/integration/manifests/placement/select-namespace.yaml
+```
+
+4. Check the controller logs 
 
-4. Check the logs of the hub cluster controller
+check the logs of the hub cluster controller
 ```shell
 kubectl --context=kind-hub-testing -n fleet-system get pod 
 ```
 
-4. Check the logs of the member cluster controller
+check the logs of the member cluster controller
 ```shell
 kubectl --context=kind-member-testing -n fleet-system get pod 
 ```
 
-5. uninstall the resources
+5.uninstall the resources
 ```shell
 make uninstall-helm
 ```

test/e2e/utils/helper.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
-	"github.com/onsi/gomega/format"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -247,26 +246,3 @@ func DeleteServiceAccount(cluster framework.Cluster, sa *corev1.ServiceAccount)
 		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
 	})
 }
-
-// AlreadyExistMatcher matches the error to be already exist
-type AlreadyExistMatcher struct {
-}
-
-// Match matches error.
-func (matcher AlreadyExistMatcher) Match(actual interface{}) (success bool, err error) {
-	if actual == nil {
-		return false, nil
-	}
-	actualError := actual.(error)
-	return apierrors.IsAlreadyExists(actualError), nil
-}
-
-// FailureMessage builds an error message.
-func (matcher AlreadyExistMatcher) FailureMessage(actual interface{}) (message string) {
-	return format.Message(actual, "to be already exist")
-}
-
-// NegatedFailureMessage builds an error message.
-func (matcher AlreadyExistMatcher) NegatedFailureMessage(actual interface{}) (message string) {
-	return format.Message(actual, "not to be already exist")
-}

test/e2e/work_api_test.go

@@ -25,7 +25,6 @@ import (
 
 	fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1"
 	fleetutil "go.goms.io/fleet/pkg/utils"
-	"go.goms.io/fleet/test/e2e/utils"
 )
 
 const (
@@ -45,7 +44,7 @@ var _ = Describe("work-api testing", Ordered, func() {
 
 	BeforeAll(func() {
 		_, err := HubCluster.KubeClientSet.CoreV1().Namespaces().Create(context.Background(), wns, metav1.CreateOptions{})
-		Expect(err).Should(SatisfyAny(Succeed(), &utils.AlreadyExistMatcher{}))
+		Expect(err).Should(SatisfyAny(Succeed(), &fleetutil.AlreadyExistMatcher{}))
 	})
 
 	Context("with a Work resource that has two manifests: Deployment & Service", func() {