From 5da914490ac5aedd1a2b70c589650144ab28f1dc Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Fri, 23 Sep 2022 11:40:31 -0700 Subject: [PATCH 01/20] Refactor workload E2E --- test/e2e/e2e_test.go | 6 +- test/e2e/join_leave_placement_test.go | 35 ++- test/e2e/utils/helper.go | 68 +----- test/e2e/utils/workload_test_utils.go | 83 +++++++ test/e2e/work_api_e2e_test.go | 4 +- test/e2e/work_load_test.go | 302 ++++++++++++++++++++++++-- 6 files changed, 405 insertions(+), 93 deletions(-) create mode 100644 test/e2e/utils/workload_test_utils.go diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 23748f10b..ec9491e43 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -165,10 +165,10 @@ var _ = BeforeSuite(func() { MemberCluster.HubURL = hubURL framework.GetClusterClient(MemberCluster) - testutils.CreateNamespace(*HubCluster, workNamespace) - ctx = context.Background() + testutils.CreateNamespace(ctx, *HubCluster, workNamespace) + By("deploy member cluster in the hub cluster") identity := rbacv1.Subject{ Name: "member-agent-sa", @@ -261,8 +261,6 @@ var _ = AfterSuite(func() { return nil }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) - testutils.DeleteNamespace(*MemberCluster, memberNamespace) - testutils.DeleteNamespace(*HubCluster, workNamespace) By("delete member cluster") testutils.DeleteMemberCluster(ctx, *HubCluster, mc) }) diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go index 875cefaa0..d43ad338e 100644 --- a/test/e2e/join_leave_placement_test.go +++ b/test/e2e/join_leave_placement_test.go @@ -22,9 +22,8 @@ import ( ) // Serial - Ginkgo will guarantee that these specs will never run in parallel with other specs. -// Ordered - Ginkgo will guarantee that specs in an Ordered container will run sequentially, in the order they are written. // This test cannot be run in parallel with other specs in the suite as it's leaving, joining, leaving and joining again. -var _ = Describe("workload orchestration testing with join/leave", Serial, Ordered, func() { +var _ = Describe("workload orchestration testing with join/leave", Serial, func() { var ( crp *v1alpha1.ClusterResourcePlacement ctx context.Context @@ -96,7 +95,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order }, }, } - testutils.CreateClusterResourcePlacement(*HubCluster, crp) + testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) By("verify the resource is not propagated to member cluster") Consistently(func() bool { @@ -124,7 +123,33 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) By("verify that the cluster resource placement is applied") - testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) + //testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) + crpStatus := v1alpha1.ClusterResourcePlacementStatus{ + Conditions: []metav1.Condition{ + { + Message: "Successfully scheduled resources for placement", + Reason: "ScheduleSucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled), + }, + { + Message: "Successfully applied resources to member clusters", + Reason: "ApplySucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), + }, + }, + SelectedResources: []v1alpha1.ResourceIdentifier{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: cr.Name, + }, + }, + TargetClusters: []string{"kind-member-testing"}, + } + testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, crp, crpStatus, 3*testutils.PollTimeout) By("verify the resource is propagated to member cluster") Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName) @@ -155,7 +180,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) By("delete the crp from the hub") - testutils.DeleteClusterResourcePlacement(*HubCluster, crp) + testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) By("verify that the resource is still on the member cluster") Consistently(func() error { diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index 0ff01d8c6..321f0579f 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -54,15 +54,6 @@ func CreateClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { }) } -// WaitClusterRole waits for cluster roles to be created. -func WaitClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { - klog.Infof("Waiting for ClusterRole(%s) to be synced", cr.Name) - gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) -} - // DeleteClusterRole deletes cluster role on cluster. func DeleteClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { ginkgo.By(fmt.Sprintf("Deleting ClusterRole(%s)", cr.Name), func() { @@ -71,39 +62,6 @@ func DeleteClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { }) } -// CreateClusterResourcePlacement created ClusterResourcePlacement and waits for ClusterResourcePlacement to exist in hub cluster. -func CreateClusterResourcePlacement(cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { - ginkgo.By(fmt.Sprintf("Creating ClusterResourcePlacement(%s)", crp.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), crp) - gomega.Expect(err).Should(gomega.Succeed()) - }) - klog.Infof("Waiting for ClusterResourcePlacement(%s) to be synced", crp.Name) - gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: crp.Name, Namespace: ""}, crp) - return err - }, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred()) -} - -// WaitConditionClusterResourcePlacement waits for ClusterResourcePlacement to present on th hub cluster with a specific condition. -func WaitConditionClusterResourcePlacement(cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement, - conditionName string, status metav1.ConditionStatus, customTimeout time.Duration) { - klog.Infof("Waiting for ClusterResourcePlacement(%s) condition(%s) status(%s) to be synced", crp.Name, conditionName, status) - gomega.Eventually(func() bool { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: crp.Name, Namespace: ""}, crp) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - cond := crp.GetCondition(conditionName) - return cond != nil && cond.Status == status - }, customTimeout, PollInterval).Should(gomega.Equal(true)) -} - -// DeleteClusterResourcePlacement is used delete ClusterResourcePlacement on the hub cluster. -func DeleteClusterResourcePlacement(cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { - ginkgo.By(fmt.Sprintf("Deleting ClusterResourcePlacement(%s)", crp.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), crp) - gomega.Expect(err).Should(gomega.SatisfyAny(gomega.Succeed(), &utils.NotFoundMatcher{})) - }) -} - // WaitWork waits for Work to be present on the hub cluster. func WaitWork(ctx context.Context, cluster framework.Cluster, workName, workNamespace string) { name := types.NamespacedName{Name: workName, Namespace: workNamespace} @@ -117,27 +75,19 @@ func WaitWork(ctx context.Context, cluster framework.Cluster, workName, workName } // CreateNamespace create namespace and waits for namespace to exist. -func CreateNamespace(cluster framework.Cluster, ns *corev1.Namespace) { - ginkgo.By(fmt.Sprintf("Creating Namespace(%s)", ns.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), ns) - gomega.Expect(err).Should(gomega.Succeed(), "Failed to create namespace %s", ns.Name) - }) - klog.Infof("Waiting for Namespace(%s) to be synced", ns.Name) +func CreateNamespace(ctx context.Context, cluster framework.Cluster, ns *corev1.Namespace) { + gomega.Expect(cluster.KubeClient.Create(ctx, ns)).Should(gomega.Succeed(), "Failed to create namespace %s in %s cluster", ns.Name, cluster.ClusterName) gomega.Eventually(func() error { - err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: ns.Name, Namespace: ""}, ns) - - return err - }, PollTimeout, PollInterval).Should(gomega.Succeed()) + return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: ns.Name}, ns) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for namespace %s to be created in %s cluster", ns.Name, cluster.ClusterName) } // DeleteNamespace delete namespace. -func DeleteNamespace(cluster framework.Cluster, ns *corev1.Namespace) { - ginkgo.By(fmt.Sprintf("Deleting Namespace(%s)", ns.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), ns) - if err != nil && !apierrors.IsNotFound(err) { - gomega.Expect(err).Should(gomega.SatisfyAny(gomega.Succeed(), &utils.NotFoundMatcher{})) - } - }) +func DeleteNamespace(ctx context.Context, cluster framework.Cluster, ns *corev1.Namespace) { + gomega.Expect(cluster.KubeClient.Delete(context.TODO(), ns)).Should(gomega.Succeed(), "Failed to delete namespace %s in %s cluster", ns.Name, cluster.ClusterName) + gomega.Eventually(func() bool { + return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: ns.Name}, ns)) + }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for namespace %s to be deleted in %s cluster", ns.Name, cluster.ClusterName) } // CreateWork creates Work object based on manifest given. diff --git a/test/e2e/utils/workload_test_utils.go b/test/e2e/utils/workload_test_utils.go new file mode 100644 index 000000000..d9c6f20dd --- /dev/null +++ b/test/e2e/utils/workload_test_utils.go @@ -0,0 +1,83 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package utils + +import ( + "context" + "fmt" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/onsi/gomega" + "go.goms.io/fleet/apis/v1alpha1" + "go.goms.io/fleet/test/e2e/framework" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// CmpClusterRole compares actual cluster role with expected cluster role. +func CmpClusterRole(ctx context.Context, cluster framework.Cluster, actualClusterRole, expectedClusterRole *rbacv1.ClusterRole) { + gomega.Eventually(func() error { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualClusterRole.Name}, actualClusterRole); err != nil { + return err + } + ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} + if diff := cmp.Diff(expectedClusterRole, actualClusterRole, ignoreOptions...); diff != "" { + return fmt.Errorf("cluster Role(%s) mismatch (-want +got):\n%s", actualClusterRole.Name, diff) + } + return nil + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for cluster role %s to be updated in %s cluster", actualClusterRole.Name, cluster.ClusterName) +} + +// CmpRole compares actual role with expected role. +func CmpRole(ctx context.Context, cluster framework.Cluster, actualRole, expectedRole *rbacv1.Role) { + gomega.Eventually(func() error { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualRole.Name, Namespace: actualRole.Namespace}, actualRole); err != nil { + return err + } + ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} + if diff := cmp.Diff(expectedRole, actualRole, ignoreOptions...); diff != "" { + return fmt.Errorf("cluster Role(%s) mismatch (-want +got):\n%s", actualRole.Name, diff) + } + return nil + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for role %s to be updated in %s cluster", actualRole.Name, cluster.ClusterName) +} + +// CreateClusterResourcePlacement created ClusterResourcePlacement and waits for ClusterResourcePlacement to exist in hub cluster. +func CreateClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { + gomega.Expect(cluster.KubeClient.Create(ctx, crp)).Should(gomega.Succeed()) + gomega.Eventually(func() error { + return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, cluster.ClusterName) +} + +// WaitCreateClusterResourcePlacementStatus waits for ClusterResourcePlacement to present on th hub cluster with a specific status. +func WaitCreateClusterResourcePlacementStatus(ctx context.Context, cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement, status v1alpha1.ClusterResourcePlacementStatus, customTimeout time.Duration) { + gomega.Eventually(func() error { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp); err != nil { + return err + } + ignoreOption := cmpopts.IgnoreFields(metav1.Condition{}, "ObservedGeneration", "LastTransitionTime") + statusDiff := cmp.Diff(status, crp.Status, ignoreOption) + if statusDiff != "" { + return fmt.Errorf("cluster resource placment(%s) status mismatch (-want +got):\n%s", crp.Name, statusDiff) + } + return nil + }, customTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for cluster resource placement %s status to be updated", crp.Name, cluster.ClusterName) +} + +// DeleteClusterResourcePlacement is used delete ClusterResourcePlacement on the hub cluster. +func DeleteClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { + gomega.Expect(cluster.KubeClient.Delete(ctx, crp)).Should(gomega.Succeed(), "Failed to delete cluster resource placement %s in %s cluster", crp.Name, cluster.ClusterName) + gomega.Eventually(func() bool { + return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)) + }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for cluster resource placement %s to be deleted in %s cluster", crp.Name, cluster.ClusterName) +} diff --git a/test/e2e/work_api_e2e_test.go b/test/e2e/work_api_e2e_test.go index d1a9312bf..3a0d595bd 100644 --- a/test/e2e/work_api_e2e_test.go +++ b/test/e2e/work_api_e2e_test.go @@ -71,11 +71,11 @@ var _ = Describe("Work API Controller test", func() { Name: resourceNamespaceName, }, } - testutils.CreateNamespace(*MemberCluster, resourceNamespace) + testutils.CreateNamespace(ctx, *MemberCluster, resourceNamespace) }) AfterEach(func() { - testutils.DeleteNamespace(*MemberCluster, resourceNamespace) + testutils.DeleteNamespace(ctx, *MemberCluster, resourceNamespace) }) Context("Work Creation Test", func() { diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index 4a31ff06c..da29ff0ea 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -9,8 +9,14 @@ import ( "context" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + workapi "sigs.k8s.io/work-api/pkg/apis/v1alpha1" "go.goms.io/fleet/apis/v1alpha1" testutils "go.goms.io/fleet/test/e2e/utils" @@ -19,23 +25,18 @@ import ( var _ = Describe("workload orchestration testing", func() { var crp *v1alpha1.ClusterResourcePlacement var ctx context.Context + labelKey := "fleet.azure.com/name" + labelValue := "test" BeforeEach(func() { ctx = context.Background() }) - AfterEach(func() { - testutils.DeleteClusterResourcePlacement(*HubCluster, crp) - }) - Context("Test Workload Orchestration", func() { - It("Apply CRP and check if work gets propagated", func() { - workName := "resource-label-selector" - labelKey := "fleet.azure.com/name" - labelValue := "test" + It("Apply CRP and check if cluster role gets propagated, update cluster role", func() { By("create the resources to be propagated") - cr := &rbacv1.ClusterRole{ - ObjectMeta: v1.ObjectMeta{ + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ Name: "test-cluster-role", Labels: map[string]string{labelKey: labelValue}, }, @@ -47,38 +48,293 @@ var _ = Describe("workload orchestration testing", func() { }, }, } - testutils.CreateClusterRole(*HubCluster, cr) + Expect(HubCluster.KubeClient.Create(ctx, clusterRole)).Should(Succeed(), "Failed to create cluster role %s in %s cluster", clusterRole.Name, HubCluster.ClusterName) By("create the cluster resource placement in the hub cluster") crp = &v1alpha1.ClusterResourcePlacement{ - ObjectMeta: v1.ObjectMeta{Name: "resource-label-selector"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-crp1"}, Spec: v1alpha1.ClusterResourcePlacementSpec{ ResourceSelectors: []v1alpha1.ClusterResourceSelector{ { Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole", - LabelSelector: &v1.LabelSelector{ - MatchLabels: map[string]string{"fleet.azure.com/name": "test"}, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: clusterRole.Labels, }, }, }, }, } - testutils.CreateClusterResourcePlacement(*HubCluster, crp) + + testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) By("check if work gets created for cluster resource placement") - testutils.WaitWork(ctx, *HubCluster, workName, memberNamespace.Name) + testutils.WaitWork(ctx, *HubCluster, crp.Name, memberNamespace.Name) - By("check if cluster resource placement is updated to Scheduled & Applied") - testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementConditionTypeScheduled), v1.ConditionTrue, testutils.PollTimeout) - testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), v1.ConditionTrue, testutils.PollTimeout) + By("check if cluster resource placement status is updated") + crpStatus := v1alpha1.ClusterResourcePlacementStatus{ + Conditions: []metav1.Condition{ + { + Message: "Successfully scheduled resources for placement", + Reason: "ScheduleSucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled), + }, + { + Message: "Successfully applied resources to member clusters", + Reason: "ApplySucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), + }, + }, + SelectedResources: []v1alpha1.ResourceIdentifier{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: clusterRole.Name, + }, + }, + TargetClusters: []string{"kind-member-testing"}, + } + testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, crp, crpStatus, 3*testutils.PollTimeout) - By("check if resource is propagated to member cluster") - testutils.WaitClusterRole(*MemberCluster, cr) + By("check if cluster role is propagated to member cluster") + ownerReferences := []metav1.OwnerReference{ + { + APIVersion: workapi.GroupVersion.String(), + BlockOwnerDeletion: pointer.Bool(false), + Kind: "AppliedWork", + Name: crp.Name, + }, + } + expectedClusterRole := clusterRole + expectedClusterRole.OwnerReferences = ownerReferences + clusterRole = &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-role", + }, + } + testutils.CmpClusterRole(ctx, *MemberCluster, clusterRole, expectedClusterRole) + + By("update cluster role in Hub cluster") + rules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + }, + } + updatedClusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRole.Name, + Labels: map[string]string{labelKey: labelValue, "fleet.azure.com/region": "us"}, + }, + Rules: rules, + } + Expect(HubCluster.KubeClient.Update(ctx, updatedClusterRole)).Should(Succeed(), "Failed to update cluster role %s in %s cluster", updatedClusterRole.Name, HubCluster.ClusterName) + + expectedClusterRole = &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-role", + Labels: updatedClusterRole.Labels, + OwnerReferences: ownerReferences, + }, + Rules: rules, + } + clusterRole = &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-role", + }, + } + + By("check if cluster role got updated in member cluster") + testutils.CmpClusterRole(ctx, *MemberCluster, clusterRole, expectedClusterRole) By("delete cluster role on hub cluster") - testutils.DeleteClusterRole(*HubCluster, cr) + Expect(HubCluster.KubeClient.Delete(ctx, clusterRole)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", clusterRole.Name, HubCluster.ClusterName) + Eventually(func() bool { + return apierrors.IsNotFound(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: clusterRole.Name}, clusterRole)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", clusterRole.Name, HubCluster.ClusterName) + + By("check if cluster role got deleted on member cluster") + Eventually(func() bool { + return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: clusterRole.Name}, clusterRole)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", clusterRole.Name, MemberCluster.ClusterName) + + By("delete cluster resource placement on hub cluster") + testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) + }) + + It("Apply CRP selecting namespace by label and check if namespace gets propagated with role, role binding, then update existing role", func() { + By("create the resources to be propagated") + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + Labels: map[string]string{labelKey: labelValue}, + }, + } + testutils.CreateNamespace(ctx, *HubCluster, namespace) + + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-reader", + Namespace: namespace.Name, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Verbs: []string{"get", "list", "watch"}, + Resources: []string{"pods"}, + }, + }, + } + Expect(HubCluster.KubeClient.Create(ctx, role)).Should(Succeed(), "Failed to create role %s in %s cluster", role.Name, HubCluster.ClusterName) + + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "read-pods", + Namespace: namespace.Name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "jane", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: role.Name, + APIGroup: "rbac.authorization.k8s.io", + }, + } + Expect(HubCluster.KubeClient.Create(ctx, roleBinding)).Should(Succeed(), "Failed to create role binding %s in %s cluster", roleBinding.Name, HubCluster.ClusterName) + + By("create the cluster resource placement in the hub cluster") + crp = &v1alpha1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{Name: "test-crp2"}, + Spec: v1alpha1.ClusterResourcePlacementSpec{ + ResourceSelectors: []v1alpha1.ClusterResourceSelector{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: namespace.Labels, + }, + }, + }, + }, + } + testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) + + By("check if work gets created for cluster resource placement") + testutils.WaitWork(ctx, *HubCluster, crp.Name, memberNamespace.Name) + + By("check if cluster resource placement status is updated") + crpStatus := v1alpha1.ClusterResourcePlacementStatus{ + Conditions: []metav1.Condition{ + { + Message: "Successfully scheduled resources for placement", + Reason: "ScheduleSucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled), + }, + { + Message: "Successfully applied resources to member clusters", + Reason: "ApplySucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), + }, + }, + SelectedResources: []v1alpha1.ResourceIdentifier{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "RoleBinding", + Name: roleBinding.Name, + Namespace: roleBinding.Namespace, + }, + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "Role", + Name: role.Name, + Namespace: role.Namespace, + }, + { + Version: "v1", + Kind: "Namespace", + Name: namespace.Name, + }, + }, + TargetClusters: []string{"kind-member-testing"}, + } + testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, crp, crpStatus, 3*testutils.PollTimeout) + + By("check if resources in namespace are propagated to member cluster") + Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, namespace)).Should(Succeed(), "Failed to get namespace %s in %s cluster", namespace.Name, MemberCluster.ClusterName) + Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, role)).Should(Succeed(), "Failed to get role %s in %s cluster", role.Name, MemberCluster.ClusterName) + Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: roleBinding.Name, Namespace: roleBinding.Namespace}, roleBinding), "Failed to get role binding %s in %s cluster", roleBinding.Name, MemberCluster.ClusterName) + + By("update role in Hub cluster") + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Verbs: []string{"get", "list", "watch", "update"}, + Resources: []string{"pods"}, + }, + } + updatedRole := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: role.Name, + Namespace: namespace.Name, + }, + Rules: rules, + } + Expect(HubCluster.KubeClient.Update(ctx, updatedRole)).Should(Succeed(), "Failed to update role %s in %s cluster", updatedRole.Name, HubCluster.ClusterName) + + role = &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: role.Name, + Namespace: namespace.Name, + }, + } + + expectedRole := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: role.Name, + Namespace: namespace.Name, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: workapi.GroupVersion.String(), + Kind: "AppliedWork", + Name: crp.Name, + BlockOwnerDeletion: pointer.Bool(false), + }, + }, + }, + Rules: rules, + } + + By("check if role got updated in member cluster") + testutils.CmpRole(ctx, *MemberCluster, role, expectedRole) + + By("delete namespace") + Expect(HubCluster.KubeClient.Delete(context.TODO(), namespace)).Should(Succeed(), "Failed to delete namespace %s in %s cluster", namespace.Name, HubCluster.ClusterName) + Eventually(func() bool { + return apierrors.IsNotFound(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, namespace)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for namespace %s to be deleted in %s cluster", namespace.Name, HubCluster.ClusterName) + + By("check if namespace got deleted on member cluster") + Eventually(func() bool { + return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, namespace)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", namespace.Name, MemberCluster.ClusterName) + + By("delete cluster resource placement on hub cluster") + testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) }) }) }) From ca7c795665b4b0ac266e95a5b69a1ee685220772 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Fri, 23 Sep 2022 11:51:55 -0700 Subject: [PATCH 02/20] Dont create member cluster namespace --- test/e2e/e2e_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index ec9491e43..fcea60fd5 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -167,8 +167,6 @@ var _ = BeforeSuite(func() { ctx = context.Background() - testutils.CreateNamespace(ctx, *HubCluster, workNamespace) - By("deploy member cluster in the hub cluster") identity := rbacv1.Subject{ Name: "member-agent-sa", From 5690b837c82725a81e8b497d2a2b6b7fae981222 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Wed, 28 Sep 2022 14:24:10 +0530 Subject: [PATCH 03/20] Status check setup helper methods --- test/e2e/e2e_test.go | 40 ++++------------------------------------ test/e2e/utils/helper.go | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index fcea60fd5..d58bcb5cd 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -198,15 +198,7 @@ var _ = BeforeSuite(func() { By("check if internal member cluster status is updated to Joined") wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcJoinedAgentStatus} - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) - } - return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) + testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, wantIMCStatus, imc, imcStatusCmpOptions) By("check if member cluster status is updated to Joined") wantMCStatus := v1alpha1.MemberClusterStatus{ @@ -214,15 +206,7 @@ var _ = BeforeSuite(func() { Conditions: mcJoinedConditions, ResourceUsage: imc.Status.ResourceUsage, } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) }) var _ = AfterSuite(func() { @@ -233,15 +217,7 @@ var _ = AfterSuite(func() { By("check if internal member cluster status is updated to Left") wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcLeftAgentStatus} - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) - } - return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) + testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, wantIMCStatus, imc, imcStatusCmpOptions) By("check if member cluster status is updated to Left") wantMCStatus := v1alpha1.MemberClusterStatus{ @@ -249,15 +225,7 @@ var _ = AfterSuite(func() { Conditions: mcLeftConditions, ResourceUsage: imc.Status.ResourceUsage, } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) By("delete member cluster") testutils.DeleteMemberCluster(ctx, *HubCluster, mc) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index 321f0579f..296e87ecd 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -10,6 +10,7 @@ import ( "fmt" "time" + "github.com/google/go-cmp/cmp" // Lint check prohibits non "_test" ending files to have dot imports for ginkgo / gomega. "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -46,6 +47,32 @@ func DeleteMemberCluster(ctx context.Context, cluster framework.Cluster, mc *v1a }, PollTimeout, PollInterval).Should(gomega.BeTrue(), "Failed to wait for member cluster %s to be deleted in %s cluster", mc.Name, cluster.ClusterName) } +// CheckMemberClusterStatus is used to check member cluster status. +func CheckMemberClusterStatus(ctx context.Context, cluster framework.Cluster, wantMCStatus v1alpha1.MemberClusterStatus, mc *v1alpha1.MemberCluster, mcStatusCmpOptions []cmp.Option) { + gomega.Eventually(func() error { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + } + return nil + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait member cluster %s to have status %s", mc.Name, wantMCStatus) +} + +// CheckInternalMemberClusterStatus is used to check internal member cluster status. +func CheckInternalMemberClusterStatus(ctx context.Context, cluster framework.Cluster, wantIMCStatus v1alpha1.InternalMemberClusterStatus, imc *v1alpha1.InternalMemberCluster, imcStatusCmpOptions []cmp.Option) { + gomega.Eventually(func() error { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { + return err + } + if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) + } + return nil + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) +} + // CreateClusterRole create cluster role in the hub cluster. func CreateClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { ginkgo.By(fmt.Sprintf("Creating ClusterRole (%s)", cr.Name), func() { From 9eb698cbf401c101599f430c6e485a9347c178f7 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 29 Sep 2022 00:55:24 +0530 Subject: [PATCH 04/20] Cmp namespace, role, rolebinding --- test/e2e/utils/workload_test_utils.go | 35 ++++++++++++++- test/e2e/work_load_test.go | 61 +++++++++++++++------------ 2 files changed, 68 insertions(+), 28 deletions(-) diff --git a/test/e2e/utils/workload_test_utils.go b/test/e2e/utils/workload_test_utils.go index d9c6f20dd..c46ada06c 100644 --- a/test/e2e/utils/workload_test_utils.go +++ b/test/e2e/utils/workload_test_utils.go @@ -15,6 +15,7 @@ import ( "github.com/onsi/gomega" "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/test/e2e/framework" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,6 +37,21 @@ func CmpClusterRole(ctx context.Context, cluster framework.Cluster, actualCluste }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for cluster role %s to be updated in %s cluster", actualClusterRole.Name, cluster.ClusterName) } +// CmpNamespace compares actual namespace with expected namespace. +func CmpNamespace(ctx context.Context, cluster framework.Cluster, actualNamespace, expectedNamespace *corev1.Namespace) { + gomega.Eventually(func() error { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualNamespace.Name}, actualNamespace); err != nil { + return err + } + ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} + if diff := cmp.Diff(expectedNamespace, actualNamespace, ignoreOptions...); diff != "" { + return fmt.Errorf(" namespace(%s) mismatch (-want +got):\n%s", actualNamespace.Name, diff) + } + return nil + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to compare actual and expected namespaces in %s cluster", cluster.ClusterName) +} + // CmpRole compares actual role with expected role. func CmpRole(ctx context.Context, cluster framework.Cluster, actualRole, expectedRole *rbacv1.Role) { gomega.Eventually(func() error { @@ -45,10 +61,25 @@ func CmpRole(ctx context.Context, cluster framework.Cluster, actualRole, expecte ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} if diff := cmp.Diff(expectedRole, actualRole, ignoreOptions...); diff != "" { - return fmt.Errorf("cluster Role(%s) mismatch (-want +got):\n%s", actualRole.Name, diff) + return fmt.Errorf("Role(%s) mismatch (-want +got):\n%s", actualRole.Name, diff) + } + return nil + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to compare actual and expected roles in %s cluster", cluster.ClusterName) +} + +// CmpRoleBinding compares actual role binding with expected role binding. +func CmpRoleBinding(ctx context.Context, cluster framework.Cluster, actualRoleBinding, expectedRoleBinding *rbacv1.RoleBinding) { + gomega.Eventually(func() error { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualRoleBinding.Name, Namespace: actualRoleBinding.Namespace}, actualRoleBinding); err != nil { + return err + } + ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} + if diff := cmp.Diff(expectedRoleBinding, actualRoleBinding, ignoreOptions...); diff != "" { + return fmt.Errorf("Role Binding(%s) mismatch (-want +got):\n%s", actualRoleBinding.Name, diff) } return nil - }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for role %s to be updated in %s cluster", actualRole.Name, cluster.ClusterName) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to compare actual and expected role bindings in %s cluster", cluster.ClusterName) } // CreateClusterResourcePlacement created ClusterResourcePlacement and waits for ClusterResourcePlacement to exist in hub cluster. diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index da29ff0ea..1bafbe6bd 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -113,7 +113,7 @@ var _ = Describe("workload orchestration testing", func() { expectedClusterRole.OwnerReferences = ownerReferences clusterRole = &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster-role", + Name: clusterRole.Name, }, } testutils.CmpClusterRole(ctx, *MemberCluster, clusterRole, expectedClusterRole) @@ -275,49 +275,58 @@ var _ = Describe("workload orchestration testing", func() { testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, crp, crpStatus, 3*testutils.PollTimeout) By("check if resources in namespace are propagated to member cluster") - Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, namespace)).Should(Succeed(), "Failed to get namespace %s in %s cluster", namespace.Name, MemberCluster.ClusterName) - Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, role)).Should(Succeed(), "Failed to get role %s in %s cluster", role.Name, MemberCluster.ClusterName) - Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: roleBinding.Name, Namespace: roleBinding.Namespace}, roleBinding), "Failed to get role binding %s in %s cluster", roleBinding.Name, MemberCluster.ClusterName) - - By("update role in Hub cluster") - rules := []rbacv1.PolicyRule{ + ownerReferences := []metav1.OwnerReference{ { - APIGroups: []string{""}, - Verbs: []string{"get", "list", "watch", "update"}, - Resources: []string{"pods"}, + APIVersion: workapi.GroupVersion.String(), + BlockOwnerDeletion: pointer.Bool(false), + Kind: "AppliedWork", + Name: crp.Name, }, } - updatedRole := &rbacv1.Role{ + expectedNamespace := namespace + expectedRole := role + expectedRoleBinding := roleBinding + expectedNamespace.OwnerReferences = ownerReferences + expectedRole.OwnerReferences = ownerReferences + expectedRoleBinding.OwnerReferences = ownerReferences + namespace = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: role.Name, - Namespace: namespace.Name, + Name: namespace.Name, }, - Rules: rules, } - Expect(HubCluster.KubeClient.Update(ctx, updatedRole)).Should(Succeed(), "Failed to update role %s in %s cluster", updatedRole.Name, HubCluster.ClusterName) - role = &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: role.Name, - Namespace: namespace.Name, + Namespace: role.Namespace, }, } + roleBinding = &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: roleBinding.Name, + Namespace: roleBinding.Namespace, + }, + } + testutils.CmpNamespace(ctx, *MemberCluster, namespace, expectedNamespace) + testutils.CmpRole(ctx, *MemberCluster, role, expectedRole) + testutils.CmpRoleBinding(ctx, *MemberCluster, roleBinding, expectedRoleBinding) - expectedRole := &rbacv1.Role{ + By("update role in Hub cluster") + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Verbs: []string{"get", "list", "watch", "update"}, + Resources: []string{"pods"}, + }, + } + updatedRole := &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: role.Name, Namespace: namespace.Name, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: workapi.GroupVersion.String(), - Kind: "AppliedWork", - Name: crp.Name, - BlockOwnerDeletion: pointer.Bool(false), - }, - }, }, Rules: rules, } + Expect(HubCluster.KubeClient.Update(ctx, updatedRole)).Should(Succeed(), "Failed to update role %s in %s cluster", updatedRole.Name, HubCluster.ClusterName) + expectedRole.Rules = rules By("check if role got updated in member cluster") testutils.CmpRole(ctx, *MemberCluster, role, expectedRole) From 78ee2cb177ed58add1162b857a71c8af891598af Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 29 Sep 2022 09:06:34 +0530 Subject: [PATCH 05/20] Use helper methods for joinLeavePlacement --- test/e2e/join_leave_placement_test.go | 43 +++------------------------ test/e2e/utils/workload_test_utils.go | 6 ++-- 2 files changed, 7 insertions(+), 42 deletions(-) diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go index d43ad338e..3bc997a3a 100644 --- a/test/e2e/join_leave_placement_test.go +++ b/test/e2e/join_leave_placement_test.go @@ -6,8 +6,6 @@ package e2e import ( "context" - "fmt" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" @@ -51,15 +49,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcLeftAgentStatus, Conditions: mcLeftConditions, } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) By("create the resources to be propagated") cr := &rbacv1.ClusterRole{ @@ -112,18 +102,9 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcJoinedAgentStatus, Conditions: mcJoinedConditions, } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) By("verify that the cluster resource placement is applied") - //testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout) crpStatus := v1alpha1.ClusterResourcePlacementStatus{ Conditions: []metav1.Condition{ { @@ -164,15 +145,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcLeftAgentStatus, Conditions: mcLeftConditions, } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) By("verify that the resource is still on the member cluster") Consistently(func() error { @@ -200,14 +173,6 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcJoinedAgentStatus, Conditions: mcJoinedConditions, } - Eventually(func() error { - if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { - return err - } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) - } - return nil - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) }) }) diff --git a/test/e2e/utils/workload_test_utils.go b/test/e2e/utils/workload_test_utils.go index c46ada06c..79649854c 100644 --- a/test/e2e/utils/workload_test_utils.go +++ b/test/e2e/utils/workload_test_utils.go @@ -31,7 +31,7 @@ func CmpClusterRole(ctx context.Context, cluster framework.Cluster, actualCluste ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} if diff := cmp.Diff(expectedClusterRole, actualClusterRole, ignoreOptions...); diff != "" { - return fmt.Errorf("cluster Role(%s) mismatch (-want +got):\n%s", actualClusterRole.Name, diff) + return fmt.Errorf("cluster role(%s) mismatch (-want +got):\n%s", actualClusterRole.Name, diff) } return nil }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for cluster role %s to be updated in %s cluster", actualClusterRole.Name, cluster.ClusterName) @@ -61,7 +61,7 @@ func CmpRole(ctx context.Context, cluster framework.Cluster, actualRole, expecte ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} if diff := cmp.Diff(expectedRole, actualRole, ignoreOptions...); diff != "" { - return fmt.Errorf("Role(%s) mismatch (-want +got):\n%s", actualRole.Name, diff) + return fmt.Errorf("role(%s) mismatch (-want +got):\n%s", actualRole.Name, diff) } return nil }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to compare actual and expected roles in %s cluster", cluster.ClusterName) @@ -76,7 +76,7 @@ func CmpRoleBinding(ctx context.Context, cluster framework.Cluster, actualRoleBi ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} if diff := cmp.Diff(expectedRoleBinding, actualRoleBinding, ignoreOptions...); diff != "" { - return fmt.Errorf("Role Binding(%s) mismatch (-want +got):\n%s", actualRoleBinding.Name, diff) + return fmt.Errorf("role binding(%s) mismatch (-want +got):\n%s", actualRoleBinding.Name, diff) } return nil }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to compare actual and expected role bindings in %s cluster", cluster.ClusterName) From d6af88e1c3d3e1fac80eaba1c7597f1fe66c68b7 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 29 Sep 2022 09:08:26 +0530 Subject: [PATCH 06/20] Fix imports --- test/e2e/join_leave_placement_test.go | 1 + test/e2e/utils/workload_test_utils.go | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go index 3bc997a3a..2aab7fa5d 100644 --- a/test/e2e/join_leave_placement_test.go +++ b/test/e2e/join_leave_placement_test.go @@ -6,6 +6,7 @@ package e2e import ( "context" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" diff --git a/test/e2e/utils/workload_test_utils.go b/test/e2e/utils/workload_test_utils.go index 79649854c..72de96820 100644 --- a/test/e2e/utils/workload_test_utils.go +++ b/test/e2e/utils/workload_test_utils.go @@ -13,13 +13,14 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/onsi/gomega" - "go.goms.io/fleet/apis/v1alpha1" - "go.goms.io/fleet/test/e2e/framework" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + + "go.goms.io/fleet/apis/v1alpha1" + "go.goms.io/fleet/test/e2e/framework" ) // CmpClusterRole compares actual cluster role with expected cluster role. From 3189988bf8b38f3dc8766b18e74fa7d9d2221c81 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 29 Sep 2022 09:32:01 +0530 Subject: [PATCH 07/20] Add resourceIgnoreOption --- test/e2e/utils/workload_test_utils.go | 24 ++++++++-------------- test/e2e/work_load_test.go | 29 ++++++++++++++------------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/test/e2e/utils/workload_test_utils.go b/test/e2e/utils/workload_test_utils.go index 72de96820..30bcbbb56 100644 --- a/test/e2e/utils/workload_test_utils.go +++ b/test/e2e/utils/workload_test_utils.go @@ -24,14 +24,12 @@ import ( ) // CmpClusterRole compares actual cluster role with expected cluster role. -func CmpClusterRole(ctx context.Context, cluster framework.Cluster, actualClusterRole, expectedClusterRole *rbacv1.ClusterRole) { +func CmpClusterRole(ctx context.Context, cluster framework.Cluster, actualClusterRole, expectedClusterRole *rbacv1.ClusterRole, cmpOptions []cmp.Option) { gomega.Eventually(func() error { if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualClusterRole.Name}, actualClusterRole); err != nil { return err } - ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), - cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} - if diff := cmp.Diff(expectedClusterRole, actualClusterRole, ignoreOptions...); diff != "" { + if diff := cmp.Diff(expectedClusterRole, actualClusterRole, cmpOptions...); diff != "" { return fmt.Errorf("cluster role(%s) mismatch (-want +got):\n%s", actualClusterRole.Name, diff) } return nil @@ -39,14 +37,12 @@ func CmpClusterRole(ctx context.Context, cluster framework.Cluster, actualCluste } // CmpNamespace compares actual namespace with expected namespace. -func CmpNamespace(ctx context.Context, cluster framework.Cluster, actualNamespace, expectedNamespace *corev1.Namespace) { +func CmpNamespace(ctx context.Context, cluster framework.Cluster, actualNamespace, expectedNamespace *corev1.Namespace, cmpOptions []cmp.Option) { gomega.Eventually(func() error { if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualNamespace.Name}, actualNamespace); err != nil { return err } - ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), - cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} - if diff := cmp.Diff(expectedNamespace, actualNamespace, ignoreOptions...); diff != "" { + if diff := cmp.Diff(expectedNamespace, actualNamespace, cmpOptions...); diff != "" { return fmt.Errorf(" namespace(%s) mismatch (-want +got):\n%s", actualNamespace.Name, diff) } return nil @@ -54,14 +50,12 @@ func CmpNamespace(ctx context.Context, cluster framework.Cluster, actualNamespac } // CmpRole compares actual role with expected role. -func CmpRole(ctx context.Context, cluster framework.Cluster, actualRole, expectedRole *rbacv1.Role) { +func CmpRole(ctx context.Context, cluster framework.Cluster, actualRole, expectedRole *rbacv1.Role, cmpOptions []cmp.Option) { gomega.Eventually(func() error { if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualRole.Name, Namespace: actualRole.Namespace}, actualRole); err != nil { return err } - ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), - cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} - if diff := cmp.Diff(expectedRole, actualRole, ignoreOptions...); diff != "" { + if diff := cmp.Diff(expectedRole, actualRole, cmpOptions...); diff != "" { return fmt.Errorf("role(%s) mismatch (-want +got):\n%s", actualRole.Name, diff) } return nil @@ -69,14 +63,12 @@ func CmpRole(ctx context.Context, cluster framework.Cluster, actualRole, expecte } // CmpRoleBinding compares actual role binding with expected role binding. -func CmpRoleBinding(ctx context.Context, cluster framework.Cluster, actualRoleBinding, expectedRoleBinding *rbacv1.RoleBinding) { +func CmpRoleBinding(ctx context.Context, cluster framework.Cluster, actualRoleBinding, expectedRoleBinding *rbacv1.RoleBinding, cmpOptions []cmp.Option) { gomega.Eventually(func() error { if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualRoleBinding.Name, Namespace: actualRoleBinding.Namespace}, actualRoleBinding); err != nil { return err } - ignoreOptions := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), - cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} - if diff := cmp.Diff(expectedRoleBinding, actualRoleBinding, ignoreOptions...); diff != "" { + if diff := cmp.Diff(expectedRoleBinding, actualRoleBinding, cmpOptions...); diff != "" { return fmt.Errorf("role binding(%s) mismatch (-want +got):\n%s", actualRoleBinding.Name, diff) } return nil diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index 1bafbe6bd..de6237e58 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -8,6 +8,8 @@ package e2e import ( "context" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -23,14 +25,13 @@ import ( ) var _ = Describe("workload orchestration testing", func() { - var crp *v1alpha1.ClusterResourcePlacement - var ctx context.Context - labelKey := "fleet.azure.com/name" - labelValue := "test" - - BeforeEach(func() { - ctx = context.Background() - }) + var ( + crp *v1alpha1.ClusterResourcePlacement + labelKey = "fleet.azure.com/name" + labelValue = "test" + resourceIgnoreOptions = []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "Annotations", "CreationTimestamp", "ManagedFields"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID")} + ) Context("Test Workload Orchestration", func() { It("Apply CRP and check if cluster role gets propagated, update cluster role", func() { @@ -116,7 +117,7 @@ var _ = Describe("workload orchestration testing", func() { Name: clusterRole.Name, }, } - testutils.CmpClusterRole(ctx, *MemberCluster, clusterRole, expectedClusterRole) + testutils.CmpClusterRole(ctx, *MemberCluster, clusterRole, expectedClusterRole, resourceIgnoreOptions) By("update cluster role in Hub cluster") rules := []rbacv1.PolicyRule{ @@ -150,7 +151,7 @@ var _ = Describe("workload orchestration testing", func() { } By("check if cluster role got updated in member cluster") - testutils.CmpClusterRole(ctx, *MemberCluster, clusterRole, expectedClusterRole) + testutils.CmpClusterRole(ctx, *MemberCluster, clusterRole, expectedClusterRole, resourceIgnoreOptions) By("delete cluster role on hub cluster") Expect(HubCluster.KubeClient.Delete(ctx, clusterRole)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", clusterRole.Name, HubCluster.ClusterName) @@ -306,9 +307,9 @@ var _ = Describe("workload orchestration testing", func() { Namespace: roleBinding.Namespace, }, } - testutils.CmpNamespace(ctx, *MemberCluster, namespace, expectedNamespace) - testutils.CmpRole(ctx, *MemberCluster, role, expectedRole) - testutils.CmpRoleBinding(ctx, *MemberCluster, roleBinding, expectedRoleBinding) + testutils.CmpNamespace(ctx, *MemberCluster, namespace, expectedNamespace, resourceIgnoreOptions) + testutils.CmpRole(ctx, *MemberCluster, role, expectedRole, resourceIgnoreOptions) + testutils.CmpRoleBinding(ctx, *MemberCluster, roleBinding, expectedRoleBinding, resourceIgnoreOptions) By("update role in Hub cluster") rules := []rbacv1.PolicyRule{ @@ -329,7 +330,7 @@ var _ = Describe("workload orchestration testing", func() { expectedRole.Rules = rules By("check if role got updated in member cluster") - testutils.CmpRole(ctx, *MemberCluster, role, expectedRole) + testutils.CmpRole(ctx, *MemberCluster, role, expectedRole, resourceIgnoreOptions) By("delete namespace") Expect(HubCluster.KubeClient.Delete(context.TODO(), namespace)).Should(Succeed(), "Failed to delete namespace %s in %s cluster", namespace.Name, HubCluster.ClusterName) From cf9508de5e229074b46efd4043fa88fcc79aae27 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 29 Sep 2022 10:00:17 +0530 Subject: [PATCH 08/20] Removed some helper methods --- test/e2e/join_leave_placement_test.go | 4 ++-- test/e2e/utils/helper.go | 19 ------------------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go index 2aab7fa5d..ab42ce51d 100644 --- a/test/e2e/join_leave_placement_test.go +++ b/test/e2e/join_leave_placement_test.go @@ -66,7 +66,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( }, }, } - testutils.CreateClusterRole(*HubCluster, cr) + Expect(HubCluster.KubeClient.Create(ctx, cr)).Should(Succeed(), "Failed to create cluster role %s in %s cluster", cr.Name, HubCluster.ClusterName) By("create the cluster resource placement in the hub cluster") crp = &v1alpha1.ClusterResourcePlacement{ @@ -162,7 +162,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) By("delete cluster role on hub cluster") - testutils.DeleteClusterRole(*HubCluster, cr) + Expect(HubCluster.KubeClient.Delete(ctx, cr)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", cr.Name, HubCluster.ClusterName) By("update member cluster in the hub cluster to join") Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index 296e87ecd..aba979750 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -11,11 +11,8 @@ import ( "time" "github.com/google/go-cmp/cmp" - // Lint check prohibits non "_test" ending files to have dot imports for ginkgo / gomega. - "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -73,22 +70,6 @@ func CheckInternalMemberClusterStatus(ctx context.Context, cluster framework.Clu }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) } -// CreateClusterRole create cluster role in the hub cluster. -func CreateClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { - ginkgo.By(fmt.Sprintf("Creating ClusterRole (%s)", cr.Name), func() { - err := cluster.KubeClient.Create(context.TODO(), cr) - gomega.Expect(err).Should(gomega.Succeed()) - }) -} - -// DeleteClusterRole deletes cluster role on cluster. -func DeleteClusterRole(cluster framework.Cluster, cr *rbacv1.ClusterRole) { - ginkgo.By(fmt.Sprintf("Deleting ClusterRole(%s)", cr.Name), func() { - err := cluster.KubeClient.Delete(context.TODO(), cr) - gomega.Expect(err).Should(gomega.Succeed()) - }) -} - // WaitWork waits for Work to be present on the hub cluster. func WaitWork(ctx context.Context, cluster framework.Cluster, workName, workNamespace string) { name := types.NamespacedName{Name: workName, Namespace: workNamespace} From 6b0e607ebe02c3e523b89c8e25c5c06f6921b575 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 29 Sep 2022 10:04:18 +0530 Subject: [PATCH 09/20] Add comment for gomega/ginkgo --- test/e2e/utils/helper.go | 1 + test/e2e/utils/workload_test_utils.go | 1 + 2 files changed, 2 insertions(+) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index aba979750..9f13a929f 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -11,6 +11,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + // Lint check prohibits non "_test" ending files to have dot imports for ginkgo / gomega. "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/e2e/utils/workload_test_utils.go b/test/e2e/utils/workload_test_utils.go index 30bcbbb56..4d57924f0 100644 --- a/test/e2e/utils/workload_test_utils.go +++ b/test/e2e/utils/workload_test_utils.go @@ -12,6 +12,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + // Lint check prohibits non "_test" ending files to have dot imports for ginkgo / gomega. "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" From 659da70a8d84d7d379034d0d4dc63ba471347172 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 29 Sep 2022 10:18:57 +0530 Subject: [PATCH 10/20] Lint fix --- test/e2e/utils/helper.go | 2 +- test/e2e/utils/workload_test_utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index 9f13a929f..4abd8d342 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -10,8 +10,8 @@ import ( "fmt" "time" - "github.com/google/go-cmp/cmp" // Lint check prohibits non "_test" ending files to have dot imports for ginkgo / gomega. + "github.com/google/go-cmp/cmp" "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/test/e2e/utils/workload_test_utils.go b/test/e2e/utils/workload_test_utils.go index 4d57924f0..71a9d8b75 100644 --- a/test/e2e/utils/workload_test_utils.go +++ b/test/e2e/utils/workload_test_utils.go @@ -10,9 +10,9 @@ import ( "fmt" "time" + // Lint check prohibits non "_test" ending files to have dot imports for ginkgo / gomega. "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - // Lint check prohibits non "_test" ending files to have dot imports for ginkgo / gomega. "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" From 110a5f4dbf3ecdba77ce7e97e64fd36101fb0b69 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Fri, 30 Sep 2022 12:54:48 +0530 Subject: [PATCH 11/20] Address comments 1 --- test/e2e/e2e_test.go | 15 +++++--- test/e2e/join_leave_placement_test.go | 10 ++--- test/e2e/utils/helper.go | 26 +++++++------ test/e2e/utils/workload_test_utils.go | 53 ++++++++++++++------------- test/e2e/work_load_test.go | 51 ++++++-------------------- 5 files changed, 67 insertions(+), 88 deletions(-) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d58bcb5cd..adfe96511 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -64,13 +64,16 @@ var ( cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), sortOption, } - mcStatusCmpOptions = []cmp.Option{ cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"), sortOption, } + crpStatusCmpOptions = []cmp.Option{ + cmpopts.IgnoreFields(metav1.Condition{}, "ObservedGeneration", "LastTransitionTime"), + sortOption, + } imcJoinedAgentStatus = []v1alpha1.AgentStatus{ { @@ -198,15 +201,16 @@ var _ = BeforeSuite(func() { By("check if internal member cluster status is updated to Joined") wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcJoinedAgentStatus} - testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, wantIMCStatus, imc, imcStatusCmpOptions) + testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, wantIMCStatus, imcStatusCmpOptions) By("check if member cluster status is updated to Joined") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc)).Should(Succeed(), "Failed to retrieve internal member cluster %s in %s cluster", imc.Name, HubCluster.ClusterName) wantMCStatus := v1alpha1.MemberClusterStatus{ AgentStatus: imc.Status.AgentStatus, Conditions: mcJoinedConditions, ResourceUsage: imc.Status.ResourceUsage, } - testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) }) var _ = AfterSuite(func() { @@ -217,15 +221,16 @@ var _ = AfterSuite(func() { By("check if internal member cluster status is updated to Left") wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcLeftAgentStatus} - testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, wantIMCStatus, imc, imcStatusCmpOptions) + testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, wantIMCStatus, imcStatusCmpOptions) By("check if member cluster status is updated to Left") + Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc)).Should(Succeed(), "Failed to retrieve internal member cluster %s in %s cluster", imc.Name, HubCluster.ClusterName) wantMCStatus := v1alpha1.MemberClusterStatus{ AgentStatus: imc.Status.AgentStatus, Conditions: mcLeftConditions, ResourceUsage: imc.Status.ResourceUsage, } - testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) By("delete member cluster") testutils.DeleteMemberCluster(ctx, *HubCluster, mc) diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go index ab42ce51d..5543c3966 100644 --- a/test/e2e/join_leave_placement_test.go +++ b/test/e2e/join_leave_placement_test.go @@ -50,7 +50,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcLeftAgentStatus, Conditions: mcLeftConditions, } - testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) By("create the resources to be propagated") cr := &rbacv1.ClusterRole{ @@ -103,7 +103,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcJoinedAgentStatus, Conditions: mcJoinedConditions, } - testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) By("verify that the cluster resource placement is applied") crpStatus := v1alpha1.ClusterResourcePlacementStatus{ @@ -131,7 +131,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( }, TargetClusters: []string{"kind-member-testing"}, } - testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, crp, crpStatus, 3*testutils.PollTimeout) + testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*testutils.PollTimeout) By("verify the resource is propagated to member cluster") Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName) @@ -146,7 +146,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcLeftAgentStatus, Conditions: mcLeftConditions, } - testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) By("verify that the resource is still on the member cluster") Consistently(func() error { @@ -174,6 +174,6 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcJoinedAgentStatus, Conditions: mcJoinedConditions, } - testutils.CheckMemberClusterStatus(ctx, *HubCluster, wantMCStatus, mc, mcStatusCmpOptions) + testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) }) }) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index 4abd8d342..a1c7565cd 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -25,7 +25,7 @@ import ( "k8s.io/klog/v2" workapi "sigs.k8s.io/work-api/pkg/apis/v1alpha1" - "go.goms.io/fleet/apis/v1alpha1" + fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/pkg/utils" "go.goms.io/fleet/test/e2e/framework" ) @@ -38,7 +38,7 @@ var ( ) // DeleteMemberCluster deletes MemberCluster in the hub cluster. -func DeleteMemberCluster(ctx context.Context, cluster framework.Cluster, mc *v1alpha1.MemberCluster) { +func DeleteMemberCluster(ctx context.Context, cluster framework.Cluster, mc *fleetv1alpha1.MemberCluster) { gomega.Expect(cluster.KubeClient.Delete(ctx, mc)).Should(gomega.Succeed(), "Failed to delete member cluster %s in %s cluster", mc.Name, cluster.ClusterName) gomega.Eventually(func() bool { return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)) @@ -46,29 +46,31 @@ func DeleteMemberCluster(ctx context.Context, cluster framework.Cluster, mc *v1a } // CheckMemberClusterStatus is used to check member cluster status. -func CheckMemberClusterStatus(ctx context.Context, cluster framework.Cluster, wantMCStatus v1alpha1.MemberClusterStatus, mc *v1alpha1.MemberCluster, mcStatusCmpOptions []cmp.Option) { +func CheckMemberClusterStatus(ctx context.Context, cluster framework.Cluster, objectKey *types.NamespacedName, wantMCStatus fleetv1alpha1.MemberClusterStatus, mcStatusCmpOptions []cmp.Option) { + gotMC := &fleetv1alpha1.MemberCluster{} gomega.Eventually(func() error { - if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: objectKey.Name}, gotMC); err != nil { return err } - if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff) + if statusDiff := cmp.Diff(wantMCStatus, gotMC.Status, mcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", gotMC.Name, statusDiff) } return nil - }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait member cluster %s to have status %s", mc.Name, wantMCStatus) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait member cluster %s to have status %s", gotMC.Name, wantMCStatus) } // CheckInternalMemberClusterStatus is used to check internal member cluster status. -func CheckInternalMemberClusterStatus(ctx context.Context, cluster framework.Cluster, wantIMCStatus v1alpha1.InternalMemberClusterStatus, imc *v1alpha1.InternalMemberCluster, imcStatusCmpOptions []cmp.Option) { +func CheckInternalMemberClusterStatus(ctx context.Context, cluster framework.Cluster, objectKey *types.NamespacedName, wantIMCStatus fleetv1alpha1.InternalMemberClusterStatus, imcStatusCmpOptions []cmp.Option) { + gotIMC := &fleetv1alpha1.InternalMemberCluster{} gomega.Eventually(func() error { - if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: objectKey.Name, Namespace: objectKey.Namespace}, gotIMC); err != nil { return err } - if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" { - return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff) + if statusDiff := cmp.Diff(wantIMCStatus, gotIMC.Status, imcStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", gotIMC.Name, statusDiff) } return nil - }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for internal member cluster %s to have status %s", gotIMC.Name, wantIMCStatus) } // WaitWork waits for Work to be present on the hub cluster. diff --git a/test/e2e/utils/workload_test_utils.go b/test/e2e/utils/workload_test_utils.go index 71a9d8b75..ae1bd19c0 100644 --- a/test/e2e/utils/workload_test_utils.go +++ b/test/e2e/utils/workload_test_utils.go @@ -12,12 +12,10 @@ import ( // Lint check prohibits non "_test" ending files to have dot imports for ginkgo / gomega. "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "go.goms.io/fleet/apis/v1alpha1" @@ -25,52 +23,56 @@ import ( ) // CmpClusterRole compares actual cluster role with expected cluster role. -func CmpClusterRole(ctx context.Context, cluster framework.Cluster, actualClusterRole, expectedClusterRole *rbacv1.ClusterRole, cmpOptions []cmp.Option) { +func CmpClusterRole(ctx context.Context, cluster framework.Cluster, objectKey *types.NamespacedName, wantClusterRole *rbacv1.ClusterRole, cmpOptions []cmp.Option) { + gotClusterRole := &rbacv1.ClusterRole{} gomega.Eventually(func() error { - if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualClusterRole.Name}, actualClusterRole); err != nil { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: objectKey.Name}, gotClusterRole); err != nil { return err } - if diff := cmp.Diff(expectedClusterRole, actualClusterRole, cmpOptions...); diff != "" { - return fmt.Errorf("cluster role(%s) mismatch (-want +got):\n%s", actualClusterRole.Name, diff) + if diff := cmp.Diff(wantClusterRole, gotClusterRole, cmpOptions...); diff != "" { + return fmt.Errorf("cluster role(%s) mismatch (-want +got):\n%s", gotClusterRole.Name, diff) } return nil - }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for cluster role %s to be updated in %s cluster", actualClusterRole.Name, cluster.ClusterName) + }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to compare actual and expected cluster roles in %s cluster", cluster.ClusterName) } // CmpNamespace compares actual namespace with expected namespace. -func CmpNamespace(ctx context.Context, cluster framework.Cluster, actualNamespace, expectedNamespace *corev1.Namespace, cmpOptions []cmp.Option) { +func CmpNamespace(ctx context.Context, cluster framework.Cluster, objectKey *types.NamespacedName, wantNamespace *corev1.Namespace, cmpOptions []cmp.Option) { + gotNamespace := &corev1.Namespace{} gomega.Eventually(func() error { - if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualNamespace.Name}, actualNamespace); err != nil { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: objectKey.Name}, gotNamespace); err != nil { return err } - if diff := cmp.Diff(expectedNamespace, actualNamespace, cmpOptions...); diff != "" { - return fmt.Errorf(" namespace(%s) mismatch (-want +got):\n%s", actualNamespace.Name, diff) + if diff := cmp.Diff(wantNamespace, gotNamespace, cmpOptions...); diff != "" { + return fmt.Errorf(" namespace(%s) mismatch (-want +got):\n%s", gotNamespace.Name, diff) } return nil }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to compare actual and expected namespaces in %s cluster", cluster.ClusterName) } // CmpRole compares actual role with expected role. -func CmpRole(ctx context.Context, cluster framework.Cluster, actualRole, expectedRole *rbacv1.Role, cmpOptions []cmp.Option) { +func CmpRole(ctx context.Context, cluster framework.Cluster, objectKey *types.NamespacedName, wantRole *rbacv1.Role, cmpOptions []cmp.Option) { + gotRole := &rbacv1.Role{} gomega.Eventually(func() error { - if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualRole.Name, Namespace: actualRole.Namespace}, actualRole); err != nil { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: objectKey.Name, Namespace: objectKey.Namespace}, gotRole); err != nil { return err } - if diff := cmp.Diff(expectedRole, actualRole, cmpOptions...); diff != "" { - return fmt.Errorf("role(%s) mismatch (-want +got):\n%s", actualRole.Name, diff) + if diff := cmp.Diff(wantRole, gotRole, cmpOptions...); diff != "" { + return fmt.Errorf("role(%s) mismatch (-want +got):\n%s", gotRole.Name, diff) } return nil }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to compare actual and expected roles in %s cluster", cluster.ClusterName) } // CmpRoleBinding compares actual role binding with expected role binding. -func CmpRoleBinding(ctx context.Context, cluster framework.Cluster, actualRoleBinding, expectedRoleBinding *rbacv1.RoleBinding, cmpOptions []cmp.Option) { +func CmpRoleBinding(ctx context.Context, cluster framework.Cluster, objectKey *types.NamespacedName, wantRoleBinding *rbacv1.RoleBinding, cmpOptions []cmp.Option) { + gotRoleBinding := &rbacv1.RoleBinding{} gomega.Eventually(func() error { - if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: actualRoleBinding.Name, Namespace: actualRoleBinding.Namespace}, actualRoleBinding); err != nil { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: objectKey.Name, Namespace: objectKey.Namespace}, gotRoleBinding); err != nil { return err } - if diff := cmp.Diff(expectedRoleBinding, actualRoleBinding, cmpOptions...); diff != "" { - return fmt.Errorf("role binding(%s) mismatch (-want +got):\n%s", actualRoleBinding.Name, diff) + if diff := cmp.Diff(wantRoleBinding, gotRoleBinding, cmpOptions...); diff != "" { + return fmt.Errorf("role binding(%s) mismatch (-want +got):\n%s", gotRoleBinding.Name, diff) } return nil }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to compare actual and expected role bindings in %s cluster", cluster.ClusterName) @@ -85,18 +87,17 @@ func CreateClusterResourcePlacement(ctx context.Context, cluster framework.Clust } // WaitCreateClusterResourcePlacementStatus waits for ClusterResourcePlacement to present on th hub cluster with a specific status. -func WaitCreateClusterResourcePlacementStatus(ctx context.Context, cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement, status v1alpha1.ClusterResourcePlacementStatus, customTimeout time.Duration) { +func WaitCreateClusterResourcePlacementStatus(ctx context.Context, cluster framework.Cluster, objectKey *types.NamespacedName, wantCRPStatus v1alpha1.ClusterResourcePlacementStatus, crpStatusCmpOptions []cmp.Option, customTimeout time.Duration) { + gotCRP := &v1alpha1.ClusterResourcePlacement{} gomega.Eventually(func() error { - if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp); err != nil { + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: objectKey.Name}, gotCRP); err != nil { return err } - ignoreOption := cmpopts.IgnoreFields(metav1.Condition{}, "ObservedGeneration", "LastTransitionTime") - statusDiff := cmp.Diff(status, crp.Status, ignoreOption) - if statusDiff != "" { - return fmt.Errorf("cluster resource placment(%s) status mismatch (-want +got):\n%s", crp.Name, statusDiff) + if statusDiff := cmp.Diff(wantCRPStatus, gotCRP.Status, crpStatusCmpOptions...); statusDiff != "" { + return fmt.Errorf("cluster resource placment(%s) status mismatch (-want +got):\n%s", gotCRP.Name, statusDiff) } return nil - }, customTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for cluster resource placement %s status to be updated", crp.Name, cluster.ClusterName) + }, customTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for cluster resource placement %s status to be updated", gotCRP.Name, cluster.ClusterName) } // DeleteClusterResourcePlacement is used delete ClusterResourcePlacement on the hub cluster. diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index de6237e58..c9dac99e6 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -67,8 +67,7 @@ var _ = Describe("workload orchestration testing", func() { }, }, } - - testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) + Expect(HubCluster.KubeClient.Create(ctx, crp)).Should(Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, HubCluster.ClusterName) By("check if work gets created for cluster resource placement") testutils.WaitWork(ctx, *HubCluster, crp.Name, memberNamespace.Name) @@ -99,7 +98,7 @@ var _ = Describe("workload orchestration testing", func() { }, TargetClusters: []string{"kind-member-testing"}, } - testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, crp, crpStatus, 3*testutils.PollTimeout) + testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*testutils.PollTimeout) By("check if cluster role is propagated to member cluster") ownerReferences := []metav1.OwnerReference{ @@ -112,12 +111,7 @@ var _ = Describe("workload orchestration testing", func() { } expectedClusterRole := clusterRole expectedClusterRole.OwnerReferences = ownerReferences - clusterRole = &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: clusterRole.Name, - }, - } - testutils.CmpClusterRole(ctx, *MemberCluster, clusterRole, expectedClusterRole, resourceIgnoreOptions) + testutils.CmpClusterRole(ctx, *MemberCluster, &types.NamespacedName{Name: clusterRole.Name}, expectedClusterRole, resourceIgnoreOptions) By("update cluster role in Hub cluster") rules := []rbacv1.PolicyRule{ @@ -136,6 +130,7 @@ var _ = Describe("workload orchestration testing", func() { } Expect(HubCluster.KubeClient.Update(ctx, updatedClusterRole)).Should(Succeed(), "Failed to update cluster role %s in %s cluster", updatedClusterRole.Name, HubCluster.ClusterName) + By("check if cluster role got updated in member cluster") expectedClusterRole = &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: "test-cluster-role", @@ -144,14 +139,7 @@ var _ = Describe("workload orchestration testing", func() { }, Rules: rules, } - clusterRole = &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-cluster-role", - }, - } - - By("check if cluster role got updated in member cluster") - testutils.CmpClusterRole(ctx, *MemberCluster, clusterRole, expectedClusterRole, resourceIgnoreOptions) + testutils.CmpClusterRole(ctx, *MemberCluster, &types.NamespacedName{Name: clusterRole.Name}, expectedClusterRole, resourceIgnoreOptions) By("delete cluster role on hub cluster") Expect(HubCluster.KubeClient.Delete(ctx, clusterRole)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", clusterRole.Name, HubCluster.ClusterName) @@ -229,7 +217,7 @@ var _ = Describe("workload orchestration testing", func() { }, }, } - testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) + Expect(HubCluster.KubeClient.Create(ctx, crp)).Should(Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, HubCluster.ClusterName) By("check if work gets created for cluster resource placement") testutils.WaitWork(ctx, *HubCluster, crp.Name, memberNamespace.Name) @@ -273,7 +261,7 @@ var _ = Describe("workload orchestration testing", func() { }, TargetClusters: []string{"kind-member-testing"}, } - testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, crp, crpStatus, 3*testutils.PollTimeout) + testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*testutils.PollTimeout) By("check if resources in namespace are propagated to member cluster") ownerReferences := []metav1.OwnerReference{ @@ -290,26 +278,9 @@ var _ = Describe("workload orchestration testing", func() { expectedNamespace.OwnerReferences = ownerReferences expectedRole.OwnerReferences = ownerReferences expectedRoleBinding.OwnerReferences = ownerReferences - namespace = &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace.Name, - }, - } - role = &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: role.Name, - Namespace: role.Namespace, - }, - } - roleBinding = &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: roleBinding.Name, - Namespace: roleBinding.Namespace, - }, - } - testutils.CmpNamespace(ctx, *MemberCluster, namespace, expectedNamespace, resourceIgnoreOptions) - testutils.CmpRole(ctx, *MemberCluster, role, expectedRole, resourceIgnoreOptions) - testutils.CmpRoleBinding(ctx, *MemberCluster, roleBinding, expectedRoleBinding, resourceIgnoreOptions) + testutils.CmpNamespace(ctx, *MemberCluster, &types.NamespacedName{Name: namespace.Name}, expectedNamespace, resourceIgnoreOptions) + testutils.CmpRole(ctx, *MemberCluster, &types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, expectedRole, resourceIgnoreOptions) + testutils.CmpRoleBinding(ctx, *MemberCluster, &types.NamespacedName{Name: roleBinding.Name, Namespace: roleBinding.Namespace}, expectedRoleBinding, resourceIgnoreOptions) By("update role in Hub cluster") rules := []rbacv1.PolicyRule{ @@ -330,7 +301,7 @@ var _ = Describe("workload orchestration testing", func() { expectedRole.Rules = rules By("check if role got updated in member cluster") - testutils.CmpRole(ctx, *MemberCluster, role, expectedRole, resourceIgnoreOptions) + testutils.CmpRole(ctx, *MemberCluster, &types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, expectedRole, resourceIgnoreOptions) By("delete namespace") Expect(HubCluster.KubeClient.Delete(context.TODO(), namespace)).Should(Succeed(), "Failed to delete namespace %s in %s cluster", namespace.Name, HubCluster.ClusterName) From 2aae94334016bbf466df31efb9ccfa6f8478a3ca Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Fri, 30 Sep 2022 13:25:24 +0530 Subject: [PATCH 12/20] Fix import --- Makefile | 4 ++-- test/e2e/utils/workload_test_utils.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 848cb333b..71ff850be 100644 --- a/Makefile +++ b/Makefile @@ -178,7 +178,7 @@ run-e2e: build-e2e creat-kind-cluster: create-hub-kind-cluster create-member-kind-cluster install-helm .PHONY: install-helm -install-helm: load-hub-docker-image load-member-docker-image install-member-agent-helm +install-helm: docker-build-hub-agent docker-build-member-agent docker-build-refresh-token load-hub-docker-image load-member-docker-image install-member-agent-helm .PHONY: e2e-tests e2e-tests: creat-kind-cluster run-e2e @@ -227,7 +227,7 @@ run-memberagent: manifests generate fmt vet ## Run a controllers from your host. ## Images ## -------------------------------------- -OUTPUT_TYPE ?= type=registry +OUTPUT_TYPE ?= type=docker BUILDX_BUILDER_NAME ?= img-builder QEMU_VERSION ?= 5.2.0-2 diff --git a/test/e2e/utils/workload_test_utils.go b/test/e2e/utils/workload_test_utils.go index ae1bd19c0..f9dd221ab 100644 --- a/test/e2e/utils/workload_test_utils.go +++ b/test/e2e/utils/workload_test_utils.go @@ -18,7 +18,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "go.goms.io/fleet/apis/v1alpha1" + fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/test/e2e/framework" ) @@ -79,7 +79,7 @@ func CmpRoleBinding(ctx context.Context, cluster framework.Cluster, objectKey *t } // CreateClusterResourcePlacement created ClusterResourcePlacement and waits for ClusterResourcePlacement to exist in hub cluster. -func CreateClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { +func CreateClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *fleetv1alpha1.ClusterResourcePlacement) { gomega.Expect(cluster.KubeClient.Create(ctx, crp)).Should(gomega.Succeed()) gomega.Eventually(func() error { return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp) @@ -87,8 +87,8 @@ func CreateClusterResourcePlacement(ctx context.Context, cluster framework.Clust } // WaitCreateClusterResourcePlacementStatus waits for ClusterResourcePlacement to present on th hub cluster with a specific status. -func WaitCreateClusterResourcePlacementStatus(ctx context.Context, cluster framework.Cluster, objectKey *types.NamespacedName, wantCRPStatus v1alpha1.ClusterResourcePlacementStatus, crpStatusCmpOptions []cmp.Option, customTimeout time.Duration) { - gotCRP := &v1alpha1.ClusterResourcePlacement{} +func WaitCreateClusterResourcePlacementStatus(ctx context.Context, cluster framework.Cluster, objectKey *types.NamespacedName, wantCRPStatus fleetv1alpha1.ClusterResourcePlacementStatus, crpStatusCmpOptions []cmp.Option, customTimeout time.Duration) { + gotCRP := &fleetv1alpha1.ClusterResourcePlacement{} gomega.Eventually(func() error { if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: objectKey.Name}, gotCRP); err != nil { return err @@ -101,7 +101,7 @@ func WaitCreateClusterResourcePlacementStatus(ctx context.Context, cluster frame } // DeleteClusterResourcePlacement is used delete ClusterResourcePlacement on the hub cluster. -func DeleteClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *v1alpha1.ClusterResourcePlacement) { +func DeleteClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *fleetv1alpha1.ClusterResourcePlacement) { gomega.Expect(cluster.KubeClient.Delete(ctx, crp)).Should(gomega.Succeed(), "Failed to delete cluster resource placement %s in %s cluster", crp.Name, cluster.ClusterName) gomega.Eventually(func() bool { return apierrors.IsNotFound(cluster.KubeClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp)) From ad465dbad0457c698e2d77f3a05cd3d4f443895b Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Sat, 1 Oct 2022 12:03:50 +0530 Subject: [PATCH 13/20] New namespace resource --- test/e2e/work_load_test.go | 41 ++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index c9dac99e6..07e49fb1f 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -158,18 +158,24 @@ var _ = Describe("workload orchestration testing", func() { It("Apply CRP selecting namespace by label and check if namespace gets propagated with role, role binding, then update existing role", func() { By("create the resources to be propagated") - namespace := &corev1.Namespace{ + namespace1 := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-namespace", + Name: "test-namespace1", Labels: map[string]string{labelKey: labelValue}, }, } - testutils.CreateNamespace(ctx, *HubCluster, namespace) + namespace2 := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace2", + }, + } + Expect(HubCluster.KubeClient.Create(ctx, namespace1)).Should(Succeed(), "Failed to create namespace %s in %s cluster", namespace1.Name, HubCluster.ClusterName) + Expect(HubCluster.KubeClient.Create(ctx, namespace2)).Should(Succeed(), "Failed to create namespace %s in %s cluster", namespace2.Name, HubCluster.ClusterName) role := &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pod-reader", - Namespace: namespace.Name, + Namespace: namespace1.Name, }, Rules: []rbacv1.PolicyRule{ { @@ -184,7 +190,7 @@ var _ = Describe("workload orchestration testing", func() { roleBinding := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "read-pods", - Namespace: namespace.Name, + Namespace: namespace1.Name, }, Subjects: []rbacv1.Subject{ { @@ -211,7 +217,7 @@ var _ = Describe("workload orchestration testing", func() { Version: "v1", Kind: "Namespace", LabelSelector: &metav1.LabelSelector{ - MatchLabels: namespace.Labels, + MatchLabels: namespace1.Labels, }, }, }, @@ -256,7 +262,7 @@ var _ = Describe("workload orchestration testing", func() { { Version: "v1", Kind: "Namespace", - Name: namespace.Name, + Name: namespace1.Name, }, }, TargetClusters: []string{"kind-member-testing"}, @@ -272,16 +278,21 @@ var _ = Describe("workload orchestration testing", func() { Name: crp.Name, }, } - expectedNamespace := namespace + expectedNamespace := namespace1 expectedRole := role expectedRoleBinding := roleBinding expectedNamespace.OwnerReferences = ownerReferences expectedRole.OwnerReferences = ownerReferences expectedRoleBinding.OwnerReferences = ownerReferences - testutils.CmpNamespace(ctx, *MemberCluster, &types.NamespacedName{Name: namespace.Name}, expectedNamespace, resourceIgnoreOptions) + testutils.CmpNamespace(ctx, *MemberCluster, &types.NamespacedName{Name: namespace1.Name}, expectedNamespace, resourceIgnoreOptions) testutils.CmpRole(ctx, *MemberCluster, &types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, expectedRole, resourceIgnoreOptions) testutils.CmpRoleBinding(ctx, *MemberCluster, &types.NamespacedName{Name: roleBinding.Name, Namespace: roleBinding.Namespace}, expectedRoleBinding, resourceIgnoreOptions) + By("check if namespace not selected by CRP doesn't exist on member cluster") + Consistently(func() bool { + return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace2.Name}, namespace2)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify namespace %s is not propagated to %s cluster", namespace2.Name, MemberCluster.ClusterName) + By("update role in Hub cluster") rules := []rbacv1.PolicyRule{ { @@ -293,7 +304,7 @@ var _ = Describe("workload orchestration testing", func() { updatedRole := &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: role.Name, - Namespace: namespace.Name, + Namespace: namespace1.Name, }, Rules: rules, } @@ -304,15 +315,15 @@ var _ = Describe("workload orchestration testing", func() { testutils.CmpRole(ctx, *MemberCluster, &types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, expectedRole, resourceIgnoreOptions) By("delete namespace") - Expect(HubCluster.KubeClient.Delete(context.TODO(), namespace)).Should(Succeed(), "Failed to delete namespace %s in %s cluster", namespace.Name, HubCluster.ClusterName) + Expect(HubCluster.KubeClient.Delete(context.TODO(), namespace1)).Should(Succeed(), "Failed to delete namespace %s in %s cluster", namespace1.Name, HubCluster.ClusterName) Eventually(func() bool { - return apierrors.IsNotFound(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, namespace)) - }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for namespace %s to be deleted in %s cluster", namespace.Name, HubCluster.ClusterName) + return apierrors.IsNotFound(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace1.Name}, namespace1)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for namespace %s to be deleted in %s cluster", namespace1.Name, HubCluster.ClusterName) By("check if namespace got deleted on member cluster") Eventually(func() bool { - return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace.Name}, namespace)) - }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", namespace.Name, MemberCluster.ClusterName) + return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace1.Name}, namespace1)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", namespace1.Name, MemberCluster.ClusterName) By("delete cluster resource placement on hub cluster") testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) From bdb23133f98f25eb370d8cc10b0984a2a843d096 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Mon, 17 Oct 2022 11:52:20 -0700 Subject: [PATCH 14/20] Address comments --- Makefile | 4 +- test/e2e/e2e_test.go | 2 +- test/e2e/join_leave_placement_test.go | 39 ++++++------- test/e2e/utils/helper.go | 8 --- test/e2e/utils/workload_test_utils.go | 8 --- test/e2e/work_api_e2e_test.go | 3 +- test/e2e/work_load_test.go | 82 ++++++++++++--------------- 7 files changed, 62 insertions(+), 84 deletions(-) diff --git a/Makefile b/Makefile index 71ff850be..848cb333b 100644 --- a/Makefile +++ b/Makefile @@ -178,7 +178,7 @@ run-e2e: build-e2e creat-kind-cluster: create-hub-kind-cluster create-member-kind-cluster install-helm .PHONY: install-helm -install-helm: docker-build-hub-agent docker-build-member-agent docker-build-refresh-token load-hub-docker-image load-member-docker-image install-member-agent-helm +install-helm: load-hub-docker-image load-member-docker-image install-member-agent-helm .PHONY: e2e-tests e2e-tests: creat-kind-cluster run-e2e @@ -227,7 +227,7 @@ run-memberagent: manifests generate fmt vet ## Run a controllers from your host. ## Images ## -------------------------------------- -OUTPUT_TYPE ?= type=docker +OUTPUT_TYPE ?= type=registry BUILDX_BUILDER_NAME ?= img-builder QEMU_VERSION ?= 5.2.0-2 diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index adfe96511..3ecaf95ea 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -71,7 +71,7 @@ var ( sortOption, } crpStatusCmpOptions = []cmp.Option{ - cmpopts.IgnoreFields(metav1.Condition{}, "ObservedGeneration", "LastTransitionTime"), + cmpopts.IgnoreFields(metav1.Condition{}, "ObservedGeneration", "LastTransitionTime", "Message"), sortOption, } diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go index 5543c3966..ee2066bb6 100644 --- a/test/e2e/join_leave_placement_test.go +++ b/test/e2e/join_leave_placement_test.go @@ -17,7 +17,7 @@ import ( "k8s.io/apimachinery/pkg/types" "go.goms.io/fleet/apis/v1alpha1" - testutils "go.goms.io/fleet/test/e2e/utils" + "go.goms.io/fleet/test/e2e/utils" ) // Serial - Ginkgo will guarantee that these specs will never run in parallel with other specs. @@ -50,7 +50,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcLeftAgentStatus, Conditions: mcLeftConditions, } - testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) + utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) By("create the resources to be propagated") cr := &rbacv1.ClusterRole{ @@ -86,12 +86,12 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( }, }, } - testutils.CreateClusterResourcePlacement(ctx, *HubCluster, crp) + Expect(HubCluster.KubeClient.Create(ctx, crp)).Should(Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, HubCluster.ClusterName) By("verify the resource is not propagated to member cluster") Consistently(func() bool { return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)) - }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) + }, utils.PollTimeout, utils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName) By("update member cluster in the hub cluster to join") Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) @@ -103,22 +103,20 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcJoinedAgentStatus, Conditions: mcJoinedConditions, } - testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) + utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) By("verify that the cluster resource placement is applied") crpStatus := v1alpha1.ClusterResourcePlacementStatus{ Conditions: []metav1.Condition{ { - Message: "Successfully scheduled resources for placement", - Reason: "ScheduleSucceeded", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled), + Reason: "ScheduleSucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled), }, { - Message: "Successfully applied resources to member clusters", - Reason: "ApplySucceeded", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), + Reason: "ApplySucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), }, }, SelectedResources: []v1alpha1.ResourceIdentifier{ @@ -131,7 +129,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( }, TargetClusters: []string{"kind-member-testing"}, } - testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*testutils.PollTimeout) + utils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*utils.PollTimeout) By("verify the resource is propagated to member cluster") Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName) @@ -146,24 +144,27 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcLeftAgentStatus, Conditions: mcLeftConditions, } - testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) + utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) By("verify that the resource is still on the member cluster") Consistently(func() error { return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr) - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) + }, utils.PollTimeout, utils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) By("delete the crp from the hub") - testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) + utils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) By("verify that the resource is still on the member cluster") Consistently(func() error { return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr) - }, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) + }, utils.PollTimeout, utils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName) By("delete cluster role on hub cluster") Expect(HubCluster.KubeClient.Delete(ctx, cr)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", cr.Name, HubCluster.ClusterName) + By("delete cluster role on member cluster") + Expect(MemberCluster.KubeClient.Delete(ctx, cr)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", cr.Name, MemberCluster.ClusterName) + By("update member cluster in the hub cluster to join") Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName) mc.Spec.State = v1alpha1.ClusterStateJoin @@ -174,6 +175,6 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( AgentStatus: imcJoinedAgentStatus, Conditions: mcJoinedConditions, } - testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) + utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions) }) }) diff --git a/test/e2e/utils/helper.go b/test/e2e/utils/helper.go index a1c7565cd..34a3d691c 100644 --- a/test/e2e/utils/helper.go +++ b/test/e2e/utils/helper.go @@ -85,14 +85,6 @@ func WaitWork(ctx context.Context, cluster framework.Cluster, workName, workName }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Work %s not synced", name) } -// CreateNamespace create namespace and waits for namespace to exist. -func CreateNamespace(ctx context.Context, cluster framework.Cluster, ns *corev1.Namespace) { - gomega.Expect(cluster.KubeClient.Create(ctx, ns)).Should(gomega.Succeed(), "Failed to create namespace %s in %s cluster", ns.Name, cluster.ClusterName) - gomega.Eventually(func() error { - return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: ns.Name}, ns) - }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to wait for namespace %s to be created in %s cluster", ns.Name, cluster.ClusterName) -} - // DeleteNamespace delete namespace. func DeleteNamespace(ctx context.Context, cluster framework.Cluster, ns *corev1.Namespace) { gomega.Expect(cluster.KubeClient.Delete(context.TODO(), ns)).Should(gomega.Succeed(), "Failed to delete namespace %s in %s cluster", ns.Name, cluster.ClusterName) diff --git a/test/e2e/utils/workload_test_utils.go b/test/e2e/utils/workload_test_utils.go index f9dd221ab..3c4fe84c7 100644 --- a/test/e2e/utils/workload_test_utils.go +++ b/test/e2e/utils/workload_test_utils.go @@ -78,14 +78,6 @@ func CmpRoleBinding(ctx context.Context, cluster framework.Cluster, objectKey *t }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to compare actual and expected role bindings in %s cluster", cluster.ClusterName) } -// CreateClusterResourcePlacement created ClusterResourcePlacement and waits for ClusterResourcePlacement to exist in hub cluster. -func CreateClusterResourcePlacement(ctx context.Context, cluster framework.Cluster, crp *fleetv1alpha1.ClusterResourcePlacement) { - gomega.Expect(cluster.KubeClient.Create(ctx, crp)).Should(gomega.Succeed()) - gomega.Eventually(func() error { - return cluster.KubeClient.Get(ctx, types.NamespacedName{Name: crp.Name}, crp) - }, PollTimeout, PollInterval).Should(gomega.Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, cluster.ClusterName) -} - // WaitCreateClusterResourcePlacementStatus waits for ClusterResourcePlacement to present on th hub cluster with a specific status. func WaitCreateClusterResourcePlacementStatus(ctx context.Context, cluster framework.Cluster, objectKey *types.NamespacedName, wantCRPStatus fleetv1alpha1.ClusterResourcePlacementStatus, crpStatusCmpOptions []cmp.Option, customTimeout time.Duration) { gotCRP := &fleetv1alpha1.ClusterResourcePlacement{} diff --git a/test/e2e/work_api_e2e_test.go b/test/e2e/work_api_e2e_test.go index 3a0d595bd..fb944c806 100644 --- a/test/e2e/work_api_e2e_test.go +++ b/test/e2e/work_api_e2e_test.go @@ -71,7 +71,8 @@ var _ = Describe("Work API Controller test", func() { Name: resourceNamespaceName, }, } - testutils.CreateNamespace(ctx, *MemberCluster, resourceNamespace) + //testutils.CreateNamespace(ctx, *MemberCluster, resourceNamespace) + Expect(MemberCluster.KubeClient.Create(ctx, resourceNamespace)).Should(Succeed(), "Failed to create namespace %s in %s cluster", resourceNamespace.Name, MemberCluster.ClusterName) }) AfterEach(func() { diff --git a/test/e2e/work_load_test.go b/test/e2e/work_load_test.go index 07e49fb1f..bbb5c4af2 100644 --- a/test/e2e/work_load_test.go +++ b/test/e2e/work_load_test.go @@ -6,22 +6,20 @@ Licensed under the MIT license. package e2e import ( - "context" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" workapi "sigs.k8s.io/work-api/pkg/apis/v1alpha1" "go.goms.io/fleet/apis/v1alpha1" - testutils "go.goms.io/fleet/test/e2e/utils" + "go.goms.io/fleet/test/e2e/utils" ) var _ = Describe("workload orchestration testing", func() { @@ -70,22 +68,20 @@ var _ = Describe("workload orchestration testing", func() { Expect(HubCluster.KubeClient.Create(ctx, crp)).Should(Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, HubCluster.ClusterName) By("check if work gets created for cluster resource placement") - testutils.WaitWork(ctx, *HubCluster, crp.Name, memberNamespace.Name) + utils.WaitWork(ctx, *HubCluster, crp.Name, memberNamespace.Name) By("check if cluster resource placement status is updated") crpStatus := v1alpha1.ClusterResourcePlacementStatus{ Conditions: []metav1.Condition{ { - Message: "Successfully scheduled resources for placement", - Reason: "ScheduleSucceeded", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled), + Reason: "ScheduleSucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled), }, { - Message: "Successfully applied resources to member clusters", - Reason: "ApplySucceeded", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), + Reason: "ApplySucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), }, }, SelectedResources: []v1alpha1.ResourceIdentifier{ @@ -98,7 +94,7 @@ var _ = Describe("workload orchestration testing", func() { }, TargetClusters: []string{"kind-member-testing"}, } - testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*testutils.PollTimeout) + utils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*utils.PollTimeout) By("check if cluster role is propagated to member cluster") ownerReferences := []metav1.OwnerReference{ @@ -111,7 +107,7 @@ var _ = Describe("workload orchestration testing", func() { } expectedClusterRole := clusterRole expectedClusterRole.OwnerReferences = ownerReferences - testutils.CmpClusterRole(ctx, *MemberCluster, &types.NamespacedName{Name: clusterRole.Name}, expectedClusterRole, resourceIgnoreOptions) + utils.CmpClusterRole(ctx, *MemberCluster, &types.NamespacedName{Name: clusterRole.Name}, expectedClusterRole, resourceIgnoreOptions) By("update cluster role in Hub cluster") rules := []rbacv1.PolicyRule{ @@ -139,21 +135,21 @@ var _ = Describe("workload orchestration testing", func() { }, Rules: rules, } - testutils.CmpClusterRole(ctx, *MemberCluster, &types.NamespacedName{Name: clusterRole.Name}, expectedClusterRole, resourceIgnoreOptions) + utils.CmpClusterRole(ctx, *MemberCluster, &types.NamespacedName{Name: clusterRole.Name}, expectedClusterRole, resourceIgnoreOptions) By("delete cluster role on hub cluster") Expect(HubCluster.KubeClient.Delete(ctx, clusterRole)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", clusterRole.Name, HubCluster.ClusterName) Eventually(func() bool { - return apierrors.IsNotFound(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: clusterRole.Name}, clusterRole)) - }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", clusterRole.Name, HubCluster.ClusterName) + return errors.IsNotFound(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: clusterRole.Name}, clusterRole)) + }, utils.PollTimeout, utils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", clusterRole.Name, HubCluster.ClusterName) By("check if cluster role got deleted on member cluster") Eventually(func() bool { - return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: clusterRole.Name}, clusterRole)) - }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", clusterRole.Name, MemberCluster.ClusterName) + return errors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: clusterRole.Name}, clusterRole)) + }, utils.PollTimeout, utils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", clusterRole.Name, MemberCluster.ClusterName) By("delete cluster resource placement on hub cluster") - testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) + utils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) }) It("Apply CRP selecting namespace by label and check if namespace gets propagated with role, role binding, then update existing role", func() { @@ -226,22 +222,20 @@ var _ = Describe("workload orchestration testing", func() { Expect(HubCluster.KubeClient.Create(ctx, crp)).Should(Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, HubCluster.ClusterName) By("check if work gets created for cluster resource placement") - testutils.WaitWork(ctx, *HubCluster, crp.Name, memberNamespace.Name) + utils.WaitWork(ctx, *HubCluster, crp.Name, memberNamespace.Name) By("check if cluster resource placement status is updated") crpStatus := v1alpha1.ClusterResourcePlacementStatus{ Conditions: []metav1.Condition{ { - Message: "Successfully scheduled resources for placement", - Reason: "ScheduleSucceeded", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled), + Reason: "ScheduleSucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled), }, { - Message: "Successfully applied resources to member clusters", - Reason: "ApplySucceeded", - Status: metav1.ConditionTrue, - Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), + Reason: "ApplySucceeded", + Status: metav1.ConditionTrue, + Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), }, }, SelectedResources: []v1alpha1.ResourceIdentifier{ @@ -267,7 +261,7 @@ var _ = Describe("workload orchestration testing", func() { }, TargetClusters: []string{"kind-member-testing"}, } - testutils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*testutils.PollTimeout) + utils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*utils.PollTimeout) By("check if resources in namespace are propagated to member cluster") ownerReferences := []metav1.OwnerReference{ @@ -284,14 +278,14 @@ var _ = Describe("workload orchestration testing", func() { expectedNamespace.OwnerReferences = ownerReferences expectedRole.OwnerReferences = ownerReferences expectedRoleBinding.OwnerReferences = ownerReferences - testutils.CmpNamespace(ctx, *MemberCluster, &types.NamespacedName{Name: namespace1.Name}, expectedNamespace, resourceIgnoreOptions) - testutils.CmpRole(ctx, *MemberCluster, &types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, expectedRole, resourceIgnoreOptions) - testutils.CmpRoleBinding(ctx, *MemberCluster, &types.NamespacedName{Name: roleBinding.Name, Namespace: roleBinding.Namespace}, expectedRoleBinding, resourceIgnoreOptions) + utils.CmpNamespace(ctx, *MemberCluster, &types.NamespacedName{Name: namespace1.Name}, expectedNamespace, resourceIgnoreOptions) + utils.CmpRole(ctx, *MemberCluster, &types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, expectedRole, resourceIgnoreOptions) + utils.CmpRoleBinding(ctx, *MemberCluster, &types.NamespacedName{Name: roleBinding.Name, Namespace: roleBinding.Namespace}, expectedRoleBinding, resourceIgnoreOptions) By("check if namespace not selected by CRP doesn't exist on member cluster") Consistently(func() bool { - return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace2.Name}, namespace2)) - }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify namespace %s is not propagated to %s cluster", namespace2.Name, MemberCluster.ClusterName) + return errors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace2.Name}, namespace2)) + }, utils.PollTimeout, utils.PollInterval).Should(BeTrue(), "Failed to verify namespace %s is not propagated to %s cluster", namespace2.Name, MemberCluster.ClusterName) By("update role in Hub cluster") rules := []rbacv1.PolicyRule{ @@ -312,21 +306,19 @@ var _ = Describe("workload orchestration testing", func() { expectedRole.Rules = rules By("check if role got updated in member cluster") - testutils.CmpRole(ctx, *MemberCluster, &types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, expectedRole, resourceIgnoreOptions) + utils.CmpRole(ctx, *MemberCluster, &types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, expectedRole, resourceIgnoreOptions) - By("delete namespace") - Expect(HubCluster.KubeClient.Delete(context.TODO(), namespace1)).Should(Succeed(), "Failed to delete namespace %s in %s cluster", namespace1.Name, HubCluster.ClusterName) - Eventually(func() bool { - return apierrors.IsNotFound(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace1.Name}, namespace1)) - }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for namespace %s to be deleted in %s cluster", namespace1.Name, HubCluster.ClusterName) + By("delete namespaces") + utils.DeleteNamespace(ctx, *HubCluster, namespace1) + utils.DeleteNamespace(ctx, *HubCluster, namespace2) By("check if namespace got deleted on member cluster") Eventually(func() bool { - return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace1.Name}, namespace1)) - }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", namespace1.Name, MemberCluster.ClusterName) + return errors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: namespace1.Name}, namespace1)) + }, utils.PollTimeout, utils.PollInterval).Should(BeTrue(), "Failed to wait for cluster role %s to be deleted in %s cluster", namespace1.Name, MemberCluster.ClusterName) By("delete cluster resource placement on hub cluster") - testutils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) + utils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp) }) }) }) From 0ac14eb19c1b76e72beb59af168ea6cd31c80d02 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Mon, 17 Oct 2022 12:04:05 -0700 Subject: [PATCH 15/20] Remove comment --- test/e2e/work_api_e2e_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/work_api_e2e_test.go b/test/e2e/work_api_e2e_test.go index fb944c806..5de0fb111 100644 --- a/test/e2e/work_api_e2e_test.go +++ b/test/e2e/work_api_e2e_test.go @@ -71,7 +71,6 @@ var _ = Describe("Work API Controller test", func() { Name: resourceNamespaceName, }, } - //testutils.CreateNamespace(ctx, *MemberCluster, resourceNamespace) Expect(MemberCluster.KubeClient.Create(ctx, resourceNamespace)).Should(Succeed(), "Failed to create namespace %s in %s cluster", resourceNamespace.Name, MemberCluster.ClusterName) }) From 1073968946cf005c121abb24fe24a3cbedfa3f15 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Mon, 17 Oct 2022 13:39:15 -0700 Subject: [PATCH 16/20] Fix cmp option --- test/e2e/join_leave_placement_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/join_leave_placement_test.go b/test/e2e/join_leave_placement_test.go index ee2066bb6..ab12a99a1 100644 --- a/test/e2e/join_leave_placement_test.go +++ b/test/e2e/join_leave_placement_test.go @@ -30,7 +30,8 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, func( mcStatusCmpOptions = []cmp.Option{ cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"), cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"), - cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }), + cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), + sortOption, } ) From 9adef01149390d6a7905a5248cbd06613f83b46f Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Mon, 17 Oct 2022 15:50:08 -0700 Subject: [PATCH 17/20] Fix E2E test failure --- test/e2e/work_api_e2e_test.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/test/e2e/work_api_e2e_test.go b/test/e2e/work_api_e2e_test.go index 5de0fb111..ee73c29a4 100644 --- a/test/e2e/work_api_e2e_test.go +++ b/test/e2e/work_api_e2e_test.go @@ -3,13 +3,13 @@ package e2e import ( "context" "fmt" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -201,6 +201,8 @@ var _ = Describe("Work API Controller test", func() { By(fmt.Sprintf("Validating that the annotation of resource's spec exists on the resource %s", manifestConfigMapName)) Expect(gotConfigMap.ObjectMeta.Annotations[specHashAnnotation]).ToNot(BeEmpty(), "SpecHash Annotation does not exist for resource %s", gotConfigMap.Name) + + testutils.DeleteWork(ctx, *HubCluster, work) }) It("Upon successful creation of 2 work resources with same manifest, work manifest is applied, and only 1 resource is created with merged owner references.", func() { @@ -363,6 +365,9 @@ var _ = Describe("Work API Controller test", func() { By(fmt.Sprintf("Validating that the annotation of resource's spec exists on the resource %s", manifestSecretName)) Expect(retrievedSecret.ObjectMeta.Annotations[specHashAnnotation]).ToNot(BeEmpty(), "SpecHash Annotation does not exist for resource %s", secret.Name) + + testutils.DeleteWork(ctx, *HubCluster, workOne) + testutils.DeleteWork(ctx, *HubCluster, workTwo) }) It("Upon successful work creation of a CRD resource, manifest is applied, and resources are created", func() { @@ -556,6 +561,8 @@ var _ = Describe("Work API Controller test", func() { "There is no spec annotation on the resource %s", crd.Name) Expect(customResource.GetAnnotations()[specHashAnnotation]).ToNot(BeEmpty(), "There is no spec annotation on the custom resource %s", customResource.GetName()) + + testutils.DeleteWork(ctx, *HubCluster, work) }) It("Manifests with dependencies within different work objects should successfully apply", func() { @@ -773,7 +780,11 @@ var _ = Describe("Work API Controller test", func() { "OwnerReference mismatch for resource %s (-want, +got):", testNamespace.Name) Expect(cmp.Diff(wantOwnerForServiceAccount, retrievedServiceAccount.OwnerReferences, cmpOptions...)).Should(BeEmpty(), "OwnerReference mismatch for resource %s (-want, +got):", testServiceAccount.Name) + + testutils.DeleteWork(ctx, *HubCluster, workForNamespace) + testutils.DeleteWork(ctx, *HubCluster, workForServiceAccount) }) + }) Context("Updating Work", func() { @@ -823,6 +834,10 @@ var _ = Describe("Work API Controller test", func() { }) + AfterEach(func() { + testutils.DeleteWork(ctx, *HubCluster, work) + }) + It("Updating Work object on the Hub Cluster should update the resource on the member cluster.", func() { updatedConfigMap := configMap.DeepCopy() updatedConfigMap.Data = map[string]string{ @@ -943,8 +958,10 @@ var _ = Describe("Work API Controller test", func() { By("Deleting the Work Object should also delete the resources in the member cluster") configMapDeleted := corev1.ConfigMap{} resourceNamespaceType := types.NamespacedName{Name: configMapBeforeDelete.Name, Namespace: resourceNamespace.Name} - Expect(MemberCluster.KubeClient.Get(ctx, resourceNamespaceType, &configMapDeleted)).Should(&utils.NotFoundMatcher{}, - "resource %s was either not deleted or encountered an error in cluster %s", configMapBeforeDelete.Name, MemberCluster.ClusterName) + // TODO: Check why removing matcher check fixed test failure + Eventually(func() bool { + return errors.IsNotFound(MemberCluster.KubeClient.Get(ctx, resourceNamespaceType, &configMapDeleted)) + }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "resource %s was either not deleted or encountered an error in cluster %s", configMapBeforeDelete.Name, MemberCluster.ClusterName) }) }) }) From 1d7226f73d69525e2551aa0d695050a41c3f65af Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Mon, 17 Oct 2022 16:00:35 -0700 Subject: [PATCH 18/20] Fix import --- test/e2e/work_api_e2e_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/work_api_e2e_test.go b/test/e2e/work_api_e2e_test.go index ee73c29a4..ef8788ff8 100644 --- a/test/e2e/work_api_e2e_test.go +++ b/test/e2e/work_api_e2e_test.go @@ -3,6 +3,7 @@ package e2e import ( "context" "fmt" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" From cb6d703611d4dc0c7a9d4bc0d7631b73d2a69602 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 18 Oct 2022 15:04:40 -0700 Subject: [PATCH 19/20] Add Delete Work in AfterEach --- test/e2e/work_api_e2e_test.go | 45 ++++++++++++++++------------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/test/e2e/work_api_e2e_test.go b/test/e2e/work_api_e2e_test.go index ef8788ff8..c0b615b10 100644 --- a/test/e2e/work_api_e2e_test.go +++ b/test/e2e/work_api_e2e_test.go @@ -10,7 +10,6 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -60,6 +59,7 @@ var _ = Describe("Work API Controller test", func() { ) resourceNamespace *corev1.Namespace + workName string ) BeforeEach(func() { @@ -80,9 +80,15 @@ var _ = Describe("Work API Controller test", func() { }) Context("Work Creation Test", func() { - It("Upon successful work creation of a single resource, work manifest is applied and resource is created", func() { - workName := testutils.RandomWorkName(5) + BeforeEach(func() { + workName = testutils.RandomWorkName(5) + }) + AfterEach(func() { + testutils.DeleteWork(ctx, *HubCluster, workapi.Work{ObjectMeta: metav1.ObjectMeta{Name: workName, Namespace: workNamespace.Name}}) + }) + + It("Upon successful work creation of a single resource, work manifest is applied and resource is created", func() { By(fmt.Sprintf("Here is the work Name %s", workName)) // Configmap will be included in this work object. @@ -202,12 +208,9 @@ var _ = Describe("Work API Controller test", func() { By(fmt.Sprintf("Validating that the annotation of resource's spec exists on the resource %s", manifestConfigMapName)) Expect(gotConfigMap.ObjectMeta.Annotations[specHashAnnotation]).ToNot(BeEmpty(), "SpecHash Annotation does not exist for resource %s", gotConfigMap.Name) - - testutils.DeleteWork(ctx, *HubCluster, work) }) It("Upon successful creation of 2 work resources with same manifest, work manifest is applied, and only 1 resource is created with merged owner references.", func() { - workNameOne := testutils.RandomWorkName(5) workNameTwo := testutils.RandomWorkName(5) manifestSecretName := "test-secret" @@ -228,7 +231,7 @@ var _ = Describe("Work API Controller test", func() { } // Creating types.NamespacedName to use in retrieving objects. - namespaceTypeOne := types.NamespacedName{Name: workNameOne, Namespace: workNamespace.Name} + namespaceTypeOne := types.NamespacedName{Name: workName, Namespace: workNamespace.Name} namespaceTypeTwo := types.NamespacedName{Name: workNameTwo, Namespace: workNamespace.Name} resourceNamespaceType := types.NamespacedName{Name: manifestSecretName, Namespace: resourceNamespace.Name} @@ -236,7 +239,7 @@ var _ = Describe("Work API Controller test", func() { manifests := testutils.AddManifests([]runtime.Object{&secret}, []workapi.Manifest{}) By(fmt.Sprintf("creating work %s of %s", namespaceTypeOne, manifestSecretName)) - testutils.CreateWork(ctx, *HubCluster, workNameOne, workNamespace.Name, manifests) + testutils.CreateWork(ctx, *HubCluster, workName, workNamespace.Name, manifests) By(fmt.Sprintf("creating work %s of %s", namespaceTypeTwo, manifestSecretName)) testutils.CreateWork(ctx, *HubCluster, workNameTwo, workNamespace.Name, manifests) @@ -323,7 +326,7 @@ var _ = Describe("Work API Controller test", func() { appliedWorkOne := workapi.AppliedWork{} Expect(MemberCluster.KubeClient.Get(ctx, namespaceTypeOne, &appliedWorkOne)).Should(Succeed(), - "Retrieving AppliedWork %s failed", workNameOne) + "Retrieving AppliedWork %s failed", workName) Expect(cmp.Diff(wantAppliedStatus, appliedWorkOne.Status, appliedWorkCmpOptions...)).Should(BeEmpty(), "Validate AppliedResourceMeta mismatch (-want, +got):") @@ -367,12 +370,12 @@ var _ = Describe("Work API Controller test", func() { Expect(retrievedSecret.ObjectMeta.Annotations[specHashAnnotation]).ToNot(BeEmpty(), "SpecHash Annotation does not exist for resource %s", secret.Name) - testutils.DeleteWork(ctx, *HubCluster, workOne) + //testutils.DeleteWork(ctx, *HubCluster, workOne) testutils.DeleteWork(ctx, *HubCluster, workTwo) }) It("Upon successful work creation of a CRD resource, manifest is applied, and resources are created", func() { - workName := testutils.RandomWorkName(5) + //workName := testutils.RandomWorkName(5) // Name of the CRD object from the manifest file crdName := "testcrds.multicluster.x-k8s.io" @@ -562,13 +565,9 @@ var _ = Describe("Work API Controller test", func() { "There is no spec annotation on the resource %s", crd.Name) Expect(customResource.GetAnnotations()[specHashAnnotation]).ToNot(BeEmpty(), "There is no spec annotation on the custom resource %s", customResource.GetName()) - - testutils.DeleteWork(ctx, *HubCluster, work) }) It("Manifests with dependencies within different work objects should successfully apply", func() { - - workNameForNamespace := testutils.RandomWorkName(5) workNameForServiceAccount := testutils.RandomWorkName(6) testNamespace := corev1.Namespace{ @@ -595,10 +594,10 @@ var _ = Describe("Work API Controller test", func() { manifestNamespace := testutils.AddManifests([]runtime.Object{&testNamespace}, []workapi.Manifest{}) manifestServiceAccount := testutils.AddManifests([]runtime.Object{&testServiceAccount}, []workapi.Manifest{}) - workForNamespace := testutils.CreateWork(ctx, *HubCluster, workNameForNamespace, workNamespace.Name, manifestNamespace) + workForNamespace := testutils.CreateWork(ctx, *HubCluster, workName, workNamespace.Name, manifestNamespace) workForServiceAccount := testutils.CreateWork(ctx, *HubCluster, workNameForServiceAccount, workNamespace.Name, manifestServiceAccount) - By(fmt.Sprintf("Applied Condition should be set to True for Work %s and %s", workNameForNamespace, workNameForServiceAccount)) + By(fmt.Sprintf("Applied Condition should be set to True for Work %s and %s", workName, workNameForServiceAccount)) wantAppliedCondition := []metav1.Condition{ { @@ -611,7 +610,7 @@ var _ = Describe("Work API Controller test", func() { receivedWorkForNamespace := workapi.Work{} receivedWorkForServiceAccount := workapi.Work{} - namespaceTypeForNamespaceWork := types.NamespacedName{Name: workNameForNamespace, Namespace: workNamespace.Name} + namespaceTypeForNamespaceWork := types.NamespacedName{Name: workName, Namespace: workNamespace.Name} Eventually(func() string { if err := HubCluster.KubeClient.Get(ctx, namespaceTypeForNamespaceWork, &receivedWorkForNamespace); err != nil { @@ -682,7 +681,7 @@ var _ = Describe("Work API Controller test", func() { By(fmt.Sprintf("AppliedWorkStatus should contain the meta for the resource %s and %s", testNamespace.Name, testServiceAccount.Name)) appliedWorkForNamespace := workapi.AppliedWork{} Expect(MemberCluster.KubeClient.Get(ctx, namespaceTypeForNamespaceWork, &appliedWorkForNamespace)).Should(Succeed(), - "Retrieving AppliedWork %s failed", workNameForNamespace) + "Retrieving AppliedWork %s failed", workName) wantAppliedWorkConditionNamespace := workapi.AppliedtWorkStatus{ AppliedResources: []workapi.AppliedResourceMeta{ @@ -782,7 +781,6 @@ var _ = Describe("Work API Controller test", func() { Expect(cmp.Diff(wantOwnerForServiceAccount, retrievedServiceAccount.OwnerReferences, cmpOptions...)).Should(BeEmpty(), "OwnerReference mismatch for resource %s (-want, +got):", testServiceAccount.Name) - testutils.DeleteWork(ctx, *HubCluster, workForNamespace) testutils.DeleteWork(ctx, *HubCluster, workForServiceAccount) }) @@ -959,10 +957,9 @@ var _ = Describe("Work API Controller test", func() { By("Deleting the Work Object should also delete the resources in the member cluster") configMapDeleted := corev1.ConfigMap{} resourceNamespaceType := types.NamespacedName{Name: configMapBeforeDelete.Name, Namespace: resourceNamespace.Name} - // TODO: Check why removing matcher check fixed test failure - Eventually(func() bool { - return errors.IsNotFound(MemberCluster.KubeClient.Get(ctx, resourceNamespaceType, &configMapDeleted)) - }, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "resource %s was either not deleted or encountered an error in cluster %s", configMapBeforeDelete.Name, MemberCluster.ClusterName) + Eventually(func() error { + return MemberCluster.KubeClient.Get(ctx, resourceNamespaceType, &configMapDeleted) + }, testutils.PollTimeout, testutils.PollInterval).Should(&utils.NotFoundMatcher{}, "resource %s was either not deleted or encountered an error in cluster %s", configMapBeforeDelete.Name, MemberCluster.ClusterName) }) }) }) From 553dc65ba57957a36ef60fc0f575eb2197533c44 Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Tue, 18 Oct 2022 15:07:48 -0700 Subject: [PATCH 20/20] Remove comments --- test/e2e/work_api_e2e_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/e2e/work_api_e2e_test.go b/test/e2e/work_api_e2e_test.go index c0b615b10..e8449df8c 100644 --- a/test/e2e/work_api_e2e_test.go +++ b/test/e2e/work_api_e2e_test.go @@ -370,13 +370,10 @@ var _ = Describe("Work API Controller test", func() { Expect(retrievedSecret.ObjectMeta.Annotations[specHashAnnotation]).ToNot(BeEmpty(), "SpecHash Annotation does not exist for resource %s", secret.Name) - //testutils.DeleteWork(ctx, *HubCluster, workOne) testutils.DeleteWork(ctx, *HubCluster, workTwo) }) It("Upon successful work creation of a CRD resource, manifest is applied, and resources are created", func() { - //workName := testutils.RandomWorkName(5) - // Name of the CRD object from the manifest file crdName := "testcrds.multicluster.x-k8s.io" crdObjectName := "test-crd-object"