Skip to content

Commit

Permalink
test: Refactor workload orchestration E2E part 1 (#286)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Arvindthiru
Arvindthiru authored Oct 18, 2022
1 parent 4b51943 commit 25b57b1
Show file tree
Hide file tree
Showing 6 changed files with 485 additions and 217 deletions.
51 changes: 10 additions & 41 deletions test/e2e/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,16 @@ var (
cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"),
sortOption,
}

mcStatusCmpOptions = []cmp.Option{
cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"),
cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"),
cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"),
sortOption,
}
crpStatusCmpOptions = []cmp.Option{
cmpopts.IgnoreFields(metav1.Condition{}, "ObservedGeneration", "LastTransitionTime", "Message"),
sortOption,
}

imcJoinedAgentStatus = []v1alpha1.AgentStatus{
{
Expand Down Expand Up @@ -165,8 +168,6 @@ var _ = BeforeSuite(func() {
MemberCluster.HubURL = hubURL
framework.GetClusterClient(MemberCluster)

testutils.CreateNamespace(*HubCluster, workNamespace)

ctx = context.Background()

By("deploy member cluster in the hub cluster")
Expand Down Expand Up @@ -200,31 +201,16 @@ var _ = BeforeSuite(func() {

By("check if internal member cluster status is updated to Joined")
wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcJoinedAgentStatus}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff)
}
return nil
}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus)
testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, wantIMCStatus, imcStatusCmpOptions)

By("check if member cluster status is updated to Joined")
Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc)).Should(Succeed(), "Failed to retrieve internal member cluster %s in %s cluster", imc.Name, HubCluster.ClusterName)
wantMCStatus := v1alpha1.MemberClusterStatus{
AgentStatus: imc.Status.AgentStatus,
Conditions: mcJoinedConditions,
ResourceUsage: imc.Status.ResourceUsage,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus)
testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)
})

var _ = AfterSuite(func() {
Expand All @@ -235,34 +221,17 @@ var _ = AfterSuite(func() {

By("check if internal member cluster status is updated to Left")
wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcLeftAgentStatus}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff)
}
return nil
}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus)
testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, wantIMCStatus, imcStatusCmpOptions)

By("check if member cluster status is updated to Left")
Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc)).Should(Succeed(), "Failed to retrieve internal member cluster %s in %s cluster", imc.Name, HubCluster.ClusterName)
wantMCStatus := v1alpha1.MemberClusterStatus{
AgentStatus: imc.Status.AgentStatus,
Conditions: mcLeftConditions,
ResourceUsage: imc.Status.ResourceUsage,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus)
testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)

testutils.DeleteNamespace(*MemberCluster, memberNamespace)
testutils.DeleteNamespace(*HubCluster, workNamespace)
By("delete member cluster")
testutils.DeleteMemberCluster(ctx, *HubCluster, mc)
})
91 changes: 42 additions & 49 deletions test/e2e/join_leave_placement_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ package e2e

import (
"context"
"fmt"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
Expand All @@ -18,21 +17,21 @@ import (
"k8s.io/apimachinery/pkg/types"

"go.goms.io/fleet/apis/v1alpha1"
testutils "go.goms.io/fleet/test/e2e/utils"
"go.goms.io/fleet/test/e2e/utils"
)

// Serial - Ginkgo will guarantee that these specs will never run in parallel with other specs.
// Ordered - Ginkgo will guarantee that specs in an Ordered container will run sequentially, in the order they are written.
// This test cannot be run in parallel with other specs in the suite as it's leaving, joining, leaving and joining again.
var _ = Describe("workload orchestration testing with join/leave", Serial, Ordered, func() {
var _ = Describe("workload orchestration testing with join/leave", Serial, func() {
var (
crp *v1alpha1.ClusterResourcePlacement
ctx context.Context

mcStatusCmpOptions = []cmp.Option{
cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"),
cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"),
cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }),
cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}),
sortOption,
}
)

Expand All @@ -52,15 +51,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
AgentStatus: imcLeftAgentStatus,
Conditions: mcLeftConditions,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus)
utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)

By("create the resources to be propagated")
cr := &rbacv1.ClusterRole{
Expand All @@ -76,7 +67,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
},
},
}
testutils.CreateClusterRole(*HubCluster, cr)
Expect(HubCluster.KubeClient.Create(ctx, cr)).Should(Succeed(), "Failed to create cluster role %s in %s cluster", cr.Name, HubCluster.ClusterName)

By("create the cluster resource placement in the hub cluster")
crp = &v1alpha1.ClusterResourcePlacement{
Expand All @@ -96,12 +87,12 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
},
},
}
testutils.CreateClusterResourcePlacement(*HubCluster, crp)
Expect(HubCluster.KubeClient.Create(ctx, crp)).Should(Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, HubCluster.ClusterName)

By("verify the resource is not propagated to member cluster")
Consistently(func() bool {
return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr))
}, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName)
}, utils.PollTimeout, utils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName)

By("update member cluster in the hub cluster to join")
Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName)
Expand All @@ -113,18 +104,33 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
AgentStatus: imcJoinedAgentStatus,
Conditions: mcJoinedConditions,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus)
utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)

By("verify that the cluster resource placement is applied")
testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout)
crpStatus := v1alpha1.ClusterResourcePlacementStatus{
Conditions: []metav1.Condition{
{
Reason: "ScheduleSucceeded",
Status: metav1.ConditionTrue,
Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled),
},
{
Reason: "ApplySucceeded",
Status: metav1.ConditionTrue,
Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied),
},
},
SelectedResources: []v1alpha1.ResourceIdentifier{
{
Group: "rbac.authorization.k8s.io",
Version: "v1",
Kind: "ClusterRole",
Name: cr.Name,
},
},
TargetClusters: []string{"kind-member-testing"},
}
utils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*utils.PollTimeout)

By("verify the resource is propagated to member cluster")
Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName)
Expand All @@ -139,31 +145,26 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
AgentStatus: imcLeftAgentStatus,
Conditions: mcLeftConditions,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus)
utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)

By("verify that the resource is still on the member cluster")
Consistently(func() error {
return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)
}, utils.PollTimeout, utils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)

By("delete the crp from the hub")
testutils.DeleteClusterResourcePlacement(*HubCluster, crp)
utils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp)

By("verify that the resource is still on the member cluster")
Consistently(func() error {
return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr)
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)
}, utils.PollTimeout, utils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)

By("delete cluster role on hub cluster")
testutils.DeleteClusterRole(*HubCluster, cr)
Expect(HubCluster.KubeClient.Delete(ctx, cr)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", cr.Name, HubCluster.ClusterName)

By("delete cluster role on member cluster")
Expect(MemberCluster.KubeClient.Delete(ctx, cr)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", cr.Name, MemberCluster.ClusterName)

By("update member cluster in the hub cluster to join")
Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName)
Expand All @@ -175,14 +176,6 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
AgentStatus: imcJoinedAgentStatus,
Conditions: mcJoinedConditions,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus)
utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)
})
})
Loading

0 comments on commit 25b57b1

Please sign in to comment.