Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

test: Refactor workload orchestration E2E part 1 #286

Merged
merged 20 commits into from
Oct 18, 2022
Merged

test: Refactor workload orchestration E2E part 1 #286

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
5da9144
Refactor workload E2E
Arvindthiru Sep 23, 2022
ca7c795
Dont create member cluster namespace
Arvindthiru Sep 23, 2022
5690b83
Status check setup helper methods
Arvindthiru Sep 28, 2022
9eb698c
Cmp namespace, role, rolebinding
Arvindthiru Sep 28, 2022
78ee2cb
Use helper methods for joinLeavePlacement
Arvindthiru Sep 29, 2022
d6af88e
Fix imports
Arvindthiru Sep 29, 2022
3189988
Add resourceIgnoreOption
Arvindthiru Sep 29, 2022
cf9508d
Removed some helper methods
Arvindthiru Sep 29, 2022
6b0e607
Add comment for gomega/ginkgo
Arvindthiru Sep 29, 2022
659da70
Lint fix
Arvindthiru Sep 29, 2022
110a5f4
Address comments 1
Arvindthiru Sep 30, 2022
2aae943
Fix import
Arvindthiru Sep 30, 2022
ad465db
New namespace resource
Arvindthiru Oct 1, 2022
bdb2313
Address comments
Arvindthiru Oct 17, 2022
0ac14eb
Remove comment
Arvindthiru Oct 17, 2022
1073968
Fix cmp option
Arvindthiru Oct 17, 2022
9adef01
Fix E2E test failure
Arvindthiru Oct 17, 2022
1d7226f
Fix import
Arvindthiru Oct 17, 2022
cb6d703
Add Delete Work in AfterEach
Arvindthiru Oct 18, 2022
553dc65
Remove comments
Arvindthiru Oct 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 10 additions & 41 deletions test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,16 @@ var (
cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"),
sortOption,
}

mcStatusCmpOptions = []cmp.Option{
cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"),
cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"),
cmpopts.IgnoreFields(v1alpha1.ResourceUsage{}, "ObservationTime"),
sortOption,
}
crpStatusCmpOptions = []cmp.Option{
cmpopts.IgnoreFields(metav1.Condition{}, "ObservedGeneration", "LastTransitionTime", "Message"),
sortOption,
}

imcJoinedAgentStatus = []v1alpha1.AgentStatus{
{
Expand Down Expand Up @@ -165,8 +168,6 @@ var _ = BeforeSuite(func() {
MemberCluster.HubURL = hubURL
framework.GetClusterClient(MemberCluster)

testutils.CreateNamespace(*HubCluster, workNamespace)

ctx = context.Background()

By("deploy member cluster in the hub cluster")
Expand Down Expand Up @@ -200,31 +201,16 @@ var _ = BeforeSuite(func() {

By("check if internal member cluster status is updated to Joined")
wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcJoinedAgentStatus}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff)
}
return nil
}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus)
testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, wantIMCStatus, imcStatusCmpOptions)

By("check if member cluster status is updated to Joined")
Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc)).Should(Succeed(), "Failed to retrieve internal member cluster %s in %s cluster", imc.Name, HubCluster.ClusterName)
wantMCStatus := v1alpha1.MemberClusterStatus{
AgentStatus: imc.Status.AgentStatus,
Conditions: mcJoinedConditions,
ResourceUsage: imc.Status.ResourceUsage,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus)
testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)
})

var _ = AfterSuite(func() {
Expand All @@ -235,34 +221,17 @@ var _ = AfterSuite(func() {

By("check if internal member cluster status is updated to Left")
wantIMCStatus := v1alpha1.InternalMemberClusterStatus{AgentStatus: imcLeftAgentStatus}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantIMCStatus, imc.Status, imcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("internal member cluster(%s) status mismatch (-want +got):\n%s", imc.Name, statusDiff)
}
return nil
}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", imc.Name, wantIMCStatus)
testutils.CheckInternalMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, wantIMCStatus, imcStatusCmpOptions)

By("check if member cluster status is updated to Left")
Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: imc.Name, Namespace: imc.Namespace}, imc)).Should(Succeed(), "Failed to retrieve internal member cluster %s in %s cluster", imc.Name, HubCluster.ClusterName)
wantMCStatus := v1alpha1.MemberClusterStatus{
AgentStatus: imc.Status.AgentStatus,
Conditions: mcLeftConditions,
ResourceUsage: imc.Status.ResourceUsage,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, 3*testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus)
testutils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)

testutils.DeleteNamespace(*MemberCluster, memberNamespace)
testutils.DeleteNamespace(*HubCluster, workNamespace)
By("delete member cluster")
testutils.DeleteMemberCluster(ctx, *HubCluster, mc)
})
91 changes: 42 additions & 49 deletions test/e2e/join_leave_placement_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ package e2e

import (
"context"
"fmt"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
Expand All @@ -18,21 +17,21 @@ import (
"k8s.io/apimachinery/pkg/types"

"go.goms.io/fleet/apis/v1alpha1"
testutils "go.goms.io/fleet/test/e2e/utils"
"go.goms.io/fleet/test/e2e/utils"
)

// Serial - Ginkgo will guarantee that these specs will never run in parallel with other specs.
// Ordered - Ginkgo will guarantee that specs in an Ordered container will run sequentially, in the order they are written.
// This test cannot be run in parallel with other specs in the suite as it's leaving, joining, leaving and joining again.
var _ = Describe("workload orchestration testing with join/leave", Serial, Ordered, func() {
var _ = Describe("workload orchestration testing with join/leave", Serial, func() {
var (
crp *v1alpha1.ClusterResourcePlacement
ctx context.Context

mcStatusCmpOptions = []cmp.Option{
cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration"),
cmpopts.IgnoreFields(v1alpha1.AgentStatus{}, "LastReceivedHeartbeat"),
cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}), cmpopts.SortSlices(func(ref1, ref2 metav1.Condition) bool { return ref1.Type < ref2.Type }),
cmpopts.IgnoreTypes(v1alpha1.ResourceUsage{}),
sortOption,
}
)

Expand All @@ -52,15 +51,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
AgentStatus: imcLeftAgentStatus,
Conditions: mcLeftConditions,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus)
utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)

By("create the resources to be propagated")
cr := &rbacv1.ClusterRole{
Expand All @@ -76,7 +67,7 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
},
},
}
testutils.CreateClusterRole(*HubCluster, cr)
Expect(HubCluster.KubeClient.Create(ctx, cr)).Should(Succeed(), "Failed to create cluster role %s in %s cluster", cr.Name, HubCluster.ClusterName)

By("create the cluster resource placement in the hub cluster")
crp = &v1alpha1.ClusterResourcePlacement{
Expand All @@ -96,12 +87,12 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
},
},
}
testutils.CreateClusterResourcePlacement(*HubCluster, crp)
Expect(HubCluster.KubeClient.Create(ctx, crp)).Should(Succeed(), "Failed to create cluster resource placement %s in %s cluster", crp.Name, HubCluster.ClusterName)

By("verify the resource is not propagated to member cluster")
Consistently(func() bool {
return apierrors.IsNotFound(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr))
}, testutils.PollTimeout, testutils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName)
}, utils.PollTimeout, utils.PollInterval).Should(BeTrue(), "Failed to verify cluster role %s is not propagated to %s cluster", cr.Name, MemberCluster.ClusterName)

By("update member cluster in the hub cluster to join")
Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName)
Expand All @@ -113,18 +104,33 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
AgentStatus: imcJoinedAgentStatus,
Conditions: mcJoinedConditions,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus)
utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)

By("verify that the cluster resource placement is applied")
testutils.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), metav1.ConditionTrue, testutils.PollTimeout)
crpStatus := v1alpha1.ClusterResourcePlacementStatus{
Conditions: []metav1.Condition{
{
Reason: "ScheduleSucceeded",
Status: metav1.ConditionTrue,
Type: string(v1alpha1.ResourcePlacementConditionTypeScheduled),
},
{
Reason: "ApplySucceeded",
Status: metav1.ConditionTrue,
Type: string(v1alpha1.ResourcePlacementStatusConditionTypeApplied),
},
},
SelectedResources: []v1alpha1.ResourceIdentifier{
{
Group: "rbac.authorization.k8s.io",
Version: "v1",
Kind: "ClusterRole",
Name: cr.Name,
},
},
TargetClusters: []string{"kind-member-testing"},
}
utils.WaitCreateClusterResourcePlacementStatus(ctx, *HubCluster, &types.NamespacedName{Name: crp.Name}, crpStatus, crpStatusCmpOptions, 3*utils.PollTimeout)

By("verify the resource is propagated to member cluster")
Expect(MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)).Should(Succeed(), "Failed to verify cluster role %s is propagated to %s cluster", cr.Name, MemberCluster.ClusterName)
Expand All @@ -139,31 +145,26 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
AgentStatus: imcLeftAgentStatus,
Conditions: mcLeftConditions,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for internal member cluster %s to have status %s", mc.Name, wantMCStatus)
utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)

By("verify that the resource is still on the member cluster")
Consistently(func() error {
return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name}, cr)
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)
}, utils.PollTimeout, utils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)

By("delete the crp from the hub")
testutils.DeleteClusterResourcePlacement(*HubCluster, crp)
utils.DeleteClusterResourcePlacement(ctx, *HubCluster, crp)

By("verify that the resource is still on the member cluster")
Consistently(func() error {
return MemberCluster.KubeClient.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: ""}, cr)
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)
}, utils.PollTimeout, utils.PollInterval).Should(Succeed(), "Failed to verify cluster role %s is still on %s cluster", cr.Name, MemberCluster.ClusterName)

By("delete cluster role on hub cluster")
testutils.DeleteClusterRole(*HubCluster, cr)
Expect(HubCluster.KubeClient.Delete(ctx, cr)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", cr.Name, HubCluster.ClusterName)

By("delete cluster role on member cluster")
Expect(MemberCluster.KubeClient.Delete(ctx, cr)).Should(Succeed(), "Failed to delete cluster role %s in %s cluster", cr.Name, MemberCluster.ClusterName)

By("update member cluster in the hub cluster to join")
Expect(HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc)).Should(Succeed(), "Failed to retrieve member cluster %s in %s cluster", mc.Name, HubCluster.ClusterName)
Expand All @@ -175,14 +176,6 @@ var _ = Describe("workload orchestration testing with join/leave", Serial, Order
AgentStatus: imcJoinedAgentStatus,
Conditions: mcJoinedConditions,
}
Eventually(func() error {
if err := HubCluster.KubeClient.Get(ctx, types.NamespacedName{Name: mc.Name}, mc); err != nil {
return err
}
if statusDiff := cmp.Diff(wantMCStatus, mc.Status, mcStatusCmpOptions...); statusDiff != "" {
return fmt.Errorf("member cluster(%s) status mismatch (-want +got):\n%s", mc.Name, statusDiff)
}
return nil
}, testutils.PollTimeout, testutils.PollInterval).Should(Succeed(), "Failed to wait for member cluster %s to have status %s", mc.Name, wantMCStatus)
utils.CheckMemberClusterStatus(ctx, *HubCluster, &types.NamespacedName{Name: mc.Name}, wantMCStatus, mcStatusCmpOptions)
})
})
Loading