Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: Workload e2e #235

Merged
merged 21 commits into from
Aug 19, 2022
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 77 additions & 5 deletions test/e2e/framework/customresources.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,20 @@ import (

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
workapi "sigs.k8s.io/work-api/pkg/apis/v1alpha1"

"go.goms.io/fleet/apis/v1alpha1"
"go.goms.io/fleet/pkg/utils"
)

// MEMBER CLUSTER

// CreateMemberCluster creates MemberCluster in the hub cluster.
func CreateMemberCluster(cluster Cluster, mc *v1alpha1.MemberCluster) {
ginkgo.By(fmt.Sprintf("Creating MemberCluster(%s/%s)", mc.Namespace, mc.Name), func() {
ginkgo.By(fmt.Sprintf("Creating MemberCluster(%s)", mc.Name), func() {
err := cluster.KubeClient.Create(context.TODO(), mc)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
Expand Down Expand Up @@ -65,8 +67,6 @@ func WaitConditionMemberCluster(cluster Cluster, mc *v1alpha1.MemberCluster, con
}, customTimeout, PollInterval).Should(gomega.Equal(true))
}

// INTERNAL MEMBER CLUSTER

// WaitInternalMemberCluster waits for InternalMemberCluster to present on th hub cluster.
func WaitInternalMemberCluster(cluster Cluster, imc *v1alpha1.InternalMemberCluster) {
klog.Infof("Waiting for InternalMemberCluster(%s) to be synced in the %s cluster", imc.Name, cluster.ClusterName)
Expand All @@ -87,3 +87,75 @@ func WaitConditionInternalMemberCluster(cluster Cluster, imc *v1alpha1.InternalM
return cond != nil && cond.Status == status
}, customTimeout, PollInterval).Should(gomega.Equal(true))
}

// CreateClusterRole create cluster role in the hub cluster.
func CreateClusterRole(cluster Cluster, cr *rbacv1.ClusterRole) {
ginkgo.By(fmt.Sprintf("Creating ClusterRole (%s)", cr.Name), func() {
err := cluster.KubeClient.Create(context.TODO(), cr)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
}

// WaitClusterRole waits for cluster roles to be created.
func WaitClusterRole(cluster Cluster, cr *rbacv1.ClusterRole) {
klog.Infof("Waiting for ClusterRole(%s) to be synced", cr.Name)
gomega.Eventually(func() error {
err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: cr.Name, Namespace: ""}, cr)
return err
}, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred())
}

// DeleteClusterRole deletes cluster role on cluster.
func DeleteClusterRole(cluster Cluster, cr *rbacv1.ClusterRole) {
ginkgo.By(fmt.Sprintf("Deleting ClusterRole(%s)", cr.Name), func() {
err := cluster.KubeClient.Delete(context.TODO(), cr)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is clusterRole part of the common file?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved to test/e2e/utils/helper.go

// CreateClusterResourcePlacement created cluster resource placement on hub cluster.
func CreateClusterResourcePlacement(cluster Cluster, crp *v1alpha1.ClusterResourcePlacement) {
ginkgo.By(fmt.Sprintf("Creating ClusterResourcePlacement(%s)", crp.Name), func() {
err := cluster.KubeClient.Create(context.TODO(), crp)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
}

// WaitClusterResourcePlacement waits for ClusterResourcePlacement to present on th hub cluster.
func WaitClusterResourcePlacement(cluster Cluster, crp *v1alpha1.ClusterResourcePlacement) {
klog.Infof("Waiting for ClusterResourcePlacement(%s) to be synced", crp.Name)
gomega.Eventually(func() error {
err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: crp.Name, Namespace: ""}, crp)
return err
}, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred())
}

// WaitConditionClusterResourcePlacement waits for ClusterResourcePlacement to present on th hub cluster with a specific condition.
func WaitConditionClusterResourcePlacement(cluster Cluster, crp *v1alpha1.ClusterResourcePlacement, conditionName string, status metav1.ConditionStatus, customTimeout time.Duration) {
klog.Infof("Waiting for ClusterResourcePlacement(%s) condition(%s) status(%s) to be synced", crp.Name, conditionName, status)
gomega.Eventually(func() bool {
err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: crp.Name, Namespace: ""}, crp)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
cond := crp.GetCondition(conditionName)
return cond != nil && cond.Status == status
}, customTimeout, PollInterval).Should(gomega.Equal(true))
}

// DeleteClusterResourcePlacement is used delete cluster resource placement on the hub cluster.
func DeleteClusterResourcePlacement(cluster Cluster, crp *v1alpha1.ClusterResourcePlacement) {
controllerutil.RemoveFinalizer(crp, utils.PlacementFinalizer)
ginkgo.By(fmt.Sprintf("Deleting ClusterResourcePlacement(%s)", crp.Name), func() {
err := cluster.KubeClient.Delete(context.TODO(), crp)
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
})
}

// WaitWork waits for Work to be present on the hub cluster.
func WaitWork(cluster Cluster, workName, workNamespace string) {
var work workapi.Work
klog.Infof("Waiting for Work(%s/%s) to be synced", workName, workNamespace)
gomega.Eventually(func() error {
err := cluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: workName, Namespace: workNamespace}, &work)
return err
}, PollTimeout, PollInterval).ShouldNot(gomega.HaveOccurred())
}
7 changes: 6 additions & 1 deletion test/e2e/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,12 @@ import (
"go.goms.io/fleet/apis/v1alpha1"
)

func NewMemberCluster(name string, heartbeat int32, identity rbacv1.Subject, state v1alpha1.ClusterState) *v1alpha1.MemberCluster {
func NewMemberCluster(name string, heartbeat int32, state v1alpha1.ClusterState) *v1alpha1.MemberCluster {
identity := rbacv1.Subject{
Name: name,
Kind: "ServiceAccount",
Namespace: "fleet-system",
}
return &v1alpha1.MemberCluster{
ObjectMeta: v1.ObjectMeta{
Name: name,
Expand Down
105 changes: 47 additions & 58 deletions test/e2e/join_leave_member_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,98 +5,87 @@ Licensed under the MIT license.
package e2e

import (
"context"
Arvindthiru marked this conversation as resolved.
Show resolved Hide resolved
"fmt"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"golang.org/x/net/context"
"go.goms.io/fleet/pkg/utils"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should be with the local group

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed

corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"

"go.goms.io/fleet/apis/v1alpha1"
"go.goms.io/fleet/pkg/utils"
"go.goms.io/fleet/test/e2e/framework"
)

var _ = Describe("Join/leave member cluster testing", func() {
var mc *v1alpha1.MemberCluster
var sa *corev1.ServiceAccount
var memberIdentity rbacv1.Subject
var memberNS *corev1.Namespace
var imc *v1alpha1.InternalMemberCluster

memberNS = NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName))

Context("member clusters don't share member identity", func() {
BeforeEach(func() {
memberIdentity = rbacv1.Subject{
Name: MemberCluster.ClusterName,
Kind: "ServiceAccount",
Namespace: "fleet-system",
}
memberNS = NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName))
By("prepare resources in member cluster")
// create testing NS in member cluster
framework.CreateNamespace(*MemberCluster, memberNS)
framework.WaitNamespace(*MemberCluster, memberNS)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

consider group them together?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed

sa = NewServiceAccount(MemberCluster.ClusterName, memberNS.Name)
framework.CreateServiceAccount(*MemberCluster, sa)

By("deploy member cluster in the hub cluster")
mc = NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin)
framework.CreateMemberCluster(*HubCluster, mc)
framework.WaitMemberCluster(*HubCluster, mc)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed


By("check if member cluster is marked as readyToJoin")
framework.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionTrue, 3*framework.PollTimeout)

By("check if internal member cluster created in the hub cluster")
imc = NewInternalMemberCluster(MemberCluster.ClusterName, memberNS.Name)
framework.WaitInternalMemberCluster(*HubCluster, imc)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shouldn't the order be reversed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FIxed

})

It("Join flow is successful ", func() {
By("Prepare resources in member cluster", func() {
// create testing NS in member cluster
framework.CreateNamespace(*MemberCluster, memberNS)
framework.WaitNamespace(*MemberCluster, memberNS)

sa = NewServiceAccount(memberIdentity.Name, memberNS.Name)
framework.CreateServiceAccount(*MemberCluster, sa)
})

By("deploy memberCluster in the hub cluster", func() {
mc = NewMemberCluster(MemberCluster.ClusterName, 60, memberIdentity, v1alpha1.ClusterStateJoin)

framework.CreateMemberCluster(*HubCluster, mc)
framework.WaitMemberCluster(*HubCluster, mc)

By("check if memberCluster is marked as readyToJoin")
framework.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionTrue, 3*framework.PollTimeout)

By("check if internalmembercluster created in the hub cluster", func() {
imc = NewInternalMemberCluster(MemberCluster.ClusterName, memberNS.Name)
framework.WaitInternalMemberCluster(*HubCluster, imc)
})
})

By("check if membercluster condition is updated to Joined", func() {
framework.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoin, v1.ConditionTrue, 3*framework.PollTimeout)
})

By("check if internalMemberCluster condition is updated to Joined", func() {
framework.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, 3*framework.PollTimeout)
})
By("check if internal member cluster condition is updated to Joined")
framework.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, 3*framework.PollTimeout)

By("check if member cluster condition is updated to Joined")
framework.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoin, v1.ConditionTrue, 3*framework.PollTimeout)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

at this point, this IT has only 2 things, why not either fold it into beforeEach or merge with the leave test

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed

})

It("leave flow is successful ", func() {
By("check if member cluster condition is updated to Joined")
framework.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoin, v1.ConditionTrue, 3*framework.PollTimeout)

By("update membercluster in the hub cluster", func() {
framework.UpdateMemberClusterState(*HubCluster, mc, v1alpha1.ClusterStateLeave)
framework.WaitMemberCluster(*HubCluster, mc)
})
By("update member cluster in the hub cluster")
framework.UpdateMemberClusterState(*HubCluster, mc, v1alpha1.ClusterStateLeave)
framework.WaitMemberCluster(*HubCluster, mc)

By("check if internalMemberCluster condition is updated to Left", func() {
framework.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionFalse, 3*framework.PollTimeout)
})
By("check if internal member cluster condition is updated to Left")
framework.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionFalse, 3*framework.PollTimeout)

By("check if memberCluster is marked as notReadyToJoin")
By("check if member cluster is marked as notReadyToJoin")
framework.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterReadyToJoin, v1.ConditionFalse, 3*framework.PollTimeout)

By("check if membercluster condition is updated to Left")
By("check if member cluster condition is updated to Left")
framework.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoin, v1.ConditionFalse, 3*framework.PollTimeout)
})

By("member namespace is deleted from hub cluster", func() {
framework.DeleteMemberCluster(*HubCluster, mc)
Eventually(func() bool {
err := HubCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: memberNS.Name, Namespace: ""}, memberNS)
return apierrors.IsNotFound(err)
}, framework.PollTimeout, framework.PollInterval).Should(Equal(true))
})
AfterEach(func() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

normally, by convention, the AfterEach is put in the front. It's like defer

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved it below BeforeEach

framework.DeleteNamespace(*MemberCluster, memberNS)
Eventually(func() bool {
err := MemberCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: memberNS.Name, Namespace: ""}, memberNS)
return apierrors.IsNotFound(err)
}, framework.PollTimeout, framework.PollInterval).Should(Equal(true))
framework.DeleteMemberCluster(*HubCluster, mc)
Eventually(func() bool {
err := HubCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: memberNS.Name, Namespace: ""}, memberNS)
return apierrors.IsNotFound(err)
}, framework.PollTimeout, framework.PollInterval).Should(Equal(true))
})
})
})
120 changes: 120 additions & 0 deletions test/e2e/work_load_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package e2e

import (
"context"
"fmt"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"

"go.goms.io/fleet/apis/v1alpha1"
"go.goms.io/fleet/pkg/utils"
"go.goms.io/fleet/test/e2e/framework"
)

var _ = Describe("workload orchestration testing", func() {
var mc *v1alpha1.MemberCluster
var sa *corev1.ServiceAccount
var memberNS *corev1.Namespace
var imc *v1alpha1.InternalMemberCluster
var cr *rbacv1.ClusterRole
var crp *v1alpha1.ClusterResourcePlacement

BeforeEach(func() {
memberNS = NewNamespace(fmt.Sprintf(utils.NamespaceNameFormat, MemberCluster.ClusterName))
Copy link
Contributor Author

@Arvindthiru Arvindthiru Aug 18, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This needs to be setup here because once we create namespace resource version gets added, this resets it once we delete between tests.

By("prepare resources in member cluster")
// create testing NS in member cluster
framework.CreateNamespace(*MemberCluster, memberNS)
framework.WaitNamespace(*MemberCluster, memberNS)
sa = NewServiceAccount(MemberCluster.ClusterName, memberNS.Name)
framework.CreateServiceAccount(*MemberCluster, sa)

By("deploy member cluster in the hub cluster")
mc = NewMemberCluster(MemberCluster.ClusterName, 60, v1alpha1.ClusterStateJoin)
framework.CreateMemberCluster(*HubCluster, mc)
framework.WaitMemberCluster(*HubCluster, mc)

By("check if internal member cluster created in the hub cluster")
imc = NewInternalMemberCluster(MemberCluster.ClusterName, memberNS.Name)
framework.WaitInternalMemberCluster(*HubCluster, imc)

By("check if internal member cluster condition is updated to Joined")
framework.WaitConditionInternalMemberCluster(*HubCluster, imc, v1alpha1.AgentJoined, v1.ConditionTrue, 3*framework.PollTimeout)
By("check if member cluster condition is updated to Joined")
framework.WaitConditionMemberCluster(*HubCluster, mc, v1alpha1.ConditionTypeMemberClusterJoin, v1.ConditionTrue, 3*framework.PollTimeout)
})

It("Apply CRP and check if work gets propagated", func() {
workName := fmt.Sprintf(utils.WorkNameFormat, "resource-label-selector")
By("create the resources to be propagated")
cr = &rbacv1.ClusterRole{
ObjectMeta: v1.ObjectMeta{
Name: "test-cluster-role",
Labels: map[string]string{"fleet.azure.com/name": "test"},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this label should be extract as a var as it's used by the CRP

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added labelKey and labelValue vars

},
Rules: []rbacv1.PolicyRule{
{
Verbs: []string{"get", "list", "watch"},
APIGroups: []string{""},
Resources: []string{"secrets"},
},
},
}
framework.CreateClusterRole(*HubCluster, cr)

By("create the cluster resource placement in the hub cluster")
crp = &v1alpha1.ClusterResourcePlacement{
ObjectMeta: v1.ObjectMeta{Name: "resource-label-selector"},
Spec: v1alpha1.ClusterResourcePlacementSpec{
ResourceSelectors: []v1alpha1.ClusterResourceSelector{
{
Group: "rbac.authorization.k8s.io",
Version: "v1",
Kind: "ClusterRole",
LabelSelector: &v1.LabelSelector{
MatchLabels: map[string]string{"fleet.azure.com/name": "test"},
},
},
},
},
}
framework.CreateClusterResourcePlacement(*HubCluster, crp)
framework.WaitClusterResourcePlacement(*HubCluster, crp)

By("check if work gets created for cluster resource placement")
framework.WaitWork(*HubCluster, workName, memberNS.Name)

By("check if cluster resource placement is updated to Scheduled & Applied")
framework.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementConditionTypeScheduled), v1.ConditionTrue, 3*framework.PollTimeout)
framework.WaitConditionClusterResourcePlacement(*HubCluster, crp, string(v1alpha1.ResourcePlacementStatusConditionTypeApplied), v1.ConditionTrue, 3*framework.PollTimeout)

By("check if resource is propagated to member cluster")
framework.WaitClusterRole(*MemberCluster, cr)
ryanzhang-oss marked this conversation as resolved.
Show resolved Hide resolved

By("delete cluster resource placement & cluster role on hub cluster")
ryanzhang-oss marked this conversation as resolved.
Show resolved Hide resolved
framework.DeleteClusterResourcePlacement(*HubCluster, crp)
Eventually(func() bool {
err := HubCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: crp.Name, Namespace: ""}, crp)
return apierrors.IsNotFound(err)
}, framework.PollTimeout, framework.PollInterval).Should(Equal(true))
framework.DeleteClusterRole(*HubCluster, cr)
})

AfterEach(func() {
framework.DeleteMemberCluster(*HubCluster, mc)
ryanzhang-oss marked this conversation as resolved.
Show resolved Hide resolved
Eventually(func() bool {
err := HubCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: memberNS.Name, Namespace: ""}, memberNS)
return apierrors.IsNotFound(err)
}, framework.PollTimeout, framework.PollInterval).Should(Equal(true))
ryanzhang-oss marked this conversation as resolved.
Show resolved Hide resolved
framework.DeleteNamespace(*MemberCluster, memberNS)
Eventually(func() bool {
err := MemberCluster.KubeClient.Get(context.TODO(), types.NamespacedName{Name: memberNS.Name, Namespace: ""}, memberNS)
return apierrors.IsNotFound(err)
}, framework.PollTimeout, framework.PollInterval).Should(Equal(true))
})
})