From 78d99444549be1060e6883e9dea13c75b74c9fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Moreno?= Date: Fri, 22 Mar 2019 15:18:18 +0100 Subject: [PATCH] Issue 130: Ability to specify container resources (#129) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Specify container resources for testing Signed-off-by: Adrián Moreno * Set Xmx and Xms Java opts to SS and C Signed-off-by: Adrián Moreno * Store BK index directory in a PVC Signed-off-by: Adrián Moreno * enable custom resource configuration Signed-off-by: wenqi * Generate deepcopy and go fmt Signed-off-by: wenqi * Fix golang errors Signed-off-by: Adrián Moreno * Adjust resource values Signed-off-by: Adrián Moreno * Set Xmx value based on memory limit Signed-off-by: Adrián Moreno * Reduce default resource consumption Signed-off-by: Adrián Moreno * Further reduce resources Signed-off-by: Adrián Moreno * Add unit test to check resource is working Signed-off-by: wenqi * Refractor test Signed-off-by: wenqi * Use literals instead of constants on unit test assertions Signed-off-by: Adrián Moreno * Wait after cluster destoryed Signed-off-by: wenqi * Go fmt Signed-off-by: wenqi * Wait for PVCs to be deleted when destroying a cluster Signed-off-by: Adrián Moreno --- example/cr-detailed.yaml | 22 ++ pkg/apis/pravega/v1alpha1/bookkeeper.go | 31 +++ pkg/apis/pravega/v1alpha1/pravega.go | 61 +++++ .../pravega/v1alpha1/zz_generated.deepcopy.go | 20 ++ pkg/controller/pravega/bookie.go | 35 ++- pkg/controller/pravega/pravega_controller.go | 21 +- .../pravega/pravega_segmentstore.go | 48 ++-- .../pravegacluster_controller_test.go | 225 ++++++++++++++++++ pkg/test/e2e/e2eutil/pravegacluster_util.go | 24 ++ 9 files changed, 432 insertions(+), 55 deletions(-) create mode 100644 pkg/controller/pravegacluster/pravegacluster_controller_test.go diff --git a/example/cr-detailed.yaml b/example/cr-detailed.yaml index 8ae6ccd7d..051675b0d 100644 --- a/example/cr-detailed.yaml +++ b/example/cr-detailed.yaml @@ -12,6 +12,13 @@ spec: pullPolicy: IfNotPresent replicas: 3 + resources: + requests: + memory: "3Gi" + cpu: "1000m" + limits: + memory: "5Gi" + cpu: "2000m" storage: ledgerVolumeClaimTemplate: @@ -51,7 +58,22 @@ spec: pravega: controllerReplicas: 1 + controllerResources: + requests: + memory: "1Gi" + cpu: "1000m" + limits: + memory: "3Gi" + cpu: "2000m" + segmentStoreReplicas: 3 + segmentStoreResources: + requests: + memory: "3Gi" + cpu: "1000m" + limits: + memory: "5Gi" + cpu: "2000m" # Turn on Pravega Debug Logging debugLogging: false diff --git a/pkg/apis/pravega/v1alpha1/bookkeeper.go b/pkg/apis/pravega/v1alpha1/bookkeeper.go index 77a4262e6..ee35d4442 100644 --- a/pkg/apis/pravega/v1alpha1/bookkeeper.go +++ b/pkg/apis/pravega/v1alpha1/bookkeeper.go @@ -46,6 +46,18 @@ const ( // MinimumBookkeeperReplicas is the minimum number of Bookkeeper replicas // accepted MinimumBookkeeperReplicas = 3 + + // DefaultBookkeeperRequestCPU is the default CPU request for BookKeeper + DefaultBookkeeperRequestCPU = "500m" + + // DefaultBookkeeperLimitCPU is the default CPU limit for BookKeeper + DefaultBookkeeperLimitCPU = "1" + + // DefaultBookkeeperRequestMemory is the default memory request for BookKeeper + DefaultBookkeeperRequestMemory = "1Gi" + + // DefaultBookkeeperLimitMemory is the limit memory limit for BookKeeper + DefaultBookkeeperLimitMemory = "2Gi" ) // BookkeeperSpec defines the configuration of BookKeeper @@ -68,6 +80,10 @@ type BookkeeperSpec struct { // ServiceAccountName configures the service account used on BookKeeper instances ServiceAccountName string `json:"serviceAccountName,omitempty"` + // BookieResources specifies the request and limit of resources that bookie can have. + // BookieResources includes CPU and memory resources + Resources *v1.ResourceRequirements `json:"resources,omitempty"` + // Options is the Bookkeeper configuration that is to override the bk_server.conf // in bookkeeper. Some examples can be found here // https://github.com/apache/bookkeeper/blob/master/docker/README.md @@ -102,9 +118,24 @@ func (s *BookkeeperSpec) withDefaults() (changed bool) { s.AutoRecovery = &boolTrue } + if s.Resources == nil { + changed = true + s.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(DefaultBookkeeperRequestCPU), + v1.ResourceMemory: resource.MustParse(DefaultBookkeeperRequestMemory), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(DefaultBookkeeperLimitCPU), + v1.ResourceMemory: resource.MustParse(DefaultBookkeeperLimitMemory), + }, + } + } + if s.Options == nil { s.Options = map[string]string{} } + return changed } diff --git a/pkg/apis/pravega/v1alpha1/pravega.go b/pkg/apis/pravega/v1alpha1/pravega.go index 625ec2d75..37231e34e 100644 --- a/pkg/apis/pravega/v1alpha1/pravega.go +++ b/pkg/apis/pravega/v1alpha1/pravega.go @@ -45,6 +45,30 @@ const ( // DefaultSegmentStoreReplicas is the default number of replicas for the Pravega // Segment Store component DefaultSegmentStoreReplicas = 1 + + // DefaultControllerRequestCPU is the default CPU request for Pravega + DefaultControllerRequestCPU = "250m" + + // DefaultControllerLimitCPU is the default CPU limit for Pravega + DefaultControllerLimitCPU = "500m" + + // DefaultControllerRequestMemory is the default memory request for Pravega + DefaultControllerRequestMemory = "512Mi" + + // DefaultControllerLimitMemory is the default memory limit for Pravega + DefaultControllerLimitMemory = "1Gi" + + // DefaultSegmentStoreRequestCPU is the default CPU request for Pravega + DefaultSegmentStoreRequestCPU = "500m" + + // DefaultSegmentStoreLimitCPU is the default CPU limit for Pravega + DefaultSegmentStoreLimitCPU = "1" + + // DefaultSegmentStoreRequestMemory is the default memory request for Pravega + DefaultSegmentStoreRequestMemory = "1Gi" + + // DefaultSegmentStoreLimitMemory is the default memory limit for Pravega + DefaultSegmentStoreLimitMemory = "2Gi" ) // PravegaSpec defines the configuration of Pravega @@ -87,6 +111,14 @@ type PravegaSpec struct { // SegmentStoreServiceAccountName configures the service account used on segment store instances. // If not specified, Kubernetes will automatically assign the default service account in the namespace SegmentStoreServiceAccountName string `json:"segmentStoreServiceAccountName,omitempty"` + + // ControllerResources specifies the request and limit of resources that controller can have. + // ControllerResources includes CPU and memory resources + ControllerResources *v1.ResourceRequirements `json:"controllerResources,omitempty"` + + // SegmentStoreResources specifies the request and limit of resources that segmentStore can have. + // SegmentStoreResources includes CPU and memory resources + SegmentStoreResources *v1.ResourceRequirements `json:"segmentStoreResources,omitempty"` } func (s *PravegaSpec) withDefaults() (changed bool) { @@ -129,10 +161,39 @@ func (s *PravegaSpec) withDefaults() (changed bool) { changed = true s.Tier2 = &Tier2Spec{} } + if s.Tier2.withDefaults() { changed = true } + if s.ControllerResources == nil { + changed = true + s.ControllerResources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(DefaultControllerRequestCPU), + v1.ResourceMemory: resource.MustParse(DefaultControllerRequestMemory), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(DefaultControllerLimitCPU), + v1.ResourceMemory: resource.MustParse(DefaultControllerLimitMemory), + }, + } + } + + if s.SegmentStoreResources == nil { + changed = true + s.SegmentStoreResources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(DefaultSegmentStoreRequestCPU), + v1.ResourceMemory: resource.MustParse(DefaultSegmentStoreRequestMemory), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(DefaultSegmentStoreLimitCPU), + v1.ResourceMemory: resource.MustParse(DefaultSegmentStoreLimitMemory), + }, + } + } + return changed } diff --git a/pkg/apis/pravega/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pravega/v1alpha1/zz_generated.deepcopy.go index 8ffb4c8ff..c8d1ab098 100644 --- a/pkg/apis/pravega/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pravega/v1alpha1/zz_generated.deepcopy.go @@ -60,6 +60,11 @@ func (in *BookkeeperSpec) DeepCopyInto(out *BookkeeperSpec) { *out = new(bool) **out = **in } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } if in.Options != nil { in, out := &in.Options, &out.Options *out = make(map[string]string, len(*in)) @@ -93,6 +98,11 @@ func (in *BookkeeperStorageSpec) DeepCopyInto(out *BookkeeperStorageSpec) { *out = new(v1.PersistentVolumeClaimSpec) (*in).DeepCopyInto(*out) } + if in.IndexVolumeClaimTemplate != nil { + in, out := &in.IndexVolumeClaimTemplate, &out.IndexVolumeClaimTemplate + *out = new(v1.PersistentVolumeClaimSpec) + (*in).DeepCopyInto(*out) + } return } @@ -389,6 +399,16 @@ func (in *PravegaSpec) DeepCopyInto(out *PravegaSpec) { *out = new(Tier2Spec) (*in).DeepCopyInto(*out) } + if in.ControllerResources != nil { + in, out := &in.ControllerResources, &out.ControllerResources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + if in.SegmentStoreResources != nil { + in, out := &in.SegmentStoreResources, &out.SegmentStoreResources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/controller/pravega/bookie.go b/pkg/controller/pravega/bookie.go index f245236c9..a9970177d 100644 --- a/pkg/controller/pravega/bookie.go +++ b/pkg/controller/pravega/bookie.go @@ -12,13 +12,13 @@ package pravega import ( "fmt" + "strings" "github.com/pravega/pravega-operator/pkg/apis/pravega/v1alpha1" "github.com/pravega/pravega-operator/pkg/util" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -122,16 +122,7 @@ func makeBookiePodSpec(clusterName string, bookkeeperSpec *v1alpha1.BookkeeperSp MountPath: "/bk/index", }, }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1000m"), - corev1.ResourceMemory: resource.MustParse("3Gi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("2000m"), - corev1.ResourceMemory: resource.MustParse("5Gi"), - }, - }, + Resources: *bookkeeperSpec.Resources, ReadinessProbe: &corev1.Probe{ Handler: corev1.Handler{ Exec: &corev1.ExecAction{ @@ -193,11 +184,31 @@ func makeBookieVolumeClaimTemplates(spec *v1alpha1.BookkeeperSpec) []corev1.Pers } func MakeBookieConfigMap(pravegaCluster *v1alpha1.PravegaCluster) *corev1.ConfigMap { + javaOpts := []string{ + "-Xms1g", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseCGroupMemoryLimitForHeap", + "-XX:MaxRAMFraction=1", + "-XX:MaxDirectMemorySize=1g", + "-XX:+UseG1GC", + "-XX:MaxGCPauseMillis=10", + "-XX:+ParallelRefProcEnabled", + "-XX:+AggressiveOpts", + "-XX:+DoEscapeAnalysis", + "-XX:ParallelGCThreads=32", + "-XX:ConcGCThreads=32", + "-XX:G1NewSizePercent=50", + "-XX:+DisableExplicitGC", + "-XX:-ResizePLAB", + } + configData := map[string]string{ - "BK_BOOKIE_EXTRA_OPTS": "-Xms1g -Xmx4g -XX:MaxDirectMemorySize=1g -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB", + "BK_BOOKIE_EXTRA_OPTS": strings.Join(javaOpts, " "), "ZK_URL": pravegaCluster.Spec.ZookeeperUri, // Set useHostNameAsBookieID to false until BookKeeper Docker // image is updated to 4.7 + // This value can be explicitly overridden when using the operator + // with images based on BookKeeper 4.7 or newer "BK_useHostNameAsBookieID": "false", "PRAVEGA_CLUSTER_NAME": pravegaCluster.ObjectMeta.Name, "WAIT_FOR": pravegaCluster.Spec.ZookeeperUri, diff --git a/pkg/controller/pravega/pravega_controller.go b/pkg/controller/pravega/pravega_controller.go index 6bcbed2a7..ac2f52622 100644 --- a/pkg/controller/pravega/pravega_controller.go +++ b/pkg/controller/pravega/pravega_controller.go @@ -20,7 +20,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -79,16 +78,7 @@ func makeControllerPodSpec(name string, pravegaSpec *api.PravegaSpec) *corev1.Po }, }, }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1000m"), - corev1.ResourceMemory: resource.MustParse("1Gi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("2000m"), - corev1.ResourceMemory: resource.MustParse("3Gi"), - }, - }, + Resources: *pravegaSpec.ControllerResources, ReadinessProbe: &corev1.Probe{ Handler: corev1.Handler{ Exec: &corev1.ExecAction{ @@ -127,7 +117,10 @@ func makeControllerPodSpec(name string, pravegaSpec *api.PravegaSpec) *corev1.Po func MakeControllerConfigMap(p *api.PravegaCluster) *corev1.ConfigMap { var javaOpts = []string{ - "-Xms1g -Xmx2g -Dpravegaservice.clusterName=" + p.Name, + "-Xms1g", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseCGroupMemoryLimitForHeap", + "-Dpravegaservice.clusterName=" + p.Name, } for name, value := range p.Spec.Pravega.Options { @@ -147,10 +140,6 @@ func MakeControllerConfigMap(p *api.PravegaCluster) *corev1.ConfigMap { "WAIT_FOR": p.Spec.ZookeeperUri, } - for name, value := range p.Spec.Pravega.Options { - configData[name] = value - } - configMap := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", diff --git a/pkg/controller/pravega/pravega_segmentstore.go b/pkg/controller/pravega/pravega_segmentstore.go index ab3e67243..2ac35f894 100644 --- a/pkg/controller/pravega/pravega_segmentstore.go +++ b/pkg/controller/pravega/pravega_segmentstore.go @@ -20,7 +20,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -99,16 +98,7 @@ func makeSegmentstorePodSpec(pravegaCluster *api.PravegaCluster) corev1.PodSpec MountPath: cacheVolumeMountPoint, }, }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1000m"), - corev1.ResourceMemory: resource.MustParse("3Gi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("2000m"), - corev1.ResourceMemory: resource.MustParse("5Gi"), - }, - }, + Resources: *pravegaSpec.SegmentStoreResources, ReadinessProbe: &corev1.Probe{ Handler: corev1.Handler{ Exec: &corev1.ExecAction{ @@ -151,44 +141,48 @@ func makeSegmentstorePodSpec(pravegaCluster *api.PravegaCluster) corev1.PodSpec return podSpec } -func MakeSegmentstoreConfigMap(pravegaCluster *api.PravegaCluster) *corev1.ConfigMap { +func MakeSegmentstoreConfigMap(p *api.PravegaCluster) *corev1.ConfigMap { javaOpts := []string{ - "-Xms1g -Xmx4g -XX:MaxDirectMemorySize=1g -Dpravegaservice.clusterName=" + pravegaCluster.Name, + "-Xms1g", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseCGroupMemoryLimitForHeap", + "-XX:MaxRAMFraction=1", + "-Dpravegaservice.clusterName=" + p.Name, } - for name, value := range pravegaCluster.Spec.Pravega.Options { + for name, value := range p.Spec.Pravega.Options { javaOpts = append(javaOpts, fmt.Sprintf("-D%v=%v", name, value)) } configData := map[string]string{ "AUTHORIZATION_ENABLED": "false", - "CLUSTER_NAME": pravegaCluster.Name, - "ZK_URL": pravegaCluster.Spec.ZookeeperUri, + "CLUSTER_NAME": p.Name, + "ZK_URL": p.Spec.ZookeeperUri, "JAVA_OPTS": strings.Join(javaOpts, " "), - "CONTROLLER_URL": util.PravegaControllerServiceURL(*pravegaCluster), + "CONTROLLER_URL": util.PravegaControllerServiceURL(*p), } // Wait for at least 3 Bookies to come up var waitFor []string - for i := int32(0); i < util.Min(3, pravegaCluster.Spec.Bookkeeper.Replicas); i++ { + for i := int32(0); i < util.Min(3, p.Spec.Bookkeeper.Replicas); i++ { waitFor = append(waitFor, fmt.Sprintf("%s-%d.%s.%s:3181", - util.StatefulSetNameForBookie(pravegaCluster.Name), + util.StatefulSetNameForBookie(p.Name), i, - util.HeadlessServiceNameForBookie(pravegaCluster.Name), - pravegaCluster.Namespace)) + util.HeadlessServiceNameForBookie(p.Name), + p.Namespace)) } configData["WAIT_FOR"] = strings.Join(waitFor, ",") - if pravegaCluster.Spec.ExternalAccess.Enabled { + if p.Spec.ExternalAccess.Enabled { configData["K8_EXTERNAL_ACCESS"] = "true" } - if pravegaCluster.Spec.Pravega.DebugLogging { + if p.Spec.Pravega.DebugLogging { configData["log.level"] = "DEBUG" } - for k, v := range getTier2StorageOptions(pravegaCluster.Spec.Pravega) { + for k, v := range getTier2StorageOptions(p.Spec.Pravega) { configData[k] = v } @@ -198,9 +192,9 @@ func MakeSegmentstoreConfigMap(pravegaCluster *api.PravegaCluster) *corev1.Confi APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: util.ConfigMapNameForSegmentstore(pravegaCluster.Name), - Namespace: pravegaCluster.Namespace, - Labels: util.LabelsForSegmentStore(pravegaCluster), + Name: util.ConfigMapNameForSegmentstore(p.Name), + Namespace: p.Namespace, + Labels: util.LabelsForSegmentStore(p), }, Data: configData, } diff --git a/pkg/controller/pravegacluster/pravegacluster_controller_test.go b/pkg/controller/pravegacluster/pravegacluster_controller_test.go new file mode 100644 index 000000000..10d0dc563 --- /dev/null +++ b/pkg/controller/pravegacluster/pravegacluster_controller_test.go @@ -0,0 +1,225 @@ +/** + * Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +package pravegacluster + +import ( + "context" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/pravega/pravega-operator/pkg/apis/pravega/v1alpha1" + "github.com/pravega/pravega-operator/pkg/util" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestBookie(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Pravega cluster") +} + +var _ = Describe("PravegaCluster Controller", func() { + const ( + Name = "example" + Namespace = "default" + ) + + var ( + s = scheme.Scheme + r *ReconcilePravegaCluster + ) + + Context("Reconcile", func() { + var ( + req reconcile.Request + p *v1alpha1.PravegaCluster + ) + + BeforeEach(func() { + req = reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: Name, + Namespace: Namespace, + }, + } + p = &v1alpha1.PravegaCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: Name, + Namespace: Namespace, + }, + } + s.AddKnownTypes(v1alpha1.SchemeGroupVersion, p) + }) + + Context("Default spec", func() { + var ( + client client.Client + err error + ) + + BeforeEach(func() { + p.WithDefaults() + client = fake.NewFakeClient(p) + r = &ReconcilePravegaCluster{client: client, scheme: s} + _, err = r.Reconcile(req) + }) + + It("shouldn't error", func() { + Ω(err).Should(BeNil()) + }) + + Context("Default bookkeeper", func() { + It("should have a default bookie resource", func() { + foundBk := &appsv1.StatefulSet{} + nn := types.NamespacedName{ + Name: util.StatefulSetNameForBookie(p.Name), + Namespace: Namespace, + } + err = client.Get(context.TODO(), nn, foundBk) + Ω(err).Should(BeNil()) + Ω(foundBk.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String()).Should(Equal("500m")) + Ω(foundBk.Spec.Template.Spec.Containers[0].Resources.Requests.Memory().String()).Should(Equal("1Gi")) + Ω(foundBk.Spec.Template.Spec.Containers[0].Resources.Limits.Cpu().String()).Should(Equal("1")) + Ω(foundBk.Spec.Template.Spec.Containers[0].Resources.Limits.Memory().String()).Should(Equal("2Gi")) + }) + }) + + Context("Default Pravega controller", func() { + It("should have a default controller resource", func() { + foundController := &appsv1.Deployment{} + nn := types.NamespacedName{ + Name: util.DeploymentNameForController(p.Name), + Namespace: Namespace, + } + err = client.Get(context.TODO(), nn, foundController) + Ω(err).Should(BeNil()) + Ω(foundController.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String()).Should(Equal("250m")) + Ω(foundController.Spec.Template.Spec.Containers[0].Resources.Requests.Memory().String()).Should(Equal("512Mi")) + Ω(foundController.Spec.Template.Spec.Containers[0].Resources.Limits.Cpu().String()).Should(Equal("500m")) + Ω(foundController.Spec.Template.Spec.Containers[0].Resources.Limits.Memory().String()).Should(Equal("1Gi")) + }) + }) + + Context("Default Pravega segmentstore", func() { + It("should have a default controller resource", func() { + foundSS := &appsv1.StatefulSet{} + nn := types.NamespacedName{ + Name: util.StatefulSetNameForSegmentstore(p.Name), + Namespace: Namespace, + } + err = client.Get(context.TODO(), nn, foundSS) + Ω(err).Should(BeNil()) + Ω(foundSS.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String()).Should(Equal("500m")) + Ω(foundSS.Spec.Template.Spec.Containers[0].Resources.Requests.Memory().String()).Should(Equal("1Gi")) + Ω(foundSS.Spec.Template.Spec.Containers[0].Resources.Limits.Cpu().String()).Should(Equal("1")) + Ω(foundSS.Spec.Template.Spec.Containers[0].Resources.Limits.Memory().String()).Should(Equal("2Gi")) + }) + }) + }) + + Context("Custom spec", func() { + var ( + client client.Client + err error + customReq *corev1.ResourceRequirements + ) + + BeforeEach(func() { + customReq = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + corev1.ResourceMemory: resource.MustParse("6Gi"), + }, + } + p.Spec = v1alpha1.ClusterSpec{ + Bookkeeper: &v1alpha1.BookkeeperSpec{ + Resources: customReq, + }, + Pravega: &v1alpha1.PravegaSpec{ + ControllerResources: customReq, + SegmentStoreResources: customReq, + }, + } + p.WithDefaults() + client = fake.NewFakeClient(p) + r = &ReconcilePravegaCluster{client: client, scheme: s} + _, err = r.Reconcile(req) + }) + + It("shouldn't error", func() { + Ω(err).Should(BeNil()) + }) + + Context("Custom bookkeeper", func() { + It("should have a custom bookie resource", func() { + foundBK := &appsv1.StatefulSet{} + nn := types.NamespacedName{ + Name: util.StatefulSetNameForBookie(p.Name), + Namespace: Namespace, + } + err = client.Get(context.TODO(), nn, foundBK) + Ω(err).Should(BeNil()) + Ω(foundBK.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String()).Should(Equal("2")) + Ω(foundBK.Spec.Template.Spec.Containers[0].Resources.Requests.Memory().String()).Should(Equal("4Gi")) + Ω(foundBK.Spec.Template.Spec.Containers[0].Resources.Limits.Cpu().String()).Should(Equal("4")) + Ω(foundBK.Spec.Template.Spec.Containers[0].Resources.Limits.Memory().String()).Should(Equal("6Gi")) + }) + }) + + Context("Custom Pravega controller", func() { + It("should have a custom controller resource", func() { + foundController := &appsv1.Deployment{} + nn := types.NamespacedName{ + Name: util.DeploymentNameForController(p.Name), + Namespace: Namespace, + } + err = client.Get(context.TODO(), nn, foundController) + Ω(err).Should(BeNil()) + Ω(foundController.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String()).Should(Equal("2")) + Ω(foundController.Spec.Template.Spec.Containers[0].Resources.Requests.Memory().String()).Should(Equal("4Gi")) + Ω(foundController.Spec.Template.Spec.Containers[0].Resources.Limits.Cpu().String()).Should(Equal("4")) + Ω(foundController.Spec.Template.Spec.Containers[0].Resources.Limits.Memory().String()).Should(Equal("6Gi")) + }) + }) + + Context("Custom Pravega segmentstore", func() { + It("should have a custom segmentstore resource", func() { + foundSS := &appsv1.StatefulSet{} + nn := types.NamespacedName{ + Name: util.StatefulSetNameForSegmentstore(p.Name), + Namespace: Namespace, + } + err = client.Get(context.TODO(), nn, foundSS) + Ω(err).Should(BeNil()) + Ω(foundSS.Spec.Template.Spec.Containers[0].Resources.Requests.Cpu().String()).Should(Equal("2")) + Ω(foundSS.Spec.Template.Spec.Containers[0].Resources.Requests.Memory().String()).Should(Equal("4Gi")) + Ω(foundSS.Spec.Template.Spec.Containers[0].Resources.Limits.Cpu().String()).Should(Equal("4")) + Ω(foundSS.Spec.Template.Spec.Containers[0].Resources.Limits.Memory().String()).Should(Equal("6Gi")) + }) + }) + }) + }) +}) diff --git a/pkg/test/e2e/e2eutil/pravegacluster_util.go b/pkg/test/e2e/e2eutil/pravegacluster_util.go index 6fc2daf53..71c8acfcc 100644 --- a/pkg/test/e2e/e2eutil/pravegacluster_util.go +++ b/pkg/test/e2e/e2eutil/pravegacluster_util.go @@ -121,6 +121,7 @@ func WaitForClusterToTerminate(t *testing.T, f *framework.Framework, ctx *framew LabelSelector: labels.SelectorFromSet(util.LabelsForPravegaCluster(p)).String(), } + // Wait for Pods to terminate err := wait.Poll(RetryInterval, 2*time.Minute, func() (done bool, err error) { podList, err := f.KubeClient.Core().Pods(p.Namespace).List(listOptions) if err != nil { @@ -143,6 +144,29 @@ func WaitForClusterToTerminate(t *testing.T, f *framework.Framework, ctx *framew return err } + // Wait for PVCs to terminate + err = wait.Poll(RetryInterval, 1*time.Minute, func() (done bool, err error) { + pvcList, err := f.KubeClient.Core().PersistentVolumeClaims(p.Namespace).List(listOptions) + if err != nil { + return false, err + } + + var names []string + for i := range pvcList.Items { + pvc := &pvcList.Items[i] + names = append(names, pvc.Name) + } + t.Logf("waiting for pvc to terminate (%v)", names) + if len(names) != 0 { + return false, nil + } + return true, nil + }) + + if err != nil { + return err + } + t.Logf("pravega cluster terminated: %s", p.Name) return nil }