diff --git a/README.md b/README.md index 0fb731c64..6f74555b5 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,8 @@ http://-pravega-controller.:10080/ Check out the [external access documentation](doc/external-access.md) if your clients need to connect to Pravega from outside Kubernetes. +Check out the [exposing Segmentstore service on single IP address ](https://github.com/pravega/pravega-operator/blob/4aa88641c3d5a1d5afbb2b9e628846639fd13290/doc/external-access.md#exposing-segmentstore-service-on-single-ip-address-and-different-ports) if your clients need to connect to Pravega Segment store on the same IP address from outside Kubernetes. + ### Scale a Pravega cluster You can scale Pravega components independently by modifying their corresponding field in the Pravega resource spec. You can either `kubectl edit` the cluster or `kubectl patch` it. If you edit it, update the number of replicas for BookKeeper, Controller, and/or Segment Store and save the updated spec. diff --git a/charts/pravega-operator/templates/crd.yaml b/charts/pravega-operator/templates/crd.yaml index e1f4c3ad0..1e50e21e2 100644 --- a/charts/pravega-operator/templates/crd.yaml +++ b/charts/pravega-operator/templates/crd.yaml @@ -38,7 +38,7 @@ spec: conversionReviewVersions: ["v1beta1", "v1alpha1"] strategy: Webhook webhookClientConfig: - caBundle: {{ .Values.webhookCert.crt }} + caBundle: {{ .Values.webhookCert.crt }} service: name: pravega-webhook-svc namespace: {{ .Release.Namespace }} @@ -79,6 +79,8 @@ spec: static: type: object properties: + caBundle: + type: string controllerSecret: type: string segmentStoreSecret: @@ -267,6 +269,10 @@ spec: additionalProperties: type: string type: object + segmentStoreLoadBalancerIP: + type: string + segmentStoreExternalTrafficPolicy: + type: string status: type: object description: PravegaCluster Status defines the observed state of PravegaCluster diff --git a/charts/pravega/README.md b/charts/pravega/README.md index 06af19e78..a2c20d555 100644 --- a/charts/pravega/README.md +++ b/charts/pravega/README.md @@ -70,6 +70,8 @@ The following table lists the configurable parameters of the Pravega chart and t | `segmentStore.resources.limits.memory` | Memory limits for segmentStore | `4Gi` | | `segmentStore.service.type` | Override the segmentStore service type, if external access is enabled (LoadBalancer/NodePort) | | | `segmentStore.service.annotations` | Annotations to add to the segmentStore service, if external access is enabled | `{}` | +| `segmentStore.service.segmentStoreLoadBalancerIP` |It is used to provide a LoadBalancerIP | | +| `segmentStore.service.segmentStoreExternalTrafficPolicy` | It is used to provide segmentStoreExternalTrafficPolicy | | | `segmentStore.jvmOptions` | JVM Options for segmentStore | `[]` | | `storage.longtermStorage.type` | Type of long term storage backend to be used (filesystem/ecs/hdfs) | `filesystem` | | `storage.longtermStorage.filesystem.pvc` | Name of the pre-created PVC, if long term storage type is filesystem | `pravega-tier2` | diff --git a/charts/pravega/templates/pravega.yaml b/charts/pravega/templates/pravega.yaml index 7e774da15..ddc80f1a9 100644 --- a/charts/pravega/templates/pravega.yaml +++ b/charts/pravega/templates/pravega.yaml @@ -33,6 +33,10 @@ spec: {{- if .Values.externalAccess.enabled }} controllerServiceAccountName: {{ .Values.serviceAccount.name }} segmentStoreServiceAccountName: {{ .Values.serviceAccount.name }} + {{- if .Values.segmentStore.service.segmentStoreLoadBalancerIP }} + segmentStoreLoadBalancerIP: {{ .Values.segmentStore.service.segmentStoreLoadBalancerIP }} + {{- end }} + segmentStoreExternalTrafficPolicy: {{ .Values.segmentStore.service.segmentStoreExternalTrafficPolicy }} {{- if .Values.controller.service.type }} controllerExtServiceType: {{ .Values.controller.service.type }} {{- end }} diff --git a/charts/pravega/values.yaml b/charts/pravega/values.yaml index e940b640c..dc6a90cd4 100644 --- a/charts/pravega/values.yaml +++ b/charts/pravega/values.yaml @@ -73,6 +73,8 @@ segmentStore: ## used to override the service type for segmentStore type: annotations: {} + segmentStoreLoadBalancerIP: + segmentStoreExternalTrafficPolicy: Local jvmOptions: ["-Xmx2g", "-XX:MaxDirectMemorySize=2g"] storage: diff --git a/deploy/crds/crd.yaml b/deploy/crds/crd.yaml index fb92cf6a6..92e54ae14 100644 --- a/deploy/crds/crd.yaml +++ b/deploy/crds/crd.yaml @@ -76,6 +76,8 @@ spec: static: type: object properties: + caBundle: + type: string controllerSecret: type: string segmentStoreSecret: @@ -264,6 +266,10 @@ spec: additionalProperties: type: string type: object + segmentStoreLoadBalancerIP: + type: string + segmentStoreExternalTrafficPolicy: + type: string status: type: object description: PravegaCluster Status defines the observed state of PravegaCluster diff --git a/doc/external-access.md b/doc/external-access.md index cd357e519..ee7f8c542 100644 --- a/doc/external-access.md +++ b/doc/external-access.md @@ -238,3 +238,31 @@ IP: 10.100.200.183 LoadBalancer Ingress: 10.247.108.104 . . . ``` +# Exposing Segmentstore Service on single IP address and Different ports + +For Exposing SegmentStoreservices on the same I/P address we will use MetalLB. +MetalLB hooks into Kubernetes cluster, and provides a network load-balancer implementation. In short, it allows to create Kubernetes services of type “LoadBalancer” in clusters that don’t run on a cloud provider and thus cannot simply hook into paid products to provide load-balancers. + +By default, Services do not share an IP address, for providing same IP address to all the services we need to set the following configurations while creating the External Service: + +1) Provide annotation key as **metallb.universe.tf/allow-shared-ip** for all the services. + +2) All the services which want to share the IP address need to have the same value for the above annotation, for example "shared-ss-ip". + +3) The port for all the services should be different + +4) All the services should use External Traffic Policy as Cluster + +5) Finally, we need to provide the I/P address that we want our service to provide to the segment store pod as spec.loadBalancerIP while creating the service + +To enable this we need to provide segmentStoreSvcAnnotations, segmentStoreLoadBalancerIP, segmentStoreExternalTrafficPolicy in the manifest as shown below + +Example: +``` +pravega: + . . . + segmentStoreLoadBalancerIP: "10.243.39.103" + segmentStoreExternalTrafficPolicy: "cluster" + segmentStoreSvcAnnotations: + metallb.universe.tf/allow-shared-ip: "shared-ss-ip" +``` diff --git a/go.mod b/go.mod index 17847a10c..9d36b1363 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( 4d63.com/gochecknoinits v0.0.0-20200108094044-eb73b47b9fc4 // indirect github.com/alecthomas/gocyclo v0.0.0-20150208221726-aa8f8b160214 // indirect github.com/alexkohler/nakedret v1.0.0 // indirect + github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf // indirect github.com/hashicorp/go-version v1.1.0 github.com/jgautheron/goconst v0.0.0-20200227150835-cda7ea3bf591 // indirect github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f // indirect diff --git a/go.sum b/go.sum index 005f9c54c..60d04c37c 100644 --- a/go.sum +++ b/go.sum @@ -400,6 +400,8 @@ github.com/gophercloud/gophercloud v0.6.0 h1:Xb2lcqZtml1XjgYZxbeayEemq7ASbeTp09m github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbGkcYwAR7EZK2WMqM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf h1:vc7Dmrk4JwS0ZPS6WZvWlwDflgDTA26jItmbSj83nug= +github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= diff --git a/pkg/apis/pravega/v1beta1/pravega.go b/pkg/apis/pravega/v1beta1/pravega.go index f34bf637f..5315242d0 100644 --- a/pkg/apis/pravega/v1beta1/pravega.go +++ b/pkg/apis/pravega/v1beta1/pravega.go @@ -147,6 +147,12 @@ type PravegaSpec struct { // Annotations to be added to the external service SegmentStoreServiceAnnotations map[string]string `json:"segmentStoreSvcAnnotations"` + + // Specifying this IP would ensure we use same IP address for all the ss services + SegmentStoreLoadBalancerIP string `json:"segmentStoreLoadBalancerIP,omitempty"` + + // SegmentStoreExternalTrafficPolicy defines the ExternalTrafficPolicy it can have cluster or local + SegmentStoreExternalTrafficPolicy string `json:"segmentStoreExternalTrafficPolicy,omitempty"` } func (s *PravegaSpec) withDefaults() (changed bool) { diff --git a/pkg/controller/pravega/pravega_segmentstore.go b/pkg/controller/pravega/pravega_segmentstore.go index e465786c4..a9e6a90cc 100644 --- a/pkg/controller/pravega/pravega_segmentstore.go +++ b/pkg/controller/pravega/pravega_segmentstore.go @@ -450,20 +450,16 @@ func generateDNSAnnotationForSvc(domainName string, podName string) (dnsAnnotati func MakeSegmentStoreExternalServices(p *api.PravegaCluster) []*corev1.Service { var service *corev1.Service - serviceType := getSSServiceType(p) services := make([]*corev1.Service, p.Spec.Pravega.SegmentStoreReplicas) - for i := int32(0); i < p.Spec.Pravega.SegmentStoreReplicas; i++ { ssPodName := p.ServiceNameForSegmentStore(i) annotationMap := p.Spec.Pravega.SegmentStoreServiceAnnotations annotationValue := generateDNSAnnotationForSvc(p.Spec.ExternalAccess.DomainName, ssPodName) - if annotationValue != "" { annotationMap = cloneMap(p.Spec.Pravega.SegmentStoreServiceAnnotations) annotationMap[externalDNSAnnotationKey] = annotationValue } - service = &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -491,6 +487,15 @@ func MakeSegmentStoreExternalServices(p *api.PravegaCluster) []*corev1.Service { }, }, } + if strings.EqualFold(p.Spec.Pravega.SegmentStoreExternalTrafficPolicy, "Cluster") == true { + service.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeCluster + } else { + service.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal + } + if p.Spec.Pravega.SegmentStoreLoadBalancerIP != "" { + service.Spec.Ports[0].Port = 12345 + i + service.Spec.LoadBalancerIP = p.Spec.Pravega.SegmentStoreLoadBalancerIP + } services[i] = service } return services diff --git a/pkg/controller/pravega/pravega_segmentstore_test.go b/pkg/controller/pravega/pravega_segmentstore_test.go index f08417b6a..89622b4ac 100644 --- a/pkg/controller/pravega/pravega_segmentstore_test.go +++ b/pkg/controller/pravega/pravega_segmentstore_test.go @@ -262,6 +262,7 @@ var _ = Describe("PravegaSegmentstore", func() { It("should set external access service type to LoadBalancer", func() { Ω(p.Spec.ExternalAccess.Type).Should(Equal(corev1.ServiceTypeClusterIP)) }) + }) }) @@ -377,6 +378,24 @@ var _ = Describe("PravegaSegmentstore", func() { Ω(err).Should(BeNil()) }) }) + Context("Create External service with SegmentStoreExternalTrafficPolicy as cluster", func() { + BeforeEach(func() { + p.Spec.Pravega.SegmentStoreExternalTrafficPolicy = "cluster" + }) + It("should create external service with ExternalTrafficPolicy type cluster", func() { + svc := pravega.MakeSegmentStoreExternalServices(p) + Ω(svc[0].Spec.ExternalTrafficPolicy).To(Equal(corev1.ServiceExternalTrafficPolicyTypeCluster)) + }) + }) + Context("Create External service with LoadBalancerIP", func() { + BeforeEach(func() { + p.Spec.Pravega.SegmentStoreLoadBalancerIP = "10.240.12.18" + }) + It("should create external service with LoadBalancerIP", func() { + svc := pravega.MakeSegmentStoreExternalServices(p) + Ω(svc[0].Spec.LoadBalancerIP).To(Equal("10.240.12.18")) + }) + }) Context("Create External service with external service type empty", func() { BeforeEach(func() { m := make(map[string]string)