From 786f57166485abd7df76f3db51c75799be69e1a9 Mon Sep 17 00:00:00 2001 From: "anisha.kj" Date: Mon, 4 May 2020 12:44:15 +0530 Subject: [PATCH] added ECS TLS secret into general pravega tls section Signed-off-by: anisha.kj --- doc/tier2.md | 139 ++++++++++++++---- .../pravega/v1alpha1/pravegacluster_types.go | 8 + pkg/controller/pravega/constants.go | 2 + .../pravega/pravega_segmentstore.go | 21 +++ .../pravega/pravega_segmentstore_test.go | 3 + 5 files changed, 143 insertions(+), 30 deletions(-) diff --git a/doc/tier2.md b/doc/tier2.md index df2a2b151..bc18b8eb6 100644 --- a/doc/tier2.md +++ b/doc/tier2.md @@ -112,37 +112,116 @@ $ kubectl create -f pvc.yaml Pravega can also use an S3-compatible storage backend such as [Dell EMC ECS](https://www.dellemc.com/sr-me/storage/ecs/index.htm) as Tier 2. -Create a file with the secret definition containing your access and secret keys. +1. Create a file with the secret definition containing your access and secret keys. + + ``` + apiVersion: v1 + kind: Secret + metadata: + name: ecs-credentials + type: Opaque + stringData: + ACCESS_KEY_ID: QWERTY@ecstestdrive.emc.com + SECRET_KEY: 0123456789 + ``` + +2. Assuming that the file is named `ecs-credentials.yaml`. + ``` + $ kubectl create -f ecs-credentials.yaml + ``` +3. Follow the [instructions to deploy Pravega manually](manual-installation.md#install-the-pravega-cluster-manually) and configure the LongTermStorage block in your `PravegaCluster` manifest with your ECS connection details and a reference to the secret above. + ``` + ... + spec: + tier2: + ecs: + configUri: http://10.247.10.52:9020?namespace=pravega + bucket: "shared" + prefix: "example" + credentials: ecs-credentials + ``` + +#### (Optional) ECS HTTPS/TLS Support on Kubernetes +Pravega connects ECS endpoint through OpenJDK based HTTP or HTTPS, so by default Pravega as an HTTPS client is configured to verify ECS server certificate. + +The ECS server certificate, or its signing CA's certificate, must present in OpenJDK's Truststore, for Pravega to establish HTTPS/TLS connection with ECS endpoint. + +Refer to the steps below to add ECS server certificate or CA's certificate into OpenJDK's Truststore: + +1. Retrieve CA certificate or the server certificate as file, e.g. "ecs-certificate.pem". + +2. Load the certificate into Kubernetes secret: + ``` + kubectl create secret generic ecs-cert --from-file ./ecs-certificate.pem + ``` + or create a file directly to contain the certificate content: + ``` + apiVersion: v1 + kind: Secret + metadata: + name: "ecs-cert" + type: Opaque + data: + ecs-certificate.pem: QmFnIEF0dH......JpYnV0ZLS0tLQo= + ``` + Assuming the above file is named `ecs-tls.yaml`, then create secret using the above file. + ``` + $ kubectl create -f ecs-tls.yaml + ``` + +3. In Pravega manifest, add the secret name defined above into "tls/static/caBundle" section. + ``` + ... + kind: "PravegaCluster" + metadata: + name: "example" + spec: + tls: + static: + caBundle: "ecs-cert" + ... + tier2: + ecs: + configUri: https://10.247.10.52:9021?namespace=pravega + bucket: "shared" + prefix: "example" + credentials: ecs-credentials + ``` +4. Pravega operator then mounts caBundle onto folder "/etc/secret-volume/ca-bundle" in container. + +5. Pravega Segmentstore container adds certificates found under "/etc/secret-volume/ca-bundle" into the default OpenJDK Truststore, in order to establish HTTPS/TLS connection with ECS. + +#### Update ECS Credentials + +There might be an operational need to update ECS credentials for a running Pravega cluster, where the following steps could be taken: + +1. Modify Segmentstore configmap, find "EXTENDEDS3_CONFIGURI", and then replace "secretKey" and/or "identity" with new values + ``` + $ kubectl edit configmap pravega-pravega-segmentstore + ``` + + ``` + ... + EXTENDEDS3_BUCKET: shared + EXTENDEDS3_CONFIGURI: https://10.243.86.64:9021?namespace=namespace%26identity=oldUser%26secretKey=oldPassword + EXTENDEDS3_PREFIX: example + ... + ``` +2. Delete all (running) Segmentstore pod(s) in one of the two approaches below: + ``` + $ kubectl delete po -l component=pravega-segmentstore + ``` + + ``` + $ kubectl delete po pravega-pravega-segmentstore-1 + $ kubectl delete po pravega-pravega-segmentstore-2 + $ kubectl delete po pravega-pravega-segmentstore-3 + ... + ``` + As StatefulSet, new Segementstore pods will be automatically created with the new ECS credentials, since the default upgrade strategy of Segmentstore is `OnDelete` instead of `RollingUpdate`. + + Since ECS supports grace period when both old and new credentials are accepted, Pravega service is technically uninterrupted during the above process. -``` -apiVersion: v1 -kind: Secret -metadata: - name: ecs-credentials -type: Opaque -stringData: - ACCESS_KEY_ID: QWERTY@ecstestdrive.emc.com - SECRET_KEY: 0123456789 -``` - -Assuming that the file is named `ecs-credentials.yaml`. - -``` -$ kubectl create -f ecs-credentials.yaml -``` - -Follow the [instructions to deploy Pravega manually](manual-installation.md#install-the-pravega-cluster-manually) and configure the Tier 2 block in your `PravegaCluster` manifest with your ECS connection details and a reference to the secret above. - -``` -... -spec: - tier2: - ecs: - configUri: http://10.247.10.52:9020?namespace=pravega - bucket: "shared" - prefix: "example" - credentials: ecs-credentials -``` ### Use HDFS as Tier 2 diff --git a/pkg/apis/pravega/v1alpha1/pravegacluster_types.go b/pkg/apis/pravega/v1alpha1/pravegacluster_types.go index 0d5a8f745..498e9f746 100644 --- a/pkg/apis/pravega/v1alpha1/pravegacluster_types.go +++ b/pkg/apis/pravega/v1alpha1/pravegacluster_types.go @@ -184,6 +184,7 @@ type TLSPolicy struct { type StaticTLS struct { ControllerSecret string `json:"controllerSecret,omitempty"` SegmentStoreSecret string `json:"segmentStoreSecret,omitempty"` + CaBundle string `json:"caBundle,omitempty"` } func (tp *TLSPolicy) IsSecureController() bool { @@ -200,6 +201,13 @@ func (tp *TLSPolicy) IsSecureSegmentStore() bool { return len(tp.Static.SegmentStoreSecret) != 0 } +func (tp *TLSPolicy) IsCaBundlePresent() bool { + if tp == nil || tp.Static == nil { + return false + } + return len(tp.Static.CaBundle) != 0 +} + type AuthenticationParameters struct { // Enabled specifies whether or not authentication is enabled // By default, authentication is not enabled diff --git a/pkg/controller/pravega/constants.go b/pkg/controller/pravega/constants.go index 97b8ec284..d19c51ce6 100644 --- a/pkg/controller/pravega/constants.go +++ b/pkg/controller/pravega/constants.go @@ -18,6 +18,8 @@ const ( segmentStoreKind = "pravega-segmentstore" tlsVolumeName = "tls-secret" tlsMountDir = "/etc/secret-volume" + caBundleVolumeName = "ca-bundle" + caBundleMountDir = "/etc/secret-volume/ca-bundle" heapDumpName = "heap-dump" heapDumpDir = "/tmp/dumpfile/heap" authVolumeName = "auth-passwd-secret" diff --git a/pkg/controller/pravega/pravega_segmentstore.go b/pkg/controller/pravega/pravega_segmentstore.go index fa7f78122..3cdb764b1 100644 --- a/pkg/controller/pravega/pravega_segmentstore.go +++ b/pkg/controller/pravega/pravega_segmentstore.go @@ -153,6 +153,8 @@ func makeSegmentstorePodSpec(p *api.PravegaCluster) corev1.PodSpec { configureSegmentstoreTLSSecret(&podSpec, p) + configureCaBundleSecret(&podSpec, p) + configureTier2Filesystem(&podSpec, p.Spec.Pravega) return podSpec @@ -324,6 +326,25 @@ func configureSegmentstoreTLSSecret(podSpec *corev1.PodSpec, p *api.PravegaClust } } +func configureCaBundleSecret(podSpec *corev1.PodSpec, p *api.PravegaCluster) { + if p.Spec.TLS.IsCaBundlePresent() { + vol := corev1.Volume{ + Name: caBundleVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: p.Spec.TLS.Static.CaBundle, + }, + }, + } + podSpec.Volumes = append(podSpec.Volumes, vol) + + podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: caBundleVolumeName, + MountPath: caBundleMountDir, + }) + } +} + func MakeSegmentStoreHeadlessService(pravegaCluster *api.PravegaCluster) *corev1.Service { return &corev1.Service{ TypeMeta: metav1.TypeMeta{ diff --git a/pkg/controller/pravega/pravega_segmentstore_test.go b/pkg/controller/pravega/pravega_segmentstore_test.go index d69c024ac..5658c3424 100644 --- a/pkg/controller/pravega/pravega_segmentstore_test.go +++ b/pkg/controller/pravega/pravega_segmentstore_test.go @@ -118,6 +118,7 @@ var _ = Describe("PravegaSegmentstore", func() { Static: &v1alpha1.StaticTLS{ ControllerSecret: "controller-secret", SegmentStoreSecret: "segmentstore-secret", + CaBundle: "ecs-cert", }, }, Authentication: &v1alpha1.AuthenticationParameters{ @@ -238,6 +239,7 @@ var _ = Describe("PravegaSegmentstore", func() { Static: &v1alpha1.StaticTLS{ ControllerSecret: "controller-secret", SegmentStoreSecret: "segmentstore-secret", + CaBundle: "ecs-cert", }, }, Authentication: &v1alpha1.AuthenticationParameters{ @@ -356,6 +358,7 @@ var _ = Describe("PravegaSegmentstore", func() { Static: &v1alpha1.StaticTLS{ ControllerSecret: "controller-secret", SegmentStoreSecret: "segmentstore-secret", + CaBundle: "ecs-cert", }, }, Authentication: &v1alpha1.AuthenticationParameters{