feat(kubernetes): Improve the security of the kubernetes/helm charts (#…

…1782) * 1747 | remove obsolete yaml files * 1747 | remove configmap and its hardcoded references * 1747 | add missing input parameter of neo4j.host * 1747 | remove obsolete secrets and parameterize the rest * 1747 | auto-generate gms secret * 1747 | remove fullName overrides * 1747 | fix parameters in subchart's values.yaml * 1747 | remove hardcoding from parameters for gms host and port * 1747 | upgrade chart version * 1747 | update helm docs * 1747 | add extraEnv, extraVolume and extraMounts * 1747 | Alters pull policy of images to 'always' for ldh Co-authored-by: shakti-garg <[email protected]>
datahub-project · Aug 20, 2020 · 236d5e6 · 236d5e6
1 parent ece9b82
commit 236d5e6
Show file tree

Hide file tree

Showing 30 changed files with 252 additions and 502 deletions.
diff --git a/contrib/kubernetes/README.md b/contrib/kubernetes/README.md
@@ -31,48 +31,10 @@ The following table lists the configuration parameters and its default values
 
 | Repository | Name | Version |
 |------------|------|---------|
-| file://./charts/datahub-frontend | datahub-frontend | 0.1.0 |
-| file://./charts/datahub-gms | datahub-gms | 0.1.0 |
-| file://./charts/datahub-mae-consumer | datahub-mae-consumer | 0.1.0 |
-| file://./charts/datahub-mce-consumer | datahub-mce-consumer | 0.1.0 |
-
-#### Chart Values
-
-| Key | Type | Default | Description |
-|-----|------|---------|-------------|
-| datahub-frontend.enabled | bool | `true` |  |
-| datahub-frontend.image.repository | string | `"linkedin/datahub-frontend"` |  |
-| datahub-frontend.image.tag | string | `"latest"` |  |
-| datahub-gms.enabled | bool | `true` |  |
-| datahub-gms.image.repository | string | `"linkedin/datahub-gms"` |  |
-| datahub-gms.image.tag | string | `"latest"` |  |
-| datahub-mae-consumer.enabled | bool | `true` |  |
-| datahub-mae-consumer.image.repository | string | `"linkedin/datahub-mae-consumer"` |  |
-| datahub-mae-consumer.image.tag | string | `"latest"` |  |
-| datahub-mce-consumer.enabled | bool | `true` |  |
-| datahub-mce-consumer.image.repository | string | `"linkedin/datahub-mce-consumer"` |  |
-| datahub-mce-consumer.image.tag | string | `"latest"` |  |
-| global.datahub.appVersion | string | `"1.0"` |  |
-| global.datahub.gms.host | string | `"datahub-gms-deployment"` |  |
-| global.datahub.gms.port | string | `"8080"` |  |
-| global.datahub.gms.secret | string | `"YouKnowNothing"` |  |
-| global.elasticsearch.host | string | `"elasticsearch"` |  |
-| global.elasticsearch.port | string | `"9200"` |  |
-| global.hostAliases[0].hostnames[0] | string | `"broker"` |  |
-| global.hostAliases[0].hostnames[1] | string | `"mysql"` |  |
-| global.hostAliases[0].hostnames[2] | string | `"elasticsearch"` |  |
-| global.hostAliases[0].hostnames[3] | string | `"neo4j"` |  |
-| global.hostAliases[0].ip | string | `"192.168.0.104"` |  |
-| global.kafka.bootstrap.server | string | `"broker:29092"` |  |
-| global.kafka.schemaregistry.url | string | `"http://schema-registry:8081"` |  |
-| global.neo4j.password | string | `"datahub"` |  |
-| global.neo4j.uri | string | `"bolt://neo4j"` |  |
-| global.neo4j.username | string | `"neo4j"` |  |
-| global.sql.datasource.driver | string | `"com.mysql.jdbc.Driver"` |  |
-| global.sql.datasource.host | string | `"mysql"` |  |
-| global.sql.datasource.password | string | `"datahub"` |  |
-| global.sql.datasource.url | string | `"jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false\u0026useSSL=true"` |  |
-| global.sql.datasource.username | string | `"datahub"` |  |
+| file://./charts/datahub-frontend | datahub-frontend | 0.2.0 |
+| file://./charts/datahub-gms | datahub-gms | 0.2.0 |
+| file://./charts/datahub-mae-consumer | datahub-mae-consumer | 0.2.0 |
+| file://./charts/datahub-mce-consumer | datahub-mce-consumer | 0.2.0 |
 
 ## Install DataHub
 Navigate to the current directory and run the below command.  Update the `datahub/values.yaml` file with valid hostname/IP address configuration for elasticsearch, neo4j, schema-registry, broker & mysql. 

diff --git a/contrib/kubernetes/datahub/Chart.yaml b/contrib/kubernetes/datahub/Chart.yaml
@@ -4,24 +4,24 @@ description: A Helm chart for LinkedIn DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.0.1
+version: 0.1.0
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: latest #0.3.1
 dependencies:
   - name: datahub-gms
-    version: 0.1.0
+    version: 0.2.0
     repository: file://./charts/datahub-gms
     condition: datahub-gms.enabled
   - name: datahub-frontend
-    version: 0.1.0
+    version: 0.2.0
     repository: file://./charts/datahub-frontend
     condition: datahub-frontend.enabled
   - name: datahub-mae-consumer
-    version: 0.1.0
+    version: 0.2.0
     repository: file://./charts/datahub-mae-consumer
     condition: datahub-mae-consumer.enabled
   - name: datahub-mce-consumer
-    version: 0.1.0
+    version: 0.2.0
     repository: file://./charts/datahub-mce-consumer
     condition: datahub-mce-consumer.enabled
diff --git a/contrib/kubernetes/datahub/README.md b/contrib/kubernetes/datahub/README.md
@@ -2,7 +2,7 @@ datahub
 =======
 A Helm chart for LinkedIn DataHub
 
-Current chart version is `0.0.1`
+Current chart version is `0.1.0`
 
 ## Chart Requirements
 
@@ -13,7 +13,7 @@ Current chart version is `0.0.1`
 | file://./charts/datahub-mae-consumer | datahub-mae-consumer | 0.1.0 |
 | file://./charts/datahub-mce-consumer | datahub-mce-consumer | 0.1.0 |
 
-## Chart Values
+#### Chart Values
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
@@ -30,9 +30,7 @@ Current chart version is `0.0.1`
 | datahub-mce-consumer.image.repository | string | `"linkedin/datahub-mce-consumer"` |  |
 | datahub-mce-consumer.image.tag | string | `"latest"` |  |
 | global.datahub.appVersion | string | `"1.0"` |  |
-| global.datahub.gms.host | string | `"datahub-gms-deployment"` |  |
 | global.datahub.gms.port | string | `"8080"` |  |
-| global.datahub.gms.secret | string | `"YouKnowNothing"` |  |
 | global.elasticsearch.host | string | `"elasticsearch"` |  |
 | global.elasticsearch.port | string | `"9200"` |  |
 | global.hostAliases[0].hostnames[0] | string | `"broker"` |  |
@@ -42,11 +40,14 @@ Current chart version is `0.0.1`
 | global.hostAliases[0].ip | string | `"192.168.0.104"` |  |
 | global.kafka.bootstrap.server | string | `"broker:29092"` |  |
 | global.kafka.schemaregistry.url | string | `"http://schema-registry:8081"` |  |
-| global.neo4j.password | string | `"datahub"` |  |
+| global.neo4j.host | string | `"neo4j:7474"` |  |
 | global.neo4j.uri | string | `"bolt://neo4j"` |  |
 | global.neo4j.username | string | `"neo4j"` |  |
+| global.neo4j.password.secretRef | string | `"neo4j-secrets"` |  |
+| global.neo4j.password.secretKey | string | `"neo4j-password"` |  |
 | global.sql.datasource.driver | string | `"com.mysql.jdbc.Driver"` |  |
 | global.sql.datasource.host | string | `"mysql"` |  |
-| global.sql.datasource.password | string | `"datahub"` |  |
 | global.sql.datasource.url | string | `"jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false\u0026useSSL=true"` |  |
 | global.sql.datasource.username | string | `"datahub"` |  |
+| global.sql.datasource.password.secretRef | string | `"mysql-secrets"` |  |
+| global.sql.datasource.password.secretKey | string | `"mysql-password"` |  |
diff --git a/contrib/kubernetes/datahub/charts/datahub-frontend/Chart.yaml b/contrib/kubernetes/datahub/charts/datahub-frontend/Chart.yaml
@@ -14,7 +14,7 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.1.0
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.

diff --git a/contrib/kubernetes/datahub/charts/datahub-frontend/README.md b/contrib/kubernetes/datahub/charts/datahub-frontend/README.md
@@ -2,18 +2,19 @@ datahub-frontend
 ================
 A Helm chart for datahub-frontend
 
-Current chart version is `0.1.0`
+Current chart version is `0.2.0`
 
 ## Chart Values
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` |  |
 | datahub.play.mem.buffer.size | string | `"10MB"` |  |
+| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` |
+| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" |
+| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" |
 | fullnameOverride | string | `"datahub-frontend"` |  |
-| global.datahub.gms.host | string | `"datahub-gms-deployment"` |  |
 | global.datahub.gms.port | string | `"8080"` |  |
-| global.datahub.gms.secret | string | `"YouKnowNothing"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"linkedin/datahub-frontend"` |  |
 | image.tag | string | `"latest"` |  |

diff --git a/contrib/kubernetes/datahub/charts/datahub-frontend/templates/deployment.yaml b/contrib/kubernetes/datahub/charts/datahub-frontend/templates/deployment.yaml
@@ -21,6 +21,10 @@ spec:
       serviceAccountName: {{ include "datahub-frontend.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      volumes:
+      {{- if .Values.extraVolumes }}
+        {{ toYaml .Values.extraVolumes | indent 8 }}
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
@@ -41,18 +45,25 @@ spec:
               port: http
           env:
             - name: DATAHUB_GMS_HOST
-              value: "{{ .Values.global.datahub.gms.host }}"
+              value: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
             - name: DATAHUB_GMS_PORT
               value: "{{ .Values.global.datahub.gms.port }}"
             - name: DATAHUB_SECRET
               valueFrom:
                 secretKeyRef:
-                  name: {{ include "datahub-frontend.fullname" . }}-secret
+                  name: {{ printf "%s-gms-secret" .Release.Name }}
                   key: datahub.gms.secret
             - name: DATAHUB_APP_VERSION
               value: "{{ .Values.global.datahub.appVersion }}"
             - name: DATAHUB_PLAY_MEM_BUFFER_SIZE
               value: "{{ .Values.datahub.play.mem.buffer.size }}"
+          {{- if .Values.extraEnvs }}
+            {{ toYaml .Values.extraEnvs | indent 10 }}
+          {{- end }}
+          volumeMounts:
+          {{- if .Values.extraVolumeMounts }}
+            {{ toYaml .Values.extraVolumeMounts | indent 10 }}
+          {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}

diff --git a/contrib/kubernetes/datahub/charts/datahub-frontend/templates/secrets.yaml b/contrib/kubernetes/datahub/charts/datahub-frontend/templates/secrets.yaml
diff --git a/contrib/kubernetes/datahub/charts/datahub-frontend/values.yaml b/contrib/kubernetes/datahub/charts/datahub-frontend/values.yaml
@@ -7,11 +7,11 @@ replicaCount: 1
 image:
   repository: linkedin/datahub-frontend
   tag: "latest"
-  pullPolicy: IfNotPresent
+  pullPolicy: Always
 
 imagePullSecrets: []
 nameOverride: ""
-fullnameOverride: "datahub-frontend"
+fullnameOverride: ""
 
 serviceAccount:
   # Specifies whether a service account should be created
@@ -50,6 +50,22 @@ ingress:
   #    hosts:
   #      - chart-example.local
 
+# Extra environment variables
+# This will be appended to the current 'env:' key. You can use any of the kubernetes env
+# syntax here
+extraEnvs: []
+  # - name: MY_ENVIRONMENT_VAR
+ #   value: the_value_goes_here
+
+extraVolumes: []
+  # - name: extras
+  #   emptyDir: {}
+
+extraVolumeMounts: []
+  # - name: extras
+  #   mountPath: /usr/share/extras
+  #   readOnly: true
+
 resources: {}
   # We usually recommend not to specify default resources and to leave this as a conscious
   # choice for the user. This also increases chances charts run on environments with little
@@ -76,6 +92,5 @@ datahub:
 global:
   datahub:
     gms:
-      host: "datahub-gms-deployment"
       port: "8080"
-      secret: "YouKnowNothing"
+    appVersion: "1.0"
diff --git a/contrib/kubernetes/datahub/charts/datahub-gms/Chart.yaml b/contrib/kubernetes/datahub/charts/datahub-gms/Chart.yaml
@@ -14,7 +14,7 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.1.0
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.

diff --git a/contrib/kubernetes/datahub/charts/datahub-gms/README.md b/contrib/kubernetes/datahub/charts/datahub-gms/README.md
@@ -2,32 +2,39 @@ datahub-gms
 ===========
 A Helm chart for LinkedIn DataHub's datahub-gms component
 
-Current chart version is `0.1.0`
+Current chart version is `0.2.0`
 
 ## Chart Values
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` |  |
+| extraEnvs | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` |
+| extraVolumes | Templatable string of additional `volumes` to be passed to the `tpl` function | "" |
+| extraVolumeMounts | Templatable string of additional `volumeMounts` to be passed to the `tpl` function | "" |
 | fullnameOverride | string | `"datahub-gms-deployment"` |  |
 | global.datahub.appVersion | string | `"1.0"` |  |
-| global.datahub.gms.host | string | `"datahub-gms-service"` |  |
 | global.datahub.gms.port | string | `"8080"` |  |
-| global.datahub.gms.secret | string | `"YouKnowNothing"` |  |
-| global.elasticsearch.host | string | `"192.168.0.104"` |  |
+| global.elasticsearch.host | string | `"elasticsearch"` |  |
 | global.elasticsearch.port | string | `"9200"` |  |
 | global.hostAliases[0].hostnames[0] | string | `"broker"` |  |
+| global.hostAliases[0].hostnames[1] | string | `"mysql"` |  |
+| global.hostAliases[0].hostnames[2] | string | `"elasticsearch"` |  |
+| global.hostAliases[0].hostnames[3] | string | `"neo4j"` |  |
 | global.hostAliases[0].ip | string | `"192.168.0.104"` |  |
-| global.kafka.bootstrap.server | string | `"192.168.0.104:29092"` |  |
-| global.kafka.schemaregistry.url | string | `"http://192.168.0.104:8081"` |  |
-| global.neo4j.password | string | `"datahub"` |  |
-| global.neo4j.uri | string | `"bolt://192.168.0.104"` |  |
+| global.kafka.bootstrap.server | string | `"broker:29092"` |  |
+| global.kafka.schemaregistry.url | string | `"http://schema-registry:8081"` |  |
+| global.neo4j.host | string | `"neo4j:7474"` |  |
+| global.neo4j.uri | string | `"bolt://neo4j"` |  |
 | global.neo4j.username | string | `"neo4j"` |  |
+| global.neo4j.password.secretRef | string | `"neo4j-secrets"` |  |
+| global.neo4j.password.secretKey | string | `"neo4j-password"` |  |
 | global.sql.datasource.driver | string | `"com.mysql.jdbc.Driver"` |  |
-| global.sql.datasource.host | string | `"192.168.0.104:3306"` |  |
-| global.sql.datasource.password | string | `"datahub"` |  |
-| global.sql.datasource.url | string | `"jdbc:mysql://192.168.0.104:3306/datahub?verifyServerCertificate=false\u0026useSSL=true"` |  |
+| global.sql.datasource.host | string | `"mysql"` |  |
+| global.sql.datasource.url | string | `"jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false\u0026useSSL=true"` |  |
 | global.sql.datasource.username | string | `"datahub"` |  |
+| global.sql.datasource.password.secretRef | string | `"mysql-secrets"` |  |
+| global.sql.datasource.password.secretKey | string | `"mysql-password"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"linkedin/datahub-gms"` |  |
 | image.tag | string | `"latest"` |  |

diff --git a/contrib/kubernetes/datahub/charts/datahub-gms/templates/configmap.yaml b/contrib/kubernetes/datahub/charts/datahub-gms/templates/configmap.yaml