From 8898ac787a09d7b1b3c640b9dd97e3b69365f4a7 Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Tue, 7 Jan 2020 15:53:19 +0100 Subject: [PATCH 1/5] feat: enable rotation of service account tokens Signed-off-by: Martin Linkhorst --- cluster/manifests/kube-proxy/configmap.yaml | 1 + cluster/manifests/psp/pod_security_policy.yaml | 1 + cluster/node-pools/master-default/userdata.yaml | 9 ++++++--- cluster/node-pools/worker-default/userdata.yaml | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cluster/manifests/kube-proxy/configmap.yaml b/cluster/manifests/kube-proxy/configmap.yaml index 96dff5412e..9ffc760397 100644 --- a/cluster/manifests/kube-proxy/configmap.yaml +++ b/cluster/manifests/kube-proxy/configmap.yaml @@ -24,6 +24,7 @@ data: enableProfiling: false featureGates: TaintBasedEvictions: true + BoundServiceAccountTokenVolume: true healthzBindAddress: 0.0.0.0:10256 hostnameOverride: "" iptables: diff --git a/cluster/manifests/psp/pod_security_policy.yaml b/cluster/manifests/psp/pod_security_policy.yaml index 653b4d4d3b..381c55c6c1 100644 --- a/cluster/manifests/psp/pod_security_policy.yaml +++ b/cluster/manifests/psp/pod_security_policy.yaml @@ -43,3 +43,4 @@ spec: - persistentVolumeClaim - downwardAPI - configMap + - projected diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml index eeb35d8b0f..86cf7f04b4 100644 --- a/cluster/node-pools/master-default/userdata.yaml +++ b/cluster/node-pools/master-default/userdata.yaml @@ -44,6 +44,7 @@ write_files: {{- if ne .NodePool.ConfigItems.pod_max_pids "-1" }} featureGates: SupportPodPidsLimit: true + BoundServiceAccountTokenVolume: true podPidsLimit: {{ .NodePool.ConfigItems.pod_max_pids }} {{- end }} maxPods: {{ nodeCIDRMaxPods (parseInt64 .Cluster.ConfigItems.node_cidr_mask_size) 8 }} @@ -120,7 +121,7 @@ write_files: - --authorization-mode=Webhook,RBAC - --authorization-webhook-config-file=/etc/kubernetes/config/authz.yaml - --admission-control-config-file=/etc/kubernetes/config/image-policy-webhook.yaml - - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,CustomResourceWebhookConversion={{.Cluster.ConfigItems.custom_resource_webhook_conversion}},CustomResourcePublishOpenAPI={{.Cluster.ConfigItems.custom_resource_publish_openapi}} + - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,CustomResourceWebhookConversion={{.Cluster.ConfigItems.custom_resource_webhook_conversion}},CustomResourcePublishOpenAPI={{.Cluster.ConfigItems.custom_resource_publish_openapi}},BoundServiceAccountTokenVolume=true - --anonymous-auth=false {{ if ne .Cluster.ConfigItems.audittrail_url "" }} - --audit-webhook-config-file=/etc/kubernetes/config/audit.yaml @@ -147,6 +148,8 @@ write_files: - --kubelet-certificate-authority=/etc/kubernetes/ssl/ca.pem - --kubelet-client-certificate=/etc/kubernetes/ssl/kubelet-client.pem - --kubelet-client-key=/etc/kubernetes/ssl/kubelet-client-key.pem + - --service-account-signing-key-file=/etc/kubernetes/ssl/service-account-private-key.pem + - --service-account-issuer=kubernetes/serviceaccount livenessProbe: httpGet: host: 127.0.0.1 @@ -487,7 +490,7 @@ write_files: - --root-ca-file=/etc/kubernetes/ssl/ca.pem - --cloud-provider=aws - --cloud-config=/etc/kubernetes/cloud-config.ini - - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true + - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,BoundServiceAccountTokenVolume=true - --horizontal-pod-autoscaler-use-rest-clients=true - --use-service-account-credentials=true - --configure-cloud-routes=false @@ -550,7 +553,7 @@ write_files: args: - --master=http://127.0.0.1:8080 - --leader-elect=true - - --feature-gates=TaintBasedEvictions=true,TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}} + - --feature-gates=TaintBasedEvictions=true,TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},BoundServiceAccountTokenVolume=true env: - name: KUBE_MAX_PD_VOLS value: "26" diff --git a/cluster/node-pools/worker-default/userdata.yaml b/cluster/node-pools/worker-default/userdata.yaml index 13b6c61f36..60cfbb9f89 100644 --- a/cluster/node-pools/worker-default/userdata.yaml +++ b/cluster/node-pools/worker-default/userdata.yaml @@ -54,6 +54,7 @@ write_files: {{- if ne .NodePool.ConfigItems.pod_max_pids "-1" }} featureGates: SupportPodPidsLimit: true + BoundServiceAccountTokenVolume: true podPidsLimit: {{ .NodePool.ConfigItems.pod_max_pids }} {{- end }} cpuManagerPolicy: {{ .NodePool.ConfigItems.cpu_manager_policy }} From c0976b7520f80e34100522ace7dd2c40393dbbfd Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Tue, 7 Jan 2020 18:03:35 +0100 Subject: [PATCH 2/5] chore: disable non-expiring service account tokens Signed-off-by: Martin Linkhorst --- cluster/node-pools/master-default/userdata.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml index 86cf7f04b4..1dd73e3780 100644 --- a/cluster/node-pools/master-default/userdata.yaml +++ b/cluster/node-pools/master-default/userdata.yaml @@ -484,6 +484,7 @@ write_files: - name: kube-controller-manager image: nonexistent.zalan.do/teapot/kube-controller-manager:fixed args: + - --controllers=*,-serviceaccount-token - --kubeconfig=/etc/kubernetes/controller-kubeconfig - --leader-elect=true - --service-account-private-key-file=/etc/kubernetes/ssl/service-account-private-key.pem From 9fa125152367b2a54e77cb728b3cb11ee1086b59 Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Wed, 8 Jan 2020 11:21:50 +0100 Subject: [PATCH 3/5] fix: allow non-root apps to read service account token Signed-off-by: Martin Linkhorst --- cluster/manifests/dashboard/deployment.yaml | 2 ++ cluster/manifests/external-dns/deployment.yaml | 2 ++ cluster/manifests/heapster/deployment.yaml | 2 ++ cluster/manifests/kube-state-metrics/deployment.yaml | 2 ++ cluster/manifests/skipper/deployment.yaml | 2 ++ 5 files changed, 10 insertions(+) diff --git a/cluster/manifests/dashboard/deployment.yaml b/cluster/manifests/dashboard/deployment.yaml index 3766646c59..3e6e3dae45 100644 --- a/cluster/manifests/dashboard/deployment.yaml +++ b/cluster/manifests/dashboard/deployment.yaml @@ -47,3 +47,5 @@ spec: readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 1000 + securityContext: + fsGroup: 1000 diff --git a/cluster/manifests/external-dns/deployment.yaml b/cluster/manifests/external-dns/deployment.yaml index 7a0c354d38..bb6a81f0f2 100644 --- a/cluster/manifests/external-dns/deployment.yaml +++ b/cluster/manifests/external-dns/deployment.yaml @@ -65,6 +65,8 @@ spec: runAsUser: 65534 capabilities: drop: ["ALL"] + securityContext: + fsGroup: 65534 {{ if eq .ConfigItems.kube_aws_iam_controller_kube_system_enable "true"}} volumes: - name: aws-iam-credentials diff --git a/cluster/manifests/heapster/deployment.yaml b/cluster/manifests/heapster/deployment.yaml index deda001d0d..1bc4f2f5d0 100644 --- a/cluster/manifests/heapster/deployment.yaml +++ b/cluster/manifests/heapster/deployment.yaml @@ -24,6 +24,8 @@ spec: value: "1" priorityClassName: system-cluster-critical serviceAccountName: heapster + securityContext: + fsGroup: 65534 containers: - image: registry.opensource.zalan.do/teapot/heapster:v1.5.4 name: heapster diff --git a/cluster/manifests/kube-state-metrics/deployment.yaml b/cluster/manifests/kube-state-metrics/deployment.yaml index 12606d6138..84b107aa0d 100644 --- a/cluster/manifests/kube-state-metrics/deployment.yaml +++ b/cluster/manifests/kube-state-metrics/deployment.yaml @@ -47,3 +47,5 @@ spec: runAsUser: 65534 capabilities: drop: ["ALL"] + securityContext: + fsGroup: 65534 diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml index 1a48824005..92da95144d 100644 --- a/cluster/manifests/skipper/deployment.yaml +++ b/cluster/manifests/skipper/deployment.yaml @@ -177,6 +177,8 @@ spec: name: skipper-ingress key: lightstep-token {{ end }} + securityContext: + fsGroup: 1000 {{ if eq .ConfigItems.enable_apimonitoring "true"}} volumes: - name: filters From 929878e3a59fa72217c058195ad6b93a7e1f6568 Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Wed, 8 Jan 2020 13:29:05 +0100 Subject: [PATCH 4/5] feat: make service account token rotation configurable Signed-off-by: Martin Linkhorst --- cluster/config-defaults.yaml | 2 ++ cluster/manifests/kube-proxy/configmap.yaml | 2 +- .../node-pools/master-default/userdata.yaml | 18 ++++++++---------- .../node-pools/worker-default/userdata.yaml | 5 +---- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 26a5b2a746..3f43ccef45 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -298,6 +298,8 @@ apiserver_proxy: "true" # when set to true, service account tokens can be used from outside the cluster # requires apiserver_proxy to be set to "true" allow_external_service_accounts: "false" +# issue service account tokens with expiration time. +rotate_service_account_tokens: "false" # use kube-aws-iam-controller for kube-system components kube_aws_iam_controller_kube_system_enable: "true" diff --git a/cluster/manifests/kube-proxy/configmap.yaml b/cluster/manifests/kube-proxy/configmap.yaml index 9ffc760397..f2d847f842 100644 --- a/cluster/manifests/kube-proxy/configmap.yaml +++ b/cluster/manifests/kube-proxy/configmap.yaml @@ -24,7 +24,7 @@ data: enableProfiling: false featureGates: TaintBasedEvictions: true - BoundServiceAccountTokenVolume: true + BoundServiceAccountTokenVolume: {{ .Cluster.ConfigItems.rotate_service_account_tokens }} healthzBindAddress: 0.0.0.0:10256 hostnameOverride: "" iptables: diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml index 1dd73e3780..071258685e 100644 --- a/cluster/node-pools/master-default/userdata.yaml +++ b/cluster/node-pools/master-default/userdata.yaml @@ -41,12 +41,9 @@ write_files: {{- if not (index .Cluster.ConfigItems "enable_cfs_quota") }} cpuCFSQuota: false {{- end }} -{{- if ne .NodePool.ConfigItems.pod_max_pids "-1" }} featureGates: - SupportPodPidsLimit: true - BoundServiceAccountTokenVolume: true + BoundServiceAccountTokenVolume: {{ .Cluster.ConfigItems.rotate_service_account_tokens }} podPidsLimit: {{ .NodePool.ConfigItems.pod_max_pids }} -{{- end }} maxPods: {{ nodeCIDRMaxPods (parseInt64 .Cluster.ConfigItems.node_cidr_mask_size) 8 }} healthzPort: 10248 healthzBindAddress: "0.0.0.0" @@ -121,8 +118,12 @@ write_files: - --authorization-mode=Webhook,RBAC - --authorization-webhook-config-file=/etc/kubernetes/config/authz.yaml - --admission-control-config-file=/etc/kubernetes/config/image-policy-webhook.yaml - - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,CustomResourceWebhookConversion={{.Cluster.ConfigItems.custom_resource_webhook_conversion}},CustomResourcePublishOpenAPI={{.Cluster.ConfigItems.custom_resource_publish_openapi}},BoundServiceAccountTokenVolume=true + - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,CustomResourceWebhookConversion={{.Cluster.ConfigItems.custom_resource_webhook_conversion}},CustomResourcePublishOpenAPI={{.Cluster.ConfigItems.custom_resource_publish_openapi}},BoundServiceAccountTokenVolume={{ .Cluster.ConfigItems.rotate_service_account_tokens }} - --anonymous-auth=false + {{- if eq .Cluster.ConfigItems.rotate_service_account_tokens "true" }} + - --service-account-signing-key-file=/etc/kubernetes/ssl/service-account-private-key.pem + - --service-account-issuer=kubernetes/serviceaccount + {{- end }} {{ if ne .Cluster.ConfigItems.audittrail_url "" }} - --audit-webhook-config-file=/etc/kubernetes/config/audit.yaml - --audit-webhook-mode=batch @@ -148,8 +149,6 @@ write_files: - --kubelet-certificate-authority=/etc/kubernetes/ssl/ca.pem - --kubelet-client-certificate=/etc/kubernetes/ssl/kubelet-client.pem - --kubelet-client-key=/etc/kubernetes/ssl/kubelet-client-key.pem - - --service-account-signing-key-file=/etc/kubernetes/ssl/service-account-private-key.pem - - --service-account-issuer=kubernetes/serviceaccount livenessProbe: httpGet: host: 127.0.0.1 @@ -484,14 +483,13 @@ write_files: - name: kube-controller-manager image: nonexistent.zalan.do/teapot/kube-controller-manager:fixed args: - - --controllers=*,-serviceaccount-token - --kubeconfig=/etc/kubernetes/controller-kubeconfig - --leader-elect=true - --service-account-private-key-file=/etc/kubernetes/ssl/service-account-private-key.pem - --root-ca-file=/etc/kubernetes/ssl/ca.pem - --cloud-provider=aws - --cloud-config=/etc/kubernetes/cloud-config.ini - - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,BoundServiceAccountTokenVolume=true + - --feature-gates=TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},TTLAfterFinished=true,BoundServiceAccountTokenVolume={{ .Cluster.ConfigItems.rotate_service_account_tokens }} - --horizontal-pod-autoscaler-use-rest-clients=true - --use-service-account-credentials=true - --configure-cloud-routes=false @@ -554,7 +552,7 @@ write_files: args: - --master=http://127.0.0.1:8080 - --leader-elect=true - - --feature-gates=TaintBasedEvictions=true,TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},BoundServiceAccountTokenVolume=true + - --feature-gates=TaintBasedEvictions=true,TaintNodesByCondition={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},ScheduleDaemonSetPods={{.Cluster.ConfigItems.experimental_schedule_daemonset_pods}},BoundServiceAccountTokenVolume={{ .Cluster.ConfigItems.rotate_service_account_tokens }} env: - name: KUBE_MAX_PD_VOLS value: "26" diff --git a/cluster/node-pools/worker-default/userdata.yaml b/cluster/node-pools/worker-default/userdata.yaml index 60cfbb9f89..6cac586d8d 100644 --- a/cluster/node-pools/worker-default/userdata.yaml +++ b/cluster/node-pools/worker-default/userdata.yaml @@ -51,12 +51,9 @@ write_files: kind: KubeletConfiguration clusterDomain: cluster.local cpuCFSQuota: false -{{- if ne .NodePool.ConfigItems.pod_max_pids "-1" }} featureGates: - SupportPodPidsLimit: true - BoundServiceAccountTokenVolume: true + BoundServiceAccountTokenVolume: {{ .Cluster.ConfigItems.rotate_service_account_tokens }} podPidsLimit: {{ .NodePool.ConfigItems.pod_max_pids }} -{{- end }} cpuManagerPolicy: {{ .NodePool.ConfigItems.cpu_manager_policy }} maxPods: {{ nodeCIDRMaxPods (parseInt64 .Cluster.ConfigItems.node_cidr_mask_size) 0 }} healthzPort: 10248 From 0940b380a758dbbc34af38981c7c45659e3dbe6c Mon Sep 17 00:00:00 2001 From: Martin Linkhorst Date: Tue, 14 Jan 2020 10:45:18 +0100 Subject: [PATCH 5/5] ref: change config so it doesn't roll workers when disabled Signed-off-by: Martin Linkhorst --- cluster/node-pools/worker-default/userdata.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster/node-pools/worker-default/userdata.yaml b/cluster/node-pools/worker-default/userdata.yaml index 6cac586d8d..c3b2468433 100644 --- a/cluster/node-pools/worker-default/userdata.yaml +++ b/cluster/node-pools/worker-default/userdata.yaml @@ -52,7 +52,10 @@ write_files: clusterDomain: cluster.local cpuCFSQuota: false featureGates: + SupportPodPidsLimit: true +{{- if eq .Cluster.ConfigItems.rotate_service_account_tokens "true" }} BoundServiceAccountTokenVolume: {{ .Cluster.ConfigItems.rotate_service_account_tokens }} +{{- end }} podPidsLimit: {{ .NodePool.ConfigItems.pod_max_pids }} cpuManagerPolicy: {{ .NodePool.ConfigItems.cpu_manager_policy }} maxPods: {{ nodeCIDRMaxPods (parseInt64 .Cluster.ConfigItems.node_cidr_mask_size) 0 }}