diff --git a/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/Chart.yaml b/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/Chart.yaml new file mode 100644 index 000000000..2f0a2104f --- /dev/null +++ b/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Helm chart for csi-driver-efs-node (elastic file system) +name: csi-driver-efs-controller +version: 0.1.0 \ No newline at end of file diff --git a/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/templates/controller-deployment.yaml b/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/templates/controller-deployment.yaml new file mode 100644 index 000000000..42d03d0a3 --- /dev/null +++ b/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/templates/controller-deployment.yaml @@ -0,0 +1,164 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: efs-csi-controller + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: efs-csi-controller + app: csi-efs + role: driver-efs-controller + high-availability-config.resources.gardener.cloud/type: controller +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: efs-csi-controller + app: csi-efs + role: driver-efs-controller + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/name: efs-csi-controller + app: csi-efs + role: driver-efs-controller + gardener.cloud/role: driver-efs-controller + networking.gardener.cloud/to-dns: allowed + networking.gardener.cloud/to-public-networks: allowed + networking.gardener.cloud/to-private-networks: allowed + networking.resources.gardener.cloud/to-kube-apiserver-tcp-443: allowed +{{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} +{{- end }} + spec: + automountServiceAccountToken: false + priorityClassName: gardener-system-300 + {{- if hasKey .Values.controller "hostNetwork" }} + hostNetwork: {{ .Values.controller.hostNetwork }} + {{- end }} +{{/* serviceAccountName: {{ .Values.controller.serviceAccount.name }}*/}} + {{- with .Values.controller.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.controller.dnsPolicy }} + dnsPolicy: {{ .Values.controller.dnsPolicy }} + {{- end }} + {{- with .Values.controller.dnsConfig }} + dnsConfig: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: aws-csi-driver-efs + image: {{ index .Values.images "csi-driver-efs" }} + args: + - controller + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + {{- if .Values.controller.tags }} + - --tags={{ include "aws-efs-csi-driver.tags" .Values.controller.tags }} + {{- end }} + - --v={{ .Values.controller.logLevel }} + - --delete-access-point-root-dir={{ hasKey .Values.controller "deleteAccessPointRootDir" | ternary .Values.controller.deleteAccessPointRootDir false }} + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_REGION + value: {{ .Values.region }} + - name: AWS_SHARED_CREDENTIALS_FILE + value: /srv/cloudprovider/credentialsFile + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: cloudprovider + mountPath: /srv/cloudprovider + ports: + - name: healthz + containerPort: {{ .Values.controller.healthPort }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.resources.driverController }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + + - name: aws-csi-provisioner + image: {{ index .Values.images "csi-provisioner" }} + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.controller.logLevel }} + - --feature-gates=Topology=true + - --kubeconfig=/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig/kubeconfig + {{- if .Values.controller.extraCreateMetadata }} + - --extra-create-metadata + {{- end }} + - --leader-election + {{- if hasKey .Values.controller "leaderElectionRenewDeadline" }} + - --leader-election-renew-deadline={{ .Values.controller.leaderElectionRenewDeadline }} + {{- end }} + {{- if hasKey .Values.controller "leaderElectionLeaseDuration" }} + - --leader-election-lease-duration={{ .Values.controller.leaderElectionLeaseDuration }} + {{- end }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: kubeconfig-csi-provisioner + mountPath: /var/run/secrets/gardener.cloud/shoot/generic-kubeconfig + readOnly: true + {{- with .Values.resources.provisioner }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + + - name: liveness-probe + image: {{ index .Values.images "csi-liveness-probe" }} + args: + - --csi-address=/csi/csi.sock + - --health-port={{ .Values.controller.healthPort }} + volumeMounts: + - name: socket-dir + mountPath: /csi + {{- with .Values.resources.livenessProbe }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: socket-dir + emptyDir: {} + - name: kubeconfig-csi-provisioner + projected: + defaultMode: 420 + sources: + - secret: + items: + - key: kubeconfig + path: kubeconfig + name: {{ .Values.global.genericTokenKubeconfigSecretName }} + optional: false + - secret: + items: + - key: token + path: token + name: shoot-access-csi-provisioner + optional: false + - name: cloudprovider + secret: + secretName: cloudprovider \ No newline at end of file diff --git a/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/templates/helpers.tpl b/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/templates/helpers.tpl new file mode 100644 index 000000000..eb354bc3e --- /dev/null +++ b/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/templates/helpers.tpl @@ -0,0 +1,56 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "aws-efs-csi-driver.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aws-efs-csi-driver.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-efs-csi-driver.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "aws-efs-csi-driver.labels" -}} +app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} +helm.sh/chart: {{ include "aws-efs-csi-driver.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create a string out of the map for controller tags flag +*/}} +{{- define "aws-efs-csi-driver.tags" -}} +{{- $tags := list -}} +{{ range $key, $val := . }} +{{- $tags = print $key ":" $val | append $tags -}} +{{- end -}} +{{- join " " $tags -}} +{{- end -}} \ No newline at end of file diff --git a/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/values.yaml b/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/values.yaml new file mode 100644 index 000000000..c43083b49 --- /dev/null +++ b/charts/internal/seed-controlplane/charts/csi-driver-efs-controller/values.yaml @@ -0,0 +1,84 @@ +fileSystemID: "" + +region: region + +nameOverride: "" +fullnameOverride: "" + +dnsPolicy: ClusterFirst + +useFIPS: false + +resources: + driverController: + requests: + cpu: 20m + memory: 50Mi + livenessProbe: + requests: + cpu: 11m + memory: 32Mi + provisioner: + requests: + cpu: 11m + memory: 38Mi + +sidecars: + livenessProbe: + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + +## controller values + +replicas: 1 + +images: + csi-driver-efs: image-repository:image-tag + csi-provisioner: image-repository:image-tag + csi-liveness-probe: image-repository:image-tag + +podAnnotations: {} + +## Controller deployment variables + +controller: + # Number for the log level verbosity + logLevel: 5 + # If set, add pv/pvc metadata to plugin create requests as parameters. + extraCreateMetadata: true + # Add additional tags to access points + tags: + {} + # environment: prod + # region: us-east-1 + # Enable if you want the controller to also delete the + # path on efs when deleteing an access point + deleteAccessPointRootDir: false + hostNetwork: false + priorityClassName: system-cluster-critical + dnsPolicy: ClusterFirst + dnsConfig: {} + additionalLabels: {} + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: efs.csi.aws.com/agent-not-ready + operator: Exists + # securityContext on the controller pod + securityContext: + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + serviceAccount: + name: efs-csi-controller-sa + annotations: {} + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role + healthPort: 9909 + regionalStsEndpoints: false + containerSecurityContext: + privileged: true + leaderElectionRenewDeadline: 10s + leaderElectionLeaseDuration: 15s \ No newline at end of file diff --git a/charts/internal/shoot-system-components/charts/csi-driver-efs-node/Chart.yaml b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/Chart.yaml new file mode 100644 index 000000000..0ebe31c93 --- /dev/null +++ b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Helm chart for csi-driver-efs-node (elastic file system) +name: csi-driver-efs-node +version: 0.1.0 \ No newline at end of file diff --git a/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/csi-driver.yaml b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/csi-driver.yaml new file mode 100644 index 000000000..ed63c4c56 --- /dev/null +++ b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/csi-driver.yaml @@ -0,0 +1,6 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: efs.csi.aws.com +spec: + attachRequired: false \ No newline at end of file diff --git a/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/helpers.tpl b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/helpers.tpl new file mode 100644 index 000000000..eb354bc3e --- /dev/null +++ b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/helpers.tpl @@ -0,0 +1,56 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "aws-efs-csi-driver.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aws-efs-csi-driver.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-efs-csi-driver.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "aws-efs-csi-driver.labels" -}} +app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} +helm.sh/chart: {{ include "aws-efs-csi-driver.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create a string out of the map for controller tags flag +*/}} +{{- define "aws-efs-csi-driver.tags" -}} +{{- $tags := list -}} +{{ range $key, $val := . }} +{{- $tags = print $key ":" $val | append $tags -}} +{{- end -}} +{{- join " " $tags -}} +{{- end -}} \ No newline at end of file diff --git a/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/node-daemonset.yaml b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/node-daemonset.yaml new file mode 100644 index 000000000..1fc4ca053 --- /dev/null +++ b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/node-daemonset.yaml @@ -0,0 +1,169 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: csi-driver-efs-node + namespace: {{ .Release.Namespace }} + labels: + app: csi + role: driver-efs-node + node.gardener.cloud/critical-component: "true" +spec: + selector: + matchLabels: + app: csi + role: driver-efs-node + node.gardener.cloud/critical-component: "true" + template: + metadata: + labels: + app: csi + role: driver-efs-node + node.gardener.cloud/critical-component: "true" + spec: + priorityClassName: system-node-critical + hostNetwork: true + dnsPolicy: {{ .Values.dnsPolicy }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + {{- with .Values.node.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: driver-efs-node + securityContext: + privileged: true + capabilities: + add: [ "SYS_ADMIN" ] + allowPrivilegeEscalation: true + image: {{ index .Values.images "csi-driver-efs" }} + args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v={{ .Values.node.logLevel }} + - --vol-metrics-opt-in={{ hasKey .Values.node "volMetricsOptIn" | ternary .Values.node.volMetricsOptIn false }} + - --vol-metrics-refresh-period={{ hasKey .Values.node "volMetricsRefreshPeriod" | ternary .Values.node.volMetricsRefreshPeriod 240 }} + - --vol-metrics-fs-rate-limit={{ hasKey .Values.node "volMetricsFsRateLimit" | ternary .Values.node.volMetricsFsRateLimit 5 }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.useFIPS }} + - name: AWS_USE_FIPS_ENDPOINT + value: "true" + {{- end }} + {{- with .Values.node.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: {{ .Values.node.kubeletPath }} + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: efs-state-dir + mountPath: /var/run/efs + - name: efs-utils-config + mountPath: /var/amazon/efs + - name: efs-utils-config-legacy + mountPath: /etc/amazon/efs-legacy + {{- with .Values.node.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: healthz + containerPort: {{ .Values.node.healthPort }} + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 + {{- with .Values.resources.driverNode }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-driver-registrar + image: {{ index .Values.images "csi-node-driver-registrar" }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.node.logLevel }} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ printf "%s/plugins/efs.csi.aws.com/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + {{- with .Values.resources.nodeDriverRegistrar }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ index .Values.images "csi-liveness-probe" }} + args: + - --csi-address=/csi/csi.sock + - --health-port={{ .Values.node.healthPort }} + - --v={{ .Values.node.logLevel }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- with .Values.resources.livenessProbe }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.livenessProbe.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: {{ .Values.node.kubeletPath }} + type: Directory + - name: plugin-dir + hostPath: + path: {{ printf "%s/plugins/efs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} + type: Directory + - name: efs-state-dir + hostPath: + path: /var/run/efs + type: DirectoryOrCreate + - name: efs-utils-config + hostPath: + path: /var/amazon/efs + type: DirectoryOrCreate + - name: efs-utils-config-legacy + hostPath: + path: /etc/amazon/efs + type: DirectoryOrCreate + {{- with .Values.node.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/node-serviceaccount.yaml b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/node-serviceaccount.yaml new file mode 100644 index 000000000..534d952b7 --- /dev/null +++ b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/node-serviceaccount.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.node.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + {{- with .Values.node.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: efs-csi-node-role + labels: + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: efs-csi-node-binding + labels: + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} +subjects: + - kind: ServiceAccount + name: {{ .Values.node.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: efs-csi-node-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/storageclass.yaml b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/storageclass.yaml new file mode 100644 index 000000000..adb598127 --- /dev/null +++ b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/templates/storageclass.yaml @@ -0,0 +1,9 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: efs-sc +provisioner: efs.csi.aws.com +parameters: + provisioningMode: efs-ap # only one currently available + fileSystemId: {{ .Values.fileSystemID }} + directoryPerms: "700" \ No newline at end of file diff --git a/charts/internal/shoot-system-components/charts/csi-driver-efs-node/values.yaml b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/values.yaml new file mode 100644 index 000000000..424aa965a --- /dev/null +++ b/charts/internal/shoot-system-components/charts/csi-driver-efs-node/values.yaml @@ -0,0 +1,104 @@ +fileSystemID: "" + +nameOverride: "" +fullnameOverride: "" + +dnsPolicy: ClusterFirst + +useFIPS: false + +images: + csi-driver-efs: image-repository:image-tag + csi-liveness-probe: image-repository:image-tag + +resources: + driverNode: + requests: + cpu: 15m + memory: 42Mi + nodeDriverRegistrar: + requests: + cpu: 11m + memory: 32Mi + livenessProbe: + requests: + cpu: 11m + memory: 32Mi + +sidecars: + livenessProbe: + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + nodeDriverRegistrar: + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + +## Node daemonset variables + +node: + # Number for the log level verbosity + logLevel: 5 + volMetricsOptIn: false + volMetricsRefreshPeriod: 240 + volMetricsFsRateLimit: 5 + hostAliases: + {} + # For cross VPC EFS, you need to poison or overwrite the DNS for the efs volume as per + # https://docs.aws.amazon.com/efs/latest/ug/efs-different-vpc.html#wt6-efs-utils-step3 + # implementing the suggested solution found here: + # https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/240#issuecomment-676849346 + # EFS Vol ID, IP, Region + # "fs-01234567": + # ip: 10.10.2.2 + # region: us-east-2 + dnsConfig: + {} + # Example config which uses the AWS nameservers + # dnsPolicy: "None" + # dnsConfig: + # nameservers: + # - 169.254.169.253 + podLabels: {} + podAnnotations: {} + additionalLabels: {} + nodeSelector: {} + # Override default strategy (RollingUpdate) to speed up deployment. + # This can be useful if helm timeouts are observed. + # type: OnDelete + tolerations: + - operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + serviceAccount: + name: efs-csi-node-sa + annotations: {} + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role + healthPort: 9809 + # securityContext on the node pod + securityContext: + # The node pod must be run as root to bind to the registration/driver sockets + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + env: [] + volumes: [] + volumeMounts: [] + kubeletPath: /var/lib/kubelet + +storageClasses: [] + +controller: + serviceAccount: + name: efs-csi-controller-sa + annotations: {} \ No newline at end of file diff --git a/charts/internal/shoot-system-components/charts/csi-driver-node/templates/clusterrole-csi-provisioner.yaml b/charts/internal/shoot-system-components/charts/csi-driver-node/templates/clusterrole-csi-provisioner.yaml index 75346a9a2..fb8711a17 100644 --- a/charts/internal/shoot-system-components/charts/csi-driver-node/templates/clusterrole-csi-provisioner.yaml +++ b/charts/internal/shoot-system-components/charts/csi-driver-node/templates/clusterrole-csi-provisioner.yaml @@ -4,30 +4,24 @@ kind: ClusterRole metadata: name: {{ include "csi-driver-node.extensionsGroup" . }}:{{ include "csi-driver-node.name" . }}:csi-provisioner rules: -- apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] -- apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update", "patch"] -- apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list"] -- apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["get", "list"] -- apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] -- apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] diff --git a/charts/internal/shoot-system-components/requirements.yaml b/charts/internal/shoot-system-components/requirements.yaml index daa1289e4..9a425c3f3 100644 --- a/charts/internal/shoot-system-components/requirements.yaml +++ b/charts/internal/shoot-system-components/requirements.yaml @@ -14,4 +14,8 @@ dependencies: - name: aws-load-balancer-controller repository: http://localhost:10191 version: 0.1.0 - condition: aws-load-balancer-controller.enabled \ No newline at end of file + condition: aws-load-balancer-controller.enabled +- name: csi-driver-efs-node + repository: http://localhost:10191 + version: 0.1.0 + condition: csi-driver-efs-node.enabled \ No newline at end of file diff --git a/charts/internal/shoot-system-components/values.yaml b/charts/internal/shoot-system-components/values.yaml index 875953b0a..7838a83a5 100644 --- a/charts/internal/shoot-system-components/values.yaml +++ b/charts/internal/shoot-system-components/values.yaml @@ -5,4 +5,6 @@ csi-driver-node: aws-custom-route-controller: enabled: false aws-load-balancer-controller: + enabled: false +csi-driver-efs-node: enabled: false \ No newline at end of file diff --git a/docs/usage/usage.md b/docs/usage/usage.md index baf4ca966..f09a7097b 100644 --- a/docs/usage/usage.md +++ b/docs/usage/usage.md @@ -484,6 +484,7 @@ spec: internal: 10.250.112.0/22 public: 10.250.96.0/22 workers: 10.250.0.0/19 + enableCsiEfs: true controlPlaneConfig: apiVersion: aws.provider.extensions.gardener.cloud/v1alpha1 kind: ControlPlaneConfig @@ -593,6 +594,8 @@ Every AWS shoot cluster will be deployed with the AWS EBS CSI driver. It is compatible with the legacy in-tree volume provisioner that was deprecated by the Kubernetes community and will be removed in future versions of Kubernetes. End-users might want to update their custom `StorageClass`es to the new `ebs.csi.aws.com` provisioner. +To deploy the efs-csi-driver add the annotation `enableCsiEfs: true` to your infrastructureConfig like in this [example](#example-shoot-manifest-one-availability-zone). + ### Node-specific Volume Limits The Kubernetes scheduler allows configurable limit for the number of volumes that can be attached to a node. See https://k8s.io/docs/concepts/storage/storage-limits/#custom-limits. diff --git a/hack/api-reference/api.md b/hack/api-reference/api.md index 364c007b0..ef8cc0c35 100644 --- a/hack/api-reference/api.md +++ b/hack/api-reference/api.md @@ -230,6 +230,19 @@ See WorkerConfig @@ -395,6 +408,36 @@ reconciliation is possible.

+

CSI +

+

+(Appears on: +InfrastructureStatus) +

+

+

CSI contains information about the created AWS CSI related resources.

+

+ + + + + + + + + + + + + +
FieldDescription
+efsFileSystemID
+ +string + +		 +

EfsSystemID contains the efsFileSystem.

+

CloudControllerManagerConfig

@@ -816,6 +859,19 @@ VPCStatus

VPC contains information about the created AWS VPC and some related resources.

+ + +csi
+ + +CSI + + + + +

CSI contains information about the created AWS CSI related resources.

+ +

InstanceMetadataOptions diff --git a/imagevector/images.yaml b/imagevector/images.yaml index dff30aceb..284fb0e7e 100644 --- a/imagevector/images.yaml +++ b/imagevector/images.yaml @@ -152,6 +152,19 @@ images: confidentiality_requirement: 'high' integrity_requirement: 'high' availability_requirement: 'low' +- name: csi-driver-efs + sourceRepository: github.com/kubernetes-sigs/aws-efs-csi-driver + repository: europe-docker.pkg.dev/gardener-project/snapshots/test/efs-i540852 + tag: "latest" + labels: + - name: 'gardener.cloud/cve-categorisation' + value: + network_exposure: 'protected' + authentication_enforced: false + user_interaction: 'end-user' + confidentiality_requirement: 'high' + integrity_requirement: 'high' + availability_requirement: 'low' - name: csi-volume-modifier sourceRepository: github.com/awslabs/volume-modifier-for-k8s # We cannot use the upstream repository here as it is not reachable using IPv6. diff --git a/pkg/apis/aws/types_infrastructure.go b/pkg/apis/aws/types_infrastructure.go index 78ad40c9a..4156ec817 100644 --- a/pkg/apis/aws/types_infrastructure.go +++ b/pkg/apis/aws/types_infrastructure.go @@ -32,6 +32,11 @@ type InfrastructureConfig struct { // See https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/resource-tagging#ignoring-changes-in-all-resources // for details of the underlying terraform implementation. IgnoreTags *IgnoreTags + + // EnableCsiEfs enables CSI EFS driver + // infra will add additional security group in bound rules and create an amazon EFS file system + // Defaults to false + EnableCsiEfs *bool } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -45,6 +50,8 @@ type InfrastructureStatus struct { IAM IAM // VPC contains information about the created AWS VPC and some related resources. VPC VPCStatus + // CSI contains information about the created AWS CSI related resources. + CSI CSI `json:"csi"` } // Networks holds information about the Kubernetes and infrastructure networks. @@ -116,6 +123,12 @@ type VPCStatus struct { SecurityGroups []SecurityGroup } +// CSI contains information about the created AWS CSI related resources. +type CSI struct { + // EfsSystemID contains the efsFileSystem. + EfsSystemID string +} + const ( // PurposeNodes is a constant describing that the respective resource is used for nodes. PurposeNodes string = "nodes" diff --git a/pkg/apis/aws/v1alpha1/types_infrastructure.go b/pkg/apis/aws/v1alpha1/types_infrastructure.go index 0abd48017..7f699c94c 100644 --- a/pkg/apis/aws/v1alpha1/types_infrastructure.go +++ b/pkg/apis/aws/v1alpha1/types_infrastructure.go @@ -35,6 +35,11 @@ type InfrastructureConfig struct { // for details of the underlying terraform implementation. // +optional IgnoreTags *IgnoreTags `json:"ignoreTags,omitempty"` + + // EnableCsiEfs enables CSI EFS driver + // infra will add additional security group in bound rules and create an amazon EFS file system + // Defaults to false + EnableCsiEfs *bool `json:"enableCsiEfs,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -48,6 +53,8 @@ type InfrastructureStatus struct { IAM IAM `json:"iam"` // VPC contains information about the created AWS VPC and some related resources. VPC VPCStatus `json:"vpc"` + // CSI contains information about the created AWS CSI related resources. + CSI CSI `json:"csi"` } // Networks holds information about the Kubernetes and infrastructure networks. @@ -125,6 +132,12 @@ type VPCStatus struct { SecurityGroups []SecurityGroup `json:"securityGroups"` } +// CSI contains information about the created AWS CSI related resources. +type CSI struct { + // EfsSystemID contains the efsFileSystem. + EfsSystemID string `json:"efsFileSystemID"` +} + const ( // PurposeNodes is a constant describing that the respective resource is used for nodes. PurposeNodes string = "nodes" diff --git a/pkg/apis/aws/v1alpha1/zz_generated.conversion.go b/pkg/apis/aws/v1alpha1/zz_generated.conversion.go index c318624b7..4cda7077e 100644 --- a/pkg/apis/aws/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/aws/v1alpha1/zz_generated.conversion.go @@ -25,6 +25,16 @@ func init() { // RegisterConversions adds conversion functions to the given scheme. // Public to allow building arbitrary schemes. func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*CSI)(nil), (*aws.CSI)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_CSI_To_aws_CSI(a.(*CSI), b.(*aws.CSI), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*aws.CSI)(nil), (*CSI)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_aws_CSI_To_v1alpha1_CSI(a.(*aws.CSI), b.(*CSI), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*CloudControllerManagerConfig)(nil), (*aws.CloudControllerManagerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_CloudControllerManagerConfig_To_aws_CloudControllerManagerConfig(a.(*CloudControllerManagerConfig), b.(*aws.CloudControllerManagerConfig), scope) }); err != nil { @@ -338,6 +348,26 @@ func RegisterConversions(s *runtime.Scheme) error { return nil } +func autoConvert_v1alpha1_CSI_To_aws_CSI(in *CSI, out *aws.CSI, s conversion.Scope) error { + out.EfsSystemID = in.EfsSystemID + return nil +} + +// Convert_v1alpha1_CSI_To_aws_CSI is an autogenerated conversion function. +func Convert_v1alpha1_CSI_To_aws_CSI(in *CSI, out *aws.CSI, s conversion.Scope) error { + return autoConvert_v1alpha1_CSI_To_aws_CSI(in, out, s) +} + +func autoConvert_aws_CSI_To_v1alpha1_CSI(in *aws.CSI, out *CSI, s conversion.Scope) error { + out.EfsSystemID = in.EfsSystemID + return nil +} + +// Convert_aws_CSI_To_v1alpha1_CSI is an autogenerated conversion function. +func Convert_aws_CSI_To_v1alpha1_CSI(in *aws.CSI, out *CSI, s conversion.Scope) error { + return autoConvert_aws_CSI_To_v1alpha1_CSI(in, out, s) +} + func autoConvert_v1alpha1_CloudControllerManagerConfig_To_aws_CloudControllerManagerConfig(in *CloudControllerManagerConfig, out *aws.CloudControllerManagerConfig, s conversion.Scope) error { out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) out.UseCustomRouteController = (*bool)(unsafe.Pointer(in.UseCustomRouteController)) @@ -567,6 +597,7 @@ func autoConvert_v1alpha1_InfrastructureConfig_To_aws_InfrastructureConfig(in *I return err } out.IgnoreTags = (*aws.IgnoreTags)(unsafe.Pointer(in.IgnoreTags)) + out.EnableCsiEfs = (*bool)(unsafe.Pointer(in.EnableCsiEfs)) return nil } @@ -582,6 +613,7 @@ func autoConvert_aws_InfrastructureConfig_To_v1alpha1_InfrastructureConfig(in *a return err } out.IgnoreTags = (*IgnoreTags)(unsafe.Pointer(in.IgnoreTags)) + out.EnableCsiEfs = (*bool)(unsafe.Pointer(in.EnableCsiEfs)) return nil } @@ -620,6 +652,9 @@ func autoConvert_v1alpha1_InfrastructureStatus_To_aws_InfrastructureStatus(in *I if err := Convert_v1alpha1_VPCStatus_To_aws_VPCStatus(&in.VPC, &out.VPC, s); err != nil { return err } + if err := Convert_v1alpha1_CSI_To_aws_CSI(&in.CSI, &out.CSI, s); err != nil { + return err + } return nil } @@ -638,6 +673,9 @@ func autoConvert_aws_InfrastructureStatus_To_v1alpha1_InfrastructureStatus(in *a if err := Convert_aws_VPCStatus_To_v1alpha1_VPCStatus(&in.VPC, &out.VPC, s); err != nil { return err } + if err := Convert_aws_CSI_To_v1alpha1_CSI(&in.CSI, &out.CSI, s); err != nil { + return err + } return nil } diff --git a/pkg/apis/aws/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/aws/v1alpha1/zz_generated.deepcopy.go index d8daa52ed..3b475de33 100644 --- a/pkg/apis/aws/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/aws/v1alpha1/zz_generated.deepcopy.go @@ -14,6 +14,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSI) DeepCopyInto(out *CSI) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSI. +func (in *CSI) DeepCopy() *CSI { + if in == nil { + return nil + } + out := new(CSI) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloudControllerManagerConfig) DeepCopyInto(out *CloudControllerManagerConfig) { *out = *in @@ -292,6 +308,11 @@ func (in *InfrastructureConfig) DeepCopyInto(out *InfrastructureConfig) { *out = new(IgnoreTags) (*in).DeepCopyInto(*out) } + if in.EnableCsiEfs != nil { + in, out := &in.EnableCsiEfs, &out.EnableCsiEfs + *out = new(bool) + **out = **in + } return } @@ -352,6 +373,7 @@ func (in *InfrastructureStatus) DeepCopyInto(out *InfrastructureStatus) { out.EC2 = in.EC2 in.IAM.DeepCopyInto(&out.IAM) in.VPC.DeepCopyInto(&out.VPC) + out.CSI = in.CSI return } diff --git a/pkg/apis/aws/zz_generated.deepcopy.go b/pkg/apis/aws/zz_generated.deepcopy.go index 3c4c1d7f5..38efe53bb 100644 --- a/pkg/apis/aws/zz_generated.deepcopy.go +++ b/pkg/apis/aws/zz_generated.deepcopy.go @@ -14,6 +14,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CSI) DeepCopyInto(out *CSI) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSI. +func (in *CSI) DeepCopy() *CSI { + if in == nil { + return nil + } + out := new(CSI) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloudControllerManagerConfig) DeepCopyInto(out *CloudControllerManagerConfig) { *out = *in @@ -292,6 +308,11 @@ func (in *InfrastructureConfig) DeepCopyInto(out *InfrastructureConfig) { *out = new(IgnoreTags) (*in).DeepCopyInto(*out) } + if in.EnableCsiEfs != nil { + in, out := &in.EnableCsiEfs, &out.EnableCsiEfs + *out = new(bool) + **out = **in + } return } @@ -352,6 +373,7 @@ func (in *InfrastructureStatus) DeepCopyInto(out *InfrastructureStatus) { out.EC2 = in.EC2 in.IAM.DeepCopyInto(&out.IAM) in.VPC.DeepCopyInto(&out.VPC) + out.CSI = in.CSI return } diff --git a/pkg/aws/client/client.go b/pkg/aws/client/client.go index b0c96f535..66652d47f 100644 --- a/pkg/aws/client/client.go +++ b/pkg/aws/client/client.go @@ -7,6 +7,7 @@ package client import ( "context" "encoding/json" + "errors" "fmt" "net/url" "reflect" @@ -19,6 +20,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/efs" + "github.com/aws/aws-sdk-go/service/efs/efsiface" "github.com/aws/aws-sdk-go/service/elb" "github.com/aws/aws-sdk-go/service/elb/elbiface" "github.com/aws/aws-sdk-go/service/elbv2" @@ -54,6 +57,7 @@ type Client struct { S3 s3iface.S3API ELB elbiface.ELBAPI ELBv2 elbv2iface.ELBV2API + EFS efsiface.EFSAPI Route53 route53iface.Route53API Route53RateLimiter *rate.Limiter Route53RateLimiterWaitTimeout time.Duration @@ -91,6 +95,7 @@ func NewClient(accessKeyID, secretAccessKey, region string) (*Client, error) { IAM: iam.New(s, config), STS: sts.New(s, config), S3: s3.New(s, config), + EFS: efs.New(s, config), Route53: route53.New(s, config), Route53RateLimiter: rate.NewLimiter(rate.Inf, 0), Route53RateLimiterWaitTimeout: 1 * time.Second, @@ -1965,6 +1970,62 @@ func (c *Client) DeleteIAMRolePolicy(ctx context.Context, policyName, roleName s return ignoreNotFound(err) } +// DescribeEfsFileSystems retrieve information about an efs file system by its ID +func (c *Client) DescribeEfsFileSystems(ctx context.Context, fileSystemID *string) (*efs.FileSystemDescription, error) { + output, err := c.EFS.DescribeFileSystemsWithContext(ctx, &efs.DescribeFileSystemsInput{ + FileSystemId: fileSystemID, + }) + if err != nil { + return nil, err + } + if len(output.FileSystems) != 1 { + return nil, fmt.Errorf("expected 1 file system, got %d", len(output.FileSystems)) + } + return output.FileSystems[0], nil +} + +// CreateEfsFileSystem creates an efs file system +func (c *Client) CreateEfsFileSystem(ctx context.Context, input *efs.CreateFileSystemInput) (*efs.FileSystemDescription, error) { + output, err := c.EFS.CreateFileSystemWithContext(ctx, input) + if ignoreAlreadyExists(err) != nil { + return nil, err + } + var fsDescription *efs.FileSystemDescription + err = c.PollImmediateUntil(ctx, func(ctx context.Context) (bool, error) { + fsDescription, err = c.DescribeEfsFileSystems(ctx, output.FileSystemId) + if err != nil { + return true, err + } + if fsDescription.LifeCycleState != nil && *fsDescription.LifeCycleState == efs.LifeCycleStateAvailable { + return true, nil + } + return false, nil + }) + return fsDescription, err +} + +// DeleteEfsFileSystem deletes an efs file system +func (c *Client) DeleteEfsFileSystem(ctx context.Context, input *efs.DeleteFileSystemInput) error { + _, err := c.EFS.DeleteFileSystemWithContext(ctx, input) + return ignoreNotFound(err) +} + +// DescribeMountTargetsEfs describes an efs mount target +func (c *Client) DescribeMountTargetsEfs(ctx context.Context, input *efs.DescribeMountTargetsInput) (*efs.DescribeMountTargetsOutput, error) { + return c.EFS.DescribeMountTargetsWithContext(ctx, input) +} + +// CreateMountTargetEfs creates an efs mount target +func (c *Client) CreateMountTargetEfs(ctx context.Context, input *efs.CreateMountTargetInput) (*efs.MountTargetDescription, error) { + return c.EFS.CreateMountTargetWithContext(ctx, input) +} + +// DeleteMountTargetEfs deletes an efs mount target +func (c *Client) DeleteMountTargetEfs(ctx context.Context, input *efs.DeleteMountTargetInput) error { + _, err := c.EFS.DeleteMountTargetWithContext(ctx, input) + return err +} + // CreateEC2Tags creates the tags for the given EC2 resource identifiers func (c *Client) CreateEC2Tags(ctx context.Context, resources []string, tags Tags) error { input := &ec2.CreateTagsInput{ @@ -1997,10 +2058,20 @@ func (c *Client) PollUntil(ctx context.Context, condition wait.ConditionWithCont return wait.PollUntilContextCancel(ctx, c.PollInterval, false, condition) } +// IsAlreadyExistsError returns true if the given error is a awserr.Error indicating that an AWS resource was not found. +func IsAlreadyExistsError(err error) bool { + var aerr awserr.Error + if errors.As(err, &aerr) && (aerr.Code() == efs.ErrCodeFileSystemAlreadyExists) { + return true + } + return false +} + // IsNotFoundError returns true if the given error is a awserr.Error indicating that an AWS resource was not found. func IsNotFoundError(err error) bool { if aerr, ok := err.(awserr.Error); ok && (aerr.Code() == elb.ErrCodeAccessPointNotFoundException || aerr.Code() == iam.ErrCodeNoSuchEntityException || aerr.Code() == "NatGatewayNotFound" || + aerr.Code() == efs.ErrCodeFileSystemNotFound || strings.HasSuffix(aerr.Code(), ".NotFound")) { return true } @@ -2015,6 +2086,13 @@ func IsAlreadyAssociatedError(err error) bool { return false } +func ignoreAlreadyExists(err error) error { + if err == nil || IsAlreadyExistsError(err) { + return nil + } + return err +} + func ignoreNotFound(err error) error { if err == nil || IsNotFoundError(err) { return nil diff --git a/pkg/aws/client/mock/mocks.go b/pkg/aws/client/mock/mocks.go index 6224a79b6..98d92c5d4 100644 --- a/pkg/aws/client/mock/mocks.go +++ b/pkg/aws/client/mock/mocks.go @@ -14,6 +14,7 @@ import ( reflect "reflect" ec2 "github.com/aws/aws-sdk-go/service/ec2" + efs "github.com/aws/aws-sdk-go/service/efs" client "github.com/gardener/gardener-extension-provider-aws/pkg/aws/client" gomock "go.uber.org/mock/gomock" sets "k8s.io/apimachinery/pkg/util/sets" @@ -126,6 +127,21 @@ func (mr *MockInterfaceMockRecorder) CreateEC2Tags(arg0, arg1, arg2 any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEC2Tags", reflect.TypeOf((*MockInterface)(nil).CreateEC2Tags), arg0, arg1, arg2) } +// CreateEfsFileSystem mocks base method. +func (m *MockInterface) CreateEfsFileSystem(arg0 context.Context, arg1 *efs.CreateFileSystemInput) (*efs.FileSystemDescription, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateEfsFileSystem", arg0, arg1) + ret0, _ := ret[0].(*efs.FileSystemDescription) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateEfsFileSystem indicates an expected call of CreateEfsFileSystem. +func (mr *MockInterfaceMockRecorder) CreateEfsFileSystem(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateEfsFileSystem", reflect.TypeOf((*MockInterface)(nil).CreateEfsFileSystem), arg0, arg1) +} + // CreateElasticIP mocks base method. func (m *MockInterface) CreateElasticIP(arg0 context.Context, arg1 *client.ElasticIP) (*client.ElasticIP, error) { m.ctrl.T.Helper() @@ -186,6 +202,21 @@ func (mr *MockInterfaceMockRecorder) CreateInternetGateway(arg0, arg1 any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateInternetGateway", reflect.TypeOf((*MockInterface)(nil).CreateInternetGateway), arg0, arg1) } +// CreateMountTargetEfs mocks base method. +func (m *MockInterface) CreateMountTargetEfs(arg0 context.Context, arg1 *efs.CreateMountTargetInput) (*efs.MountTargetDescription, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateMountTargetEfs", arg0, arg1) + ret0, _ := ret[0].(*efs.MountTargetDescription) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateMountTargetEfs indicates an expected call of CreateMountTargetEfs. +func (mr *MockInterfaceMockRecorder) CreateMountTargetEfs(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateMountTargetEfs", reflect.TypeOf((*MockInterface)(nil).CreateMountTargetEfs), arg0, arg1) +} + // CreateNATGateway mocks base method. func (m *MockInterface) CreateNATGateway(arg0 context.Context, arg1 *client.NATGateway) (*client.NATGateway, error) { m.ctrl.T.Helper() @@ -418,6 +449,20 @@ func (mr *MockInterfaceMockRecorder) DeleteELBV2(arg0, arg1 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteELBV2", reflect.TypeOf((*MockInterface)(nil).DeleteELBV2), arg0, arg1) } +// DeleteEfsFileSystem mocks base method. +func (m *MockInterface) DeleteEfsFileSystem(arg0 context.Context, arg1 *efs.DeleteFileSystemInput) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteEfsFileSystem", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteEfsFileSystem indicates an expected call of DeleteEfsFileSystem. +func (mr *MockInterfaceMockRecorder) DeleteEfsFileSystem(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteEfsFileSystem", reflect.TypeOf((*MockInterface)(nil).DeleteEfsFileSystem), arg0, arg1) +} + // DeleteElasticIP mocks base method. func (m *MockInterface) DeleteElasticIP(arg0 context.Context, arg1 string) error { m.ctrl.T.Helper() @@ -502,6 +547,20 @@ func (mr *MockInterfaceMockRecorder) DeleteKeyPair(arg0, arg1 any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteKeyPair", reflect.TypeOf((*MockInterface)(nil).DeleteKeyPair), arg0, arg1) } +// DeleteMountTargetEfs mocks base method. +func (m *MockInterface) DeleteMountTargetEfs(arg0 context.Context, arg1 *efs.DeleteMountTargetInput) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteMountTargetEfs", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteMountTargetEfs indicates an expected call of DeleteMountTargetEfs. +func (mr *MockInterfaceMockRecorder) DeleteMountTargetEfs(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteMountTargetEfs", reflect.TypeOf((*MockInterface)(nil).DeleteMountTargetEfs), arg0, arg1) +} + // DeleteNATGateway mocks base method. func (m *MockInterface) DeleteNATGateway(arg0 context.Context, arg1 string) error { m.ctrl.T.Helper() @@ -656,6 +715,36 @@ func (mr *MockInterfaceMockRecorder) DeleteVpcEndpointRouteTableAssociation(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVpcEndpointRouteTableAssociation", reflect.TypeOf((*MockInterface)(nil).DeleteVpcEndpointRouteTableAssociation), arg0, arg1, arg2) } +// DescribeEfsFileSystems mocks base method. +func (m *MockInterface) DescribeEfsFileSystems(arg0 context.Context, arg1 *string) (*efs.FileSystemDescription, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeEfsFileSystems", arg0, arg1) + ret0, _ := ret[0].(*efs.FileSystemDescription) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeEfsFileSystems indicates an expected call of DescribeEfsFileSystems. +func (mr *MockInterfaceMockRecorder) DescribeEfsFileSystems(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeEfsFileSystems", reflect.TypeOf((*MockInterface)(nil).DescribeEfsFileSystems), arg0, arg1) +} + +// DescribeMountTargetsEfs mocks base method. +func (m *MockInterface) DescribeMountTargetsEfs(arg0 context.Context, arg1 *efs.DescribeMountTargetsInput) (*efs.DescribeMountTargetsOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DescribeMountTargetsEfs", arg0, arg1) + ret0, _ := ret[0].(*efs.DescribeMountTargetsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeMountTargetsEfs indicates an expected call of DescribeMountTargetsEfs. +func (mr *MockInterfaceMockRecorder) DescribeMountTargetsEfs(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeMountTargetsEfs", reflect.TypeOf((*MockInterface)(nil).DescribeMountTargetsEfs), arg0, arg1) +} + // DetachInternetGateway mocks base method. func (m *MockInterface) DetachInternetGateway(arg0 context.Context, arg1, arg2 string) error { m.ctrl.T.Helper() diff --git a/pkg/aws/client/tags.go b/pkg/aws/client/tags.go index bb00945da..359b2fc6a 100644 --- a/pkg/aws/client/tags.go +++ b/pkg/aws/client/tags.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/efs" ) // Tags is map of string key to string values. Duplicate keys are not supported in AWS. @@ -51,6 +52,15 @@ func (tags Tags) ToEC2Tags() []*ec2.Tag { return cp } +// ToEfsTags exports the tags map as a EC2 Tag array. +func (tags Tags) ToEfsTags() []*efs.Tag { + var cp []*efs.Tag + for k, v := range tags { + cp = append(cp, &efs.Tag{Key: aws.String(k), Value: aws.String(v)}) + } + return cp +} + // ToFilters exports the tags map as a EC2 Filter array. func (tags Tags) ToFilters() []*ec2.Filter { if tags == nil { diff --git a/pkg/aws/client/types.go b/pkg/aws/client/types.go index 9a938ee83..ae73436bd 100644 --- a/pkg/aws/client/types.go +++ b/pkg/aws/client/types.go @@ -9,6 +9,7 @@ import ( "sort" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/efs" "k8s.io/apimachinery/pkg/util/sets" ) @@ -164,6 +165,14 @@ type Interface interface { // EC2 tags CreateEC2Tags(ctx context.Context, resources []string, tags Tags) error DeleteEC2Tags(ctx context.Context, resources []string, tags Tags) error + + // Efs + DescribeEfsFileSystems(ctx context.Context, fileSystemID *string) (*efs.FileSystemDescription, error) + CreateEfsFileSystem(ctx context.Context, input *efs.CreateFileSystemInput) (*efs.FileSystemDescription, error) + DeleteEfsFileSystem(ctx context.Context, input *efs.DeleteFileSystemInput) error + DescribeMountTargetsEfs(ctx context.Context, input *efs.DescribeMountTargetsInput) (*efs.DescribeMountTargetsOutput, error) + CreateMountTargetEfs(ctx context.Context, input *efs.CreateMountTargetInput) (*efs.MountTargetDescription, error) + DeleteMountTargetEfs(ctx context.Context, input *efs.DeleteMountTargetInput) error } // Factory creates instances of Interface. diff --git a/pkg/aws/types.go b/pkg/aws/types.go index 3e7c773f2..4d9fafe9b 100644 --- a/pkg/aws/types.go +++ b/pkg/aws/types.go @@ -119,6 +119,11 @@ const ( CSISnapshotValidationName = "csi-snapshot-validation" // CSIVolumeModifierName is the constant for the name of the csi-volume-modifier. CSIVolumeModifierName = "csi-volume-modifier" + + CSIDriverEfsName = "csi-driver-efs" + CSIEfsNodeName = "csi-driver-efs-node" + CSIEfsControllerName = "csi-driver-efs-controller" + CSIDriverEbfImageName = "csi-driver-efs" ) var ( diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index 99dfed6bc..b75681146 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -189,6 +189,17 @@ var ( {Type: &corev1.Service{}, Name: aws.CSISnapshotValidationName}, }, }, + { + Name: "csi-driver-efs-controller", + Images: []string{ + aws.CSIDriverEbfImageName, + aws.CSILivenessProbeImageName, + aws.CSIProvisionerImageName, + }, + Objects: []*chart.Object{ + {Type: &appsv1.Deployment{}, Name: "efs-csi-controller"}, + }, + }, }, } @@ -275,6 +286,22 @@ var ( {Type: &rbacv1.RoleBinding{}, Name: aws.UsernamePrefix + aws.CSIVolumeModifierName}, }, }, + { + Name: aws.CSIEfsNodeName, + Images: []string{ + aws.CSIDriverEbfImageName, + aws.CSINodeDriverRegistrarImageName, + aws.CSILivenessProbeImageName, + }, + Objects: []*chart.Object{ + // csi-driver-efs-node + {Type: &appsv1.DaemonSet{}, Name: "csi-driver-efs-node"}, + {Type: &storagev1.CSIDriver{}, Name: "efs.csi.aws.com"}, + {Type: &corev1.ServiceAccount{}, Name: "efs-csi-node-sa"}, + {Type: &rbacv1.ClusterRole{}, Name: "efs-csi-node-role"}, + {Type: &rbacv1.ClusterRoleBinding{}, Name: "efs-csi-node-binding"}, + }, + }, }, } @@ -329,12 +356,9 @@ func (vp *valuesProvider) GetConfigChartValues( cp *extensionsv1alpha1.ControlPlane, _ *extensionscontroller.Cluster, ) (map[string]interface{}, error) { - // Decode infrastructureProviderStatus - infraStatus := &apisaws.InfrastructureStatus{} - if cp.Spec.InfrastructureProviderStatus != nil { - if _, _, err := vp.decoder.Decode(cp.Spec.InfrastructureProviderStatus.Raw, nil, infraStatus); err != nil { - return nil, fmt.Errorf("could not decode infrastructureProviderStatus of controlplane '%s': %w", k8sclient.ObjectKeyFromObject(cp), err) - } + infraStatus, err := vp.decodeInfrastructureStatus(cp) + if err != nil { + return nil, err } // Get config chart values @@ -350,20 +374,20 @@ func (vp *valuesProvider) GetControlPlaneChartValues( checksums map[string]string, scaledDown bool, ) (map[string]interface{}, error) { - // Decode providerConfig - cpConfig := &apisaws.ControlPlaneConfig{} - if cp.Spec.ProviderConfig != nil { - if _, _, err := vp.decoder.Decode(cp.Spec.ProviderConfig.Raw, nil, cpConfig); err != nil { - return nil, fmt.Errorf("could not decode providerConfig of controlplane '%s': %w", k8sclient.ObjectKeyFromObject(cp), err) - } + cpConfig, err := vp.decodeControlPlaneConfig(cp) + if err != nil { + return nil, err } - // Decode infrastructureProviderStatus - infraStatus := &apisaws.InfrastructureStatus{} - if cp.Spec.InfrastructureProviderStatus != nil { - if _, _, err := vp.decoder.Decode(cp.Spec.InfrastructureProviderStatus.Raw, nil, infraStatus); err != nil { - return nil, fmt.Errorf("could not decode infrastructureProviderStatus of controlplane '%s': %w", k8sclient.ObjectKeyFromObject(cp), err) - } + infraStatus, err := vp.decodeInfrastructureStatus(cp) + if err != nil { + return nil, err + } + + // TODO use function of https://github.com/gardener/gardener-extension-provider-aws/pull/1068 + infraConfig, err := vp.decodeInfrastructureConfig(cluster) + if err != nil { + return nil, err } // TODO(rfranzke): Delete this in a future release. @@ -379,7 +403,7 @@ func (vp *valuesProvider) GetControlPlaneChartValues( } } - return getControlPlaneChartValues(cpConfig, cp, infraStatus, cluster, secretsReader, checksums, scaledDown, gep19Monitoring) + return getControlPlaneChartValues(cpConfig, cp, infraStatus, cluster, secretsReader, infraConfig, checksums, scaledDown, gep19Monitoring) } // GetControlPlaneShootChartValues returns the values for the control plane shoot chart applied by the generic actuator. @@ -390,15 +414,22 @@ func (vp *valuesProvider) GetControlPlaneShootChartValues( secretsReader secretsmanager.Reader, _ map[string]string, ) (map[string]interface{}, error) { - // Decode providerConfig - cpConfig := &apisaws.ControlPlaneConfig{} - if cp.Spec.ProviderConfig != nil { - if _, _, err := vp.decoder.Decode(cp.Spec.ProviderConfig.Raw, nil, cpConfig); err != nil { - return nil, fmt.Errorf("could not decode providerConfig of controlplane '%s': %w", k8sclient.ObjectKeyFromObject(cp), err) - } + cpConfig, err := vp.decodeControlPlaneConfig(cp) + if err != nil { + return nil, err + } + + infraConfig, err := vp.decodeInfrastructureConfig(cluster) + if err != nil { + return nil, err + } + + infraStatus, err := vp.decodeInfrastructureStatus(cp) + if err != nil { + return nil, err } - return getControlPlaneShootChartValues(cluster, cpConfig, cp, secretsReader) + return getControlPlaneShootChartValues(cluster, cpConfig, cp, secretsReader, infraConfig, infraStatus) } // GetControlPlaneShootCRDsChartValues returns the values for the control plane shoot CRDs chart applied by the generic actuator. @@ -408,11 +439,9 @@ func (vp *valuesProvider) GetControlPlaneShootCRDsChartValues( cp *extensionsv1alpha1.ControlPlane, _ *extensionscontroller.Cluster, ) (map[string]interface{}, error) { - cpConfig := &apisaws.ControlPlaneConfig{} - if cp.Spec.ProviderConfig != nil { - if _, _, err := vp.decoder.Decode(cp.Spec.ProviderConfig.Raw, nil, cpConfig); err != nil { - return nil, fmt.Errorf("could not decode providerConfig of controlplane '%s': %w", k8sclient.ObjectKeyFromObject(cp), err) - } + cpConfig, err := vp.decodeControlPlaneConfig(cp) + if err != nil { + return nil, err } return map[string]interface{}{ @@ -453,6 +482,36 @@ func (vp *valuesProvider) GetStorageClassesChartValues( }, nil } +func (vp *valuesProvider) decodeControlPlaneConfig(cp *extensionsv1alpha1.ControlPlane) (*apisaws.ControlPlaneConfig, error) { + cpConfig := &apisaws.ControlPlaneConfig{} + if cp.Spec.ProviderConfig != nil { + if _, _, err := vp.decoder.Decode(cp.Spec.ProviderConfig.Raw, nil, cpConfig); err != nil { + return nil, fmt.Errorf("could not decode providerConfig of controlplane '%s': %w", k8sclient.ObjectKeyFromObject(cp), err) + } + } + return cpConfig, nil +} + +func (vp *valuesProvider) decodeInfrastructureStatus(cp *extensionsv1alpha1.ControlPlane) (*apisaws.InfrastructureStatus, error) { + infraStatus := &apisaws.InfrastructureStatus{} + if cp.Spec.InfrastructureProviderStatus != nil { + if _, _, err := vp.decoder.Decode(cp.Spec.InfrastructureProviderStatus.Raw, nil, infraStatus); err != nil { + return nil, fmt.Errorf("could not decode infrastructureProviderStatus of controlplane '%s': %w", k8sclient.ObjectKeyFromObject(cp), err) + } + } + return infraStatus, nil +} + +func (vp *valuesProvider) decodeInfrastructureConfig(cluster *extensionscontroller.Cluster) (*apisaws.InfrastructureConfig, error) { + infraConfig := &apisaws.InfrastructureConfig{} + if cluster.Shoot != nil && cluster.Shoot.Spec.Provider.InfrastructureConfig != nil { + if _, _, err := vp.decoder.Decode(cluster.Shoot.Spec.Provider.InfrastructureConfig.Raw, nil, infraConfig); err != nil { + return nil, fmt.Errorf("could not decode InfrastructureConfig of cluster shoot '%s': %w", k8sclient.ObjectKeyFromObject(cluster.Shoot), err) + } + } + return infraConfig, nil +} + // getConfigChartValues collects and returns the configuration chart values. func getConfigChartValues( infraStatus *apisaws.InfrastructureStatus, @@ -480,6 +539,7 @@ func getControlPlaneChartValues( infraStatus *apisaws.InfrastructureStatus, cluster *extensionscontroller.Cluster, secretsReader secretsmanager.Reader, + infraConfig *apisaws.InfrastructureConfig, checksums map[string]string, scaledDown bool, gep19Monitoring bool, @@ -504,6 +564,8 @@ func getControlPlaneChartValues( return nil, err } + csiEfs := getCSIEfsControllerChartValues(cp, infraConfig, cluster, scaledDown) + return map[string]interface{}{ "global": map[string]interface{}{ "genericTokenKubeconfigSecretName": extensionscontroller.GenericTokenKubeconfigSecretNameFromCluster(cluster), @@ -512,6 +574,7 @@ func getControlPlaneChartValues( aws.AWSCustomRouteControllerName: crc, aws.AWSLoadBalancerControllerName: alb, aws.CSIControllerName: csi, + aws.CSIEfsControllerName: csiEfs, }, nil } @@ -686,12 +749,34 @@ func getCSIControllerChartValues( }, nil } +// getCSIManilaControllerChartValues collects and returns the CSIController chart values. +func getCSIEfsControllerChartValues( + cp *extensionsv1alpha1.ControlPlane, + infraConfig *apisaws.InfrastructureConfig, + cluster *extensionscontroller.Cluster, + scaledDown bool, +) map[string]interface{} { + csiEfsEnabled := isCSIEfsEnabled(infraConfig) + values := map[string]interface{}{ + "enabled": csiEfsEnabled, + } + + if csiEfsEnabled { + values["replicas"] = extensionscontroller.GetControlPlaneReplicas(cluster, scaledDown, 1) + values["region"] = cp.Spec.Region + } + + return values +} + // getControlPlaneShootChartValues collects and returns the control plane shoot chart values. func getControlPlaneShootChartValues( cluster *extensionscontroller.Cluster, cpConfig *apisaws.ControlPlaneConfig, cp *extensionsv1alpha1.ControlPlane, secretsReader secretsmanager.Reader, + infraConfig *apisaws.InfrastructureConfig, + infraStatus *apisaws.InfrastructureStatus, ) (map[string]interface{}, error) { kubernetesVersion := cluster.Shoot.Spec.Kubernetes.Version @@ -725,10 +810,33 @@ func getControlPlaneShootChartValues( return nil, err } + csiDriverEfsValues := getControlPlaneShootChartCSIEfsValues(infraConfig, infraStatus) + return map[string]interface{}{ aws.CloudControllerManagerName: map[string]interface{}{"enabled": true}, aws.AWSCustomRouteControllerName: map[string]interface{}{"enabled": customRouteControllerEnabled}, aws.AWSLoadBalancerControllerName: albValues, aws.CSINodeName: csiDriverNodeValues, + aws.CSIEfsNodeName: csiDriverEfsValues, }, nil } + +func isCSIEfsEnabled(infraConfig *apisaws.InfrastructureConfig) bool { + return infraConfig != nil && infraConfig.EnableCsiEfs != nil && *infraConfig.EnableCsiEfs +} + +func getControlPlaneShootChartCSIEfsValues( + infraConfig *apisaws.InfrastructureConfig, + infraStatus *apisaws.InfrastructureStatus, +) map[string]interface{} { + csiEsfEnabled := isCSIEfsEnabled(infraConfig) + values := map[string]interface{}{ + "enabled": csiEsfEnabled, + } + + if csiEsfEnabled { + values["fileSystemID"] = infraStatus.CSI.EfsSystemID + } + + return values +} diff --git a/pkg/controller/controlplane/valuesprovider_test.go b/pkg/controller/controlplane/valuesprovider_test.go index ea87ce9ee..9d04a699d 100644 --- a/pkg/controller/controlplane/valuesprovider_test.go +++ b/pkg/controller/controlplane/valuesprovider_test.go @@ -330,6 +330,7 @@ var _ = Describe("ValuesProvider", func() { "topologyAwareRoutingEnabled": false, }, }), + aws.CSIEfsControllerName: enabledFalse, })) }) @@ -366,6 +367,7 @@ var _ = Describe("ValuesProvider", func() { "topologyAwareRoutingEnabled": false, }, }), + aws.CSIEfsControllerName: enabledFalse, })) }) @@ -401,6 +403,7 @@ var _ = Describe("ValuesProvider", func() { "topologyAwareRoutingEnabled": false, }, }), + aws.CSIEfsControllerName: enabledFalse, })) }) @@ -437,6 +440,7 @@ var _ = Describe("ValuesProvider", func() { "topologyAwareRoutingEnabled": false, }, }), + aws.CSIEfsControllerName: enabledFalse, })) }) @@ -516,6 +520,7 @@ var _ = Describe("ValuesProvider", func() { "caBundle": "", }, }), + aws.CSIEfsNodeName: enabledFalse, })) }) }) @@ -540,6 +545,7 @@ var _ = Describe("ValuesProvider", func() { "caBundle": "", }, }), + aws.CSIEfsNodeName: enabledFalse, })) }) }) @@ -579,6 +585,7 @@ var _ = Describe("ValuesProvider", func() { "caBundle": "", }, }), + aws.CSIEfsNodeName: enabledFalse, })) }) }) diff --git a/pkg/controller/infrastructure/infraflow/context.go b/pkg/controller/infrastructure/infraflow/context.go index 7af28ade3..874c5a8f0 100644 --- a/pkg/controller/infrastructure/infraflow/context.go +++ b/pkg/controller/infrastructure/infraflow/context.go @@ -88,6 +88,8 @@ const ( KeyPairFingerprint = "KeyPairFingerprint" // KeyPairSpecFingerprint is the key to store the fingerprint of the public key from the spec KeyPairSpecFingerprint = "KeyPairSpecFingerprint" + // NameEfsSystemID is the key for the + NameEfsSystemID = "efsSystemID" // ChildIdVPCEndpoints is the child key for the VPC endpoints ChildIdVPCEndpoints = "VPCEndpoints" @@ -210,6 +212,7 @@ func (c *FlowContext) computeInfrastructureStatus() *awsv1alpha1.InfrastructureS ec2KeyName := ptr.Deref(c.state.Get(NameKeyPair), "") iamInstanceProfileName := ptr.Deref(c.state.Get(NameIAMInstanceProfile), "") arnIAMRole := ptr.Deref(c.state.Get(ARNIAMRole), "") + efsSystemID := ptr.Deref(c.state.Get(NameEfsSystemID), "") if c.config.Networks.VPC.ID != nil { vpcID = *c.config.Networks.VPC.ID @@ -279,6 +282,10 @@ func (c *FlowContext) computeInfrastructureStatus() *awsv1alpha1.InfrastructureS } } + if efsSystemID != "" { + status.CSI.EfsSystemID = efsSystemID + } + return status } @@ -356,6 +363,10 @@ func (c *FlowContext) zoneSuffixHelpers(zoneName string) *ZoneSuffixHelper { } } +func (c *FlowContext) isCsiEfsEnabled() bool { + return c.config != nil && c.config.EnableCsiEfs != nil && *c.config.EnableCsiEfs +} + // ZoneSuffixHelper provides methods to create suffices for various resources type ZoneSuffixHelper struct { suffix string diff --git a/pkg/controller/infrastructure/infraflow/delete.go b/pkg/controller/infrastructure/infraflow/delete.go index d89b2abd4..119e5e360 100644 --- a/pkg/controller/infrastructure/infraflow/delete.go +++ b/pkg/controller/infrastructure/infraflow/delete.go @@ -9,8 +9,10 @@ import ( "fmt" "time" + "github.com/aws/aws-sdk-go/service/efs" "github.com/gardener/gardener/extensions/pkg/util" "github.com/gardener/gardener/pkg/utils/flow" + "k8s.io/utils/ptr" "github.com/gardener/gardener-extension-provider-aws/pkg/apis/aws/helper" awsclient "github.com/gardener/gardener-extension-provider-aws/pkg/aws/client" @@ -57,6 +59,11 @@ func (c *FlowContext) buildDeleteGraph() *flow.Graph { c.deleteIAMRole, Timeout(defaultTimeout), Dependencies(deleteIAMInstanceProfile, deleteIAMRolePolicy)) + // TODO check if should use dependency? + _ = c.AddTask(g, "delete efs file system", + c.deleteEfsFileSystem, + DoIf(c.isCsiEfsEnabled()), Timeout(defaultTimeout)) + deleteZones := c.AddTask(g, "delete zones resources", c.deleteZones, DoIf(c.hasVPC()), Timeout(defaultLongTimeout)) @@ -342,3 +349,39 @@ func (c *FlowContext) deleteKeyPair(ctx context.Context) error { c.state.Delete(NameKeyPair) return nil } + +func (c *FlowContext) deleteEfsFileSystem(ctx context.Context) error { + efsSystemID := c.state.Get(NameEfsSystemID) + if efsSystemID == nil { + return nil + } + log := LogFromContext(ctx) + + efsMounts, err := c.client.DescribeMountTargetsEfs(ctx, &efs.DescribeMountTargetsInput{ + FileSystemId: efsSystemID, + }) + if err != nil { + return err + } + + for _, mount := range efsMounts.MountTargets { + log.Info("deleting...", "efsMountTarget", ptr.Deref(mount.MountTargetId, "")) + err = c.client.DeleteMountTargetEfs(ctx, &efs.DeleteMountTargetInput{ + MountTargetId: mount.MountTargetId, + }) + if err != nil { + return err + } + } + + log.Info("deleting...", "efsFileSystem", *efsSystemID) + err = c.client.DeleteEfsFileSystem(ctx, &efs.DeleteFileSystemInput{ + FileSystemId: efsSystemID, + }) + if err != nil { + return err + } + + c.state.Delete(NameEfsSystemID) + return nil +} diff --git a/pkg/controller/infrastructure/infraflow/reconcile.go b/pkg/controller/infrastructure/infraflow/reconcile.go index 3d5fe21ff..529ee2b24 100644 --- a/pkg/controller/infrastructure/infraflow/reconcile.go +++ b/pkg/controller/infrastructure/infraflow/reconcile.go @@ -18,6 +18,7 @@ import ( "time" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/efs" "github.com/gardener/gardener/pkg/utils/flow" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" @@ -88,6 +89,10 @@ func (c *FlowContext) buildReconcileGraph() *flow.Graph { c.ensureZones, Timeout(defaultLongTimeout), Dependencies(ensureVpc, ensureNodesSecurityGroup, ensureVpcIPv6CidrBloc, ensureMainRouteTable)) + _ = c.AddTask(g, "ensure efs file system", + c.ensureEfsFileSystem, + DoIf(c.isCsiEfsEnabled()), Timeout(defaultTimeout), Dependencies(ensureZones)) + _ = c.AddTask(g, "ensure egress CIDRs", c.ensureEgressCIDRs, Timeout(defaultLongTimeout), Dependencies(ensureZones)) @@ -517,6 +522,15 @@ func (c *FlowContext) ensureNodesSecurityGroup(ctx context.Context) error { Protocol: "udp", CidrBlocks: []string{zone.Public}, }) + if c.isCsiEfsEnabled() { + desired.Rules = append(desired.Rules, &awsclient.SecurityGroupRule{ + Type: awsclient.SecurityGroupRuleTypeIngress, + FromPort: 2049, + ToPort: 2049, + Protocol: "tcp", + CidrBlocks: []string{zone.Internal}, + }) + } } current, err := FindExisting(ctx, c.state.Get(IdentifierNodesSecurityGroup), c.commonTagsWithSuffix("nodes"), c.client.GetSecurityGroup, c.client.FindSecurityGroupsByTags, @@ -1276,7 +1290,7 @@ const iamRolePolicyTemplate = `{ "Resource": [ "*" ] - }{{ if .enableECRAccess }}, + } {{ if .enableECRAccess }}, { "Effect": "Allow", "Action": [ @@ -1392,6 +1406,70 @@ func (c *FlowContext) ensureKeyPair(ctx context.Context) error { return nil } +func (c *FlowContext) ensureEfsFileSystem(ctx context.Context) error { + if c.state.Get(NameEfsSystemID) != nil { + return nil + } + + // TODO add flags like: performanceMode, Throughput... + inputCreate := &efs.CreateFileSystemInput{ + Tags: c.commonTags.ToEfsTags(), + CreationToken: ptr.To(createEfsCreationToken(c.namespace)), + Encrypted: ptr.To(true), + } + efsCreate, err := c.client.CreateEfsFileSystem(ctx, inputCreate) + if err != nil { + return err + } + + if efsCreate.FileSystemId == nil { + return fmt.Errorf("the created file system id is ") + } + + securityGroupID := c.state.Get(IdentifierNodesSecurityGroup) + if securityGroupID == nil { + return fmt.Errorf("security group not found in state") + } + + childZones := c.state.GetChild(ChildIdZones) + for _, zoneKey := range childZones.GetChildrenKeys() { + zoneChild := childZones.GetChild(zoneKey) + subnetID := zoneChild.Get(IdentifierZoneSubnetWorkers) + if subnetID == nil { + return fmt.Errorf("subnet not found in state") + } + inputMount := &efs.CreateMountTargetInput{ + FileSystemId: efsCreate.FileSystemId, + SecurityGroups: []*string{securityGroupID}, + SubnetId: subnetID, + } + _, err = c.client.CreateMountTargetEfs(ctx, inputMount) + if err != nil { + return err + } + } + + c.state.Set(NameEfsSystemID, *efsCreate.FileSystemId) + return nil +} + +func createEfsCreationToken(namespace string) string { + var efsCreationToken string + tokenCandidate := fmt.Sprintf("efs-token-%s", namespace) + // only allow ASCII chars + for _, r := range tokenCandidate { + if r <= 127 { + efsCreationToken += string(r) + } + } + // restrict string to 64 characters + if len(efsCreationToken) > 64 { + efsCreationToken = efsCreationToken[:64] + } + + return efsCreationToken +} + func (c *FlowContext) getSubnetZoneChildByItem(item *awsclient.Subnet) Whiteboard { return c.getSubnetZoneChild(getZoneName(item)) }