From f8f739a8cb326317f18b4c21548f2c4640ab0c71 Mon Sep 17 00:00:00 2001 From: Hitesh Makol Date: Tue, 24 Dec 2024 18:06:14 +0530 Subject: [PATCH 1/6] Karpenter Helm Chart - includes prerequisites - IAM_Role, Tagging, AWS_AUth as well --- charts/karpenter/.helmignore | 23 + charts/karpenter/Chart.yaml | 24 + .../karpenter/templates/aws_auth_update.yaml | 40 ++ charts/karpenter/templates/karpenter.yaml | 424 ++++++++++++++++++ .../templates/karpenter_iam_roles.yaml | 152 +++++++ charts/karpenter/templates/rbac.yaml | 31 ++ charts/karpenter/templates/tagging.yaml | 62 +++ charts/karpenter/values.yaml | 15 + 8 files changed, 771 insertions(+) create mode 100644 charts/karpenter/.helmignore create mode 100644 charts/karpenter/Chart.yaml create mode 100644 charts/karpenter/templates/aws_auth_update.yaml create mode 100644 charts/karpenter/templates/karpenter.yaml create mode 100644 charts/karpenter/templates/karpenter_iam_roles.yaml create mode 100644 charts/karpenter/templates/rbac.yaml create mode 100644 charts/karpenter/templates/tagging.yaml create mode 100644 charts/karpenter/values.yaml diff --git a/charts/karpenter/.helmignore b/charts/karpenter/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/charts/karpenter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml new file mode 100644 index 00000000..08938495 --- /dev/null +++ b/charts/karpenter/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: new +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/charts/karpenter/templates/aws_auth_update.yaml b/charts/karpenter/templates/aws_auth_update.yaml new file mode 100644 index 00000000..1791e638 --- /dev/null +++ b/charts/karpenter/templates/aws_auth_update.yaml @@ -0,0 +1,40 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: update-aws-auth + namespace: kube-system +spec: + template: + spec: + serviceAccountName: updater-sa + containers: + - name: aws-auth-updater + image: bitnami/kubectl:latest + command: + - /bin/sh + - -c + - | + echo "Fetching aws-auth ConfigMap..." + kubectl get configmap aws-auth -n kube-system -o yaml > /tmp/aws-auth.yaml + + echo "Validating fetched aws-auth ConfigMap..." + cat /tmp/aws-auth.yaml + + echo "Modifying aws-auth ConfigMap..." + # Replace with Helm templating for EC2PrivateDNSName + yq eval '(.data.mapRoles |= . + "\n- groups:\n - system:bootstrappers\n - system:nodes\n rolearn: arn:{{ .Values.awsPartition }}:iam::{{ .Values.awsAccountId }}:role/KarpenterNodeRole-{{ .Values.clusterName }}\n username: system:node:{{ .Values.ec2PrivateDNSName }}")' -i /tmp/aws-auth.yaml + + echo "Validating modified aws-auth ConfigMap..." + cat /tmp/aws-auth.yaml + + echo "Applying updated aws-auth ConfigMap..." + kubectl apply -f /tmp/aws-auth.yaml -n kube-system --validate=false + volumeMounts: + - name: tmp-volume + mountPath: /tmp + volumes: + - name: tmp-volume + emptyDir: {} + restartPolicy: Never + backoffLimit: 4 + diff --git a/charts/karpenter/templates/karpenter.yaml b/charts/karpenter/templates/karpenter.yaml new file mode 100644 index 00000000..c6cddd65 --- /dev/null +++ b/charts/karpenter/templates/karpenter.yaml @@ -0,0 +1,424 @@ +--- +# Source: karpenter/templates/poddisruptionbudget.yaml +#apiVersion: policy/v1 +#kind: PodDisruptionBudget +#metadata: +# name: karpenter +# namespace: default +# labels: +# helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} +# app.kubernetes.io/name: karpenter +# app.kubernetes.io/instance: karpenter +# app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" +# app.kubernetes.io/managed-by: Helm +#spec: +# maxUnavailable: 1 +# selector: +# matchLabels: +# app.kubernetes.io/name: karpenter +# app.kubernetes.io/instance: karpenter +#--- +# Source: karpenter/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: karpenter + namespace: default + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm + annotations: + eks.amazonaws.com/role-arn: arn:{{ .Values.awsPartition }}:iam::{{ .Values.awsAccountId }}:role/KarpenterControllerRole-{{ .Values.karpenterVersion }} +--- +# Source: karpenter/templates/aggregate-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: karpenter-admin + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: ["karpenter.k8s.aws"] + resources: ["ec2nodeclasses"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] +--- +# Source: karpenter/templates/clusterrole-core.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: karpenter-core + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +rules: + # Read + - apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes", "volumeattachments"] + verbs: ["get", "watch", "list"] + - apiGroups: ["apps"] + resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] + verbs: ["list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", list, "watch"] + # Write + - apiGroups: ["karpenter.sh"] + resources: ["nodeclaims", "nodeclaims/status"] + verbs: ["create", "delete", "update", "patch"] + - apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["patch", "delete", "update"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["delete"] +--- +# Source: karpenter/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: karpenter + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +rules: + # Read + - apiGroups: ["karpenter.k8s.aws"] + resources: ["ec2nodeclasses"] + verbs: ["get", "list", "watch"] + # Write + - apiGroups: ["karpenter.k8s.aws"] + resources: ["ec2nodeclasses", "ec2nodeclasses/status"] + verbs: ["patch", "update"] +--- +# Source: karpenter/templates/clusterrole-core.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: karpenter-core + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: karpenter-core +subjects: + - kind: ServiceAccount + name: karpenter + namespace: default +--- +# Source: karpenter/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: karpenter + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: karpenter +subjects: + - kind: ServiceAccount + name: karpenter + namespace: default +--- +# Source: karpenter/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: karpenter + namespace: default + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +rules: + # Read + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch"] + # Write + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["patch", "update"] + resourceNames: + - "karpenter-leader-election" + # Cannot specify resourceNames on create + # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] +--- +# Source: karpenter/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: karpenter-dns + namespace: kube-system + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +rules: + # Read + - apiGroups: [""] + resources: ["services"] + resourceNames: ["kube-dns"] + verbs: ["get"] +--- +# Source: karpenter/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: karpenter + namespace: default + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: karpenter +subjects: + - kind: ServiceAccount + name: karpenter + namespace: default +--- +# Source: karpenter/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: karpenter-dns + namespace: kube-system + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: karpenter-dns +subjects: + - kind: ServiceAccount + name: karpenter + namespace: default +--- +# Source: karpenter/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: karpenter + namespace: default + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 8080 + targetPort: http-metrics + protocol: TCP + selector: + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter +--- +# Source: karpenter/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karpenter + namespace: default + labels: + helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 2 + revisionHistoryLimit: 10 + strategy: + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + template: + metadata: + labels: + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + annotations: + spec: + serviceAccountName: karpenter + securityContext: + fsGroup: 65532 + priorityClassName: "system-cluster-critical" + dnsPolicy: ClusterFirst + schedulerName: "default-scheduler" + containers: + - name: controller + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + image: public.ecr.aws/karpenter/controller:{{ .Values.karpenterVersion }} + imagePullPolicy: IfNotPresent + env: + - name: KUBERNETES_MIN_VERSION + value: "1.19.0-0" + - name: KARPENTER_SERVICE + value: karpenter + - name: LOG_LEVEL + value: "info" + - name: LOG_OUTPUT_PATHS + value: "stdout" + - name: LOG_ERROR_OUTPUT_PATHS + value: "stderr" + - name: METRICS_PORT + value: "8080" + - name: HEALTH_PROBE_PORT + value: "8081" + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MEMORY_LIMIT + valueFrom: + resourceFieldRef: + containerName: controller + divisor: "0" + resource: limits.memory + - name: FEATURE_GATES + value: "SpotToSpotConsolidation=false,NodeRepair=false" + - name: BATCH_MAX_DURATION + value: "10s" + - name: BATCH_IDLE_DURATION + value: "1s" + - name: VM_MEMORY_OVERHEAD_PERCENT + value: "0.075" + - name: RESERVED_ENIS + value: "0" + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: http + containerPort: 8081 + protocol: TCP + livenessProbe: + initialDelaySeconds: 30 + timeoutSeconds: 30 + httpGet: + path: /healthz + port: http + readinessProbe: + initialDelaySeconds: 5 + timeoutSeconds: 30 + httpGet: + path: /readyz + port: http + resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 1 + memory: 1Gi + nodeSelector: + kubernetes.io/os: linux + # The template below patches the .Values.affinity to add a default label selector where not specificed + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: karpenter.sh/nodepool + operator: DoesNotExist + - key: eks.amazonaws.com/nodegroup + operator: In + values: + - {{ .Values.nodegroupName }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/instance: karpenter + app.kubernetes.io/name: karpenter + topologyKey: kubernetes.io/hostname + # The template below patches the .Values.topologySpreadConstraints to add a default label selector where not specificed + topologySpreadConstraints: + - labelSelector: + matchLabels: + app.kubernetes.io/instance: karpenter + app.kubernetes.io/name: karpenter + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + tolerations: + - key: CriticalAddonsOnly + operator: Exists diff --git a/charts/karpenter/templates/karpenter_iam_roles.yaml b/charts/karpenter/templates/karpenter_iam_roles.yaml new file mode 100644 index 00000000..2878c2c3 --- /dev/null +++ b/charts/karpenter/templates/karpenter_iam_roles.yaml @@ -0,0 +1,152 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: create-karpenter-iam-roles +spec: + template: + spec: + containers: + - name: create-iam-roles + image: amazonlinux:2 + command: + - /bin/sh + - -c + - | + # Install AWS CLI + yum update -y + yum install -y aws-cli + + # Now run the rest of your commands + echo '{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + }' > node-trust-policy.json + + aws iam create-role --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ + --assume-role-policy-document file://node-trust-policy.json + + aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKSWorkerNodePolicy" + + aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKS_CNI_Policy" + + aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + + aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonSSMManagedInstanceCore" + + # Create controller trust policy + cat << EOF > controller-trust-policy.json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:oidc-provider/${OIDC_ENDPOINT#*//}" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "${OIDC_ENDPOINT#*//}:aud": "sts.amazonaws.com", + "${OIDC_ENDPOINT#*//}:sub": "system:serviceaccount:${KARPENTER_NAMESPACE}:karpenter" + } + } + } + ] + } + EOF + + aws iam create-role --role-name "KarpenterControllerRole-${CLUSTER_NAME}" \ + --assume-role-policy-document file://controller-trust-policy.json + + # Create controller policy + cat << EOF > controller-policy.json + { + "Statement": [ + { + "Action": [ + "ssm:GetParameter", + "ec2:DescribeImages", + "ec2:RunInstances", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DeleteLaunchTemplate", + "ec2:CreateTags", + "ec2:CreateLaunchTemplate", + "ec2:CreateFleet", + "ec2:DescribeSpotPriceHistory", + "pricing:GetProducts" + ], + "Effect": "Allow", + "Resource": "*", + "Sid": "Karpenter" + }, + { + "Action": "ec2:TerminateInstances", + "Condition": { + "StringLike": { + "ec2:ResourceTag/karpenter.sh/nodepool": "*" + } + }, + "Effect": "Allow", + "Resource": "*", + "Sid": "ConditionalEC2Termination" + }, + { + "Effect": "Allow", + "Action": "iam:PassRole", + "Resource": "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}", + "Sid": "PassNodeIAMRole" + }, + { + "Effect": "Allow", + "Action": "eks:DescribeCluster", + "Resource": "arn:${AWS_PARTITION}:eks:${AWS_REGION}:${AWS_ACCOUNT_ID}:cluster/${CLUSTER_NAME}", + "Sid": "EKSClusterEndpointLookup" + } + ], + "Version": "2012-10-17" + } + EOF + + aws iam put-role-policy --role-name "KarpenterControllerRole-${CLUSTER_NAME}" \ + --policy-name "KarpenterControllerPolicy-${CLUSTER_NAME}" \ + --policy-document file://controller-policy.json + env: + - name: AWS_REGION + value: "{{ .Values.awsRegion }}" + - name: CLUSTER_NAME + value: "{{ .Values.clusterName }}" + - name: AWS_PARTITION + value: "{{ .Values.awsPartition }}" + - name: AWS_ACCOUNT_ID + value: "{{ .Values.awsAccountId }}" + - name: OIDC_ENDPOINT + value: "{{ .Values.oidcEndpoint }}" + - name: KARPENTER_NAMESPACE + value: "{{ .Values.karpenterNamespace }}" + - name: K8S_VERSION + value: "{{ .Values.k8sVersion }}" + - name: ARM_AMI_ID + value: "{{ .Values.amiIds.arm }}" + - name: AMD_AMI_ID + value: "{{ .Values.amiIds.amd }}" + - name: GPU_AMI_ID + value: "{{ .Values.amiIds.gpu }}" + restartPolicy: Never + diff --git a/charts/karpenter/templates/rbac.yaml b/charts/karpenter/templates/rbac.yaml new file mode 100644 index 00000000..b0ebb769 --- /dev/null +++ b/charts/karpenter/templates/rbac.yaml @@ -0,0 +1,31 @@ +# rbac.yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: updater-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: update-aws-auth +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "patch", "update"] + resourceNames: ["aws-auth"] # Restrict to only this ConfigMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: update-aws-auth-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: update-aws-auth +subjects: + - kind: ServiceAccount + name: updater-sa + namespace: kube-system # Namespace where aws-auth resides + diff --git a/charts/karpenter/templates/tagging.yaml b/charts/karpenter/templates/tagging.yaml new file mode 100644 index 00000000..d5a216b4 --- /dev/null +++ b/charts/karpenter/templates/tagging.yaml @@ -0,0 +1,62 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: karpenter-tagging-job +spec: + template: + spec: + containers: + - name: karpenter-tagging + image: amazonlinux:2 + imagePullPolicy: IfNotPresent + env: + - name: AWS_REGION + value: "{{ .Values.awsRegion }}" # AWS Region from values.yaml + - name: CLUSTER_NAME + value: "{{ .Values.clusterName }}" # EKS Cluster Name from values.yaml + command: + - "/bin/sh" + - "-c" + - | + #!/bin/bash + set -e + + # Install AWS CLI + yum update -y + yum install -y aws-cli + + # Export environment variables for AWS CLI + export AWS_REGION="${AWS_REGION}" + export CLUSTER_NAME="${CLUSTER_NAME}" + + # Add tags to subnets for all nodegroups + for NODEGROUP in $(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" --region "${AWS_REGION}" --query 'nodegroups' --output text); do + aws ec2 create-tags \ + --region "${AWS_REGION}" \ + --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ + --resources $(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" --nodegroup-name "${NODEGROUP}" \ + --region "${AWS_REGION}" --query 'nodegroup.subnets' --output text ) + done + + # Add tags to security groups for the first nodegroup + NODEGROUP=$(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" --region "${AWS_REGION}" --query 'nodegroups[0]' --output text) + + LAUNCH_TEMPLATE=$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" --nodegroup-name "${NODEGROUP}" \ + --region "${AWS_REGION}" --query 'nodegroup.launchTemplate.{id:id,version:version}' --output text | tr -s "\t" ",") + + # If using Cluster security group + SECURITY_GROUPS=$(aws eks describe-cluster --name "${CLUSTER_NAME}" --region "${AWS_REGION}" \ + --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) + + # If using Launch template security groups + SECURITY_GROUPS="$(aws ec2 describe-launch-template-versions --launch-template-id "${LAUNCH_TEMPLATE%,*}" \ + --versions "${LAUNCH_TEMPLATE#*,}" --region "${AWS_REGION}" \ + --query 'LaunchTemplateVersions[0].LaunchTemplateData.[NetworkInterfaces[0].Groups||SecurityGroupIds]' --output text)" + + aws ec2 create-tags \ + --region "${AWS_REGION}" \ + --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ + --resources "${SECURITY_GROUPS}" + restartPolicy: Never + backoffLimit: 4 + diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml new file mode 100644 index 00000000..8f06c529 --- /dev/null +++ b/charts/karpenter/values.yaml @@ -0,0 +1,15 @@ +# values.yaml +clusterName: "demo-cluster" # Replace with your actual cluster name +karpenterNamespace: "kube-system" # Default namespace for Karpenter +karpenterVersion: "1.1.1" +awsPartition: "aws" # Change to aws-cn or aws-us-gov if applicable +ec2PrivateDNSName: "{{EC2PrivateDNSName}}" +awsRegion: "ap-south-1" # This will be populated from the AWS CLI +oidcEndpoint: "https://oidc.eks.ap-south-1.amazonaws.com/id/89026BA50DBC3476D1DA2DA155EBB574" # This will be populated from the AWS CLI +awsAccountId: "084828584148" # This will be populated from the AWS CLI +k8sVersion: "1.30" # Kubernetes version you're using +amiIds: + arm: "ami-0e3f1069f5a85fc29" # Placeholder for ARM AMI ID + amd: "ami-0308b0e27f09b0b25" # Placeholder for AMD AMI ID + gpu: "ami-0761bb90cfee48660" # Placeholder for GPU AMI ID + From 0a7a98309b8fab0bc8a779fcb4342e063f6a6148 Mon Sep 17 00:00:00 2001 From: hiteshmakol1 Date: Tue, 24 Dec 2024 18:11:08 +0530 Subject: [PATCH 2/6] Update values.yaml Updated Values.yaml --- charts/karpenter/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 8f06c529..4849af24 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -5,8 +5,8 @@ karpenterVersion: "1.1.1" awsPartition: "aws" # Change to aws-cn or aws-us-gov if applicable ec2PrivateDNSName: "{{EC2PrivateDNSName}}" awsRegion: "ap-south-1" # This will be populated from the AWS CLI -oidcEndpoint: "https://oidc.eks.ap-south-1.amazonaws.com/id/89026BA50DBC3476D1DA2DA155EBB574" # This will be populated from the AWS CLI -awsAccountId: "084828584148" # This will be populated from the AWS CLI +oidcEndpoint: "https://oidc.eks.ap-south-1.amazonaws.com/id/xyx" # This will be populated from the AWS CLI +awsAccountId: "9999999999" # This will be populated from the AWS CLI k8sVersion: "1.30" # Kubernetes version you're using amiIds: arm: "ami-0e3f1069f5a85fc29" # Placeholder for ARM AMI ID From 6f6c3e173dd4a8b99ff8fe0f8a3804998e0415ff Mon Sep 17 00:00:00 2001 From: Hitesh Makol Date: Thu, 26 Dec 2024 16:40:33 +0530 Subject: [PATCH 3/6] Added Dependency Chart, updated values.yaml --- charts/karpenter/Chart.yaml | 26 +- .../karpenter/templates/aws_auth_update.yaml | 40 -- charts/karpenter/templates/karpenter.yaml | 424 ------------------ .../templates/karpenter_iam_roles.yaml | 152 ------- charts/karpenter/templates/rbac.yaml | 31 -- charts/karpenter/templates/tagging.yaml | 62 --- charts/karpenter/values.yaml | 36 +- 7 files changed, 27 insertions(+), 744 deletions(-) delete mode 100644 charts/karpenter/templates/aws_auth_update.yaml delete mode 100644 charts/karpenter/templates/karpenter.yaml delete mode 100644 charts/karpenter/templates/karpenter_iam_roles.yaml delete mode 100644 charts/karpenter/templates/rbac.yaml delete mode 100644 charts/karpenter/templates/tagging.yaml diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml index 08938495..7cae808f 100644 --- a/charts/karpenter/Chart.yaml +++ b/charts/karpenter/Chart.yaml @@ -1,24 +1,8 @@ apiVersion: v2 -name: new -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) +name: my-karpenter-chart version: 0.1.0 +dependencies: + - name: karpenter + version: 1.1.1 + repository: oci://public.ecr.aws/karpenter -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/charts/karpenter/templates/aws_auth_update.yaml b/charts/karpenter/templates/aws_auth_update.yaml deleted file mode 100644 index 1791e638..00000000 --- a/charts/karpenter/templates/aws_auth_update.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: update-aws-auth - namespace: kube-system -spec: - template: - spec: - serviceAccountName: updater-sa - containers: - - name: aws-auth-updater - image: bitnami/kubectl:latest - command: - - /bin/sh - - -c - - | - echo "Fetching aws-auth ConfigMap..." - kubectl get configmap aws-auth -n kube-system -o yaml > /tmp/aws-auth.yaml - - echo "Validating fetched aws-auth ConfigMap..." - cat /tmp/aws-auth.yaml - - echo "Modifying aws-auth ConfigMap..." - # Replace with Helm templating for EC2PrivateDNSName - yq eval '(.data.mapRoles |= . + "\n- groups:\n - system:bootstrappers\n - system:nodes\n rolearn: arn:{{ .Values.awsPartition }}:iam::{{ .Values.awsAccountId }}:role/KarpenterNodeRole-{{ .Values.clusterName }}\n username: system:node:{{ .Values.ec2PrivateDNSName }}")' -i /tmp/aws-auth.yaml - - echo "Validating modified aws-auth ConfigMap..." - cat /tmp/aws-auth.yaml - - echo "Applying updated aws-auth ConfigMap..." - kubectl apply -f /tmp/aws-auth.yaml -n kube-system --validate=false - volumeMounts: - - name: tmp-volume - mountPath: /tmp - volumes: - - name: tmp-volume - emptyDir: {} - restartPolicy: Never - backoffLimit: 4 - diff --git a/charts/karpenter/templates/karpenter.yaml b/charts/karpenter/templates/karpenter.yaml deleted file mode 100644 index c6cddd65..00000000 --- a/charts/karpenter/templates/karpenter.yaml +++ /dev/null @@ -1,424 +0,0 @@ ---- -# Source: karpenter/templates/poddisruptionbudget.yaml -#apiVersion: policy/v1 -#kind: PodDisruptionBudget -#metadata: -# name: karpenter -# namespace: default -# labels: -# helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} -# app.kubernetes.io/name: karpenter -# app.kubernetes.io/instance: karpenter -# app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" -# app.kubernetes.io/managed-by: Helm -#spec: -# maxUnavailable: 1 -# selector: -# matchLabels: -# app.kubernetes.io/name: karpenter -# app.kubernetes.io/instance: karpenter -#--- -# Source: karpenter/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: karpenter - namespace: default - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm - annotations: - eks.amazonaws.com/role-arn: arn:{{ .Values.awsPartition }}:iam::{{ .Values.awsAccountId }}:role/KarpenterControllerRole-{{ .Values.karpenterVersion }} ---- -# Source: karpenter/templates/aggregate-clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: karpenter-admin - labels: - rbac.authorization.k8s.io/aggregate-to-admin: "true" - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -rules: - - apiGroups: ["karpenter.sh"] - resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] - verbs: ["get", "list", "watch", "create", "delete", "patch"] - - apiGroups: ["karpenter.k8s.aws"] - resources: ["ec2nodeclasses"] - verbs: ["get", "list", "watch", "create", "delete", "patch"] ---- -# Source: karpenter/templates/clusterrole-core.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: karpenter-core - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -rules: - # Read - - apiGroups: ["karpenter.sh"] - resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csinodes", "volumeattachments"] - verbs: ["get", "watch", "list"] - - apiGroups: ["apps"] - resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] - verbs: ["list", "watch"] - - apiGroups: ["policy"] - resources: ["poddisruptionbudgets"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["get", list, "watch"] - # Write - - apiGroups: ["karpenter.sh"] - resources: ["nodeclaims", "nodeclaims/status"] - verbs: ["create", "delete", "update", "patch"] - - apiGroups: ["karpenter.sh"] - resources: ["nodepools", "nodepools/status"] - verbs: ["update", "patch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["patch", "delete", "update"] - - apiGroups: [""] - resources: ["pods/eviction"] - verbs: ["create"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["delete"] ---- -# Source: karpenter/templates/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: karpenter - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -rules: - # Read - - apiGroups: ["karpenter.k8s.aws"] - resources: ["ec2nodeclasses"] - verbs: ["get", "list", "watch"] - # Write - - apiGroups: ["karpenter.k8s.aws"] - resources: ["ec2nodeclasses", "ec2nodeclasses/status"] - verbs: ["patch", "update"] ---- -# Source: karpenter/templates/clusterrole-core.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: karpenter-core - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: karpenter-core -subjects: - - kind: ServiceAccount - name: karpenter - namespace: default ---- -# Source: karpenter/templates/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: karpenter - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: karpenter -subjects: - - kind: ServiceAccount - name: karpenter - namespace: default ---- -# Source: karpenter/templates/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: karpenter - namespace: default - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -rules: - # Read - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch"] - # Write - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["patch", "update"] - resourceNames: - - "karpenter-leader-election" - # Cannot specify resourceNames on create - # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create"] ---- -# Source: karpenter/templates/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: karpenter-dns - namespace: kube-system - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -rules: - # Read - - apiGroups: [""] - resources: ["services"] - resourceNames: ["kube-dns"] - verbs: ["get"] ---- -# Source: karpenter/templates/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: karpenter - namespace: default - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: karpenter -subjects: - - kind: ServiceAccount - name: karpenter - namespace: default ---- -# Source: karpenter/templates/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: karpenter-dns - namespace: kube-system - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: karpenter-dns -subjects: - - kind: ServiceAccount - name: karpenter - namespace: default ---- -# Source: karpenter/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: karpenter - namespace: default - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -spec: - type: ClusterIP - ports: - - name: http-metrics - port: 8080 - targetPort: http-metrics - protocol: TCP - selector: - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter ---- -# Source: karpenter/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: karpenter - namespace: default - labels: - helm.sh/chart: karpenter-{{ .Values.karpenterVersion }} - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - app.kubernetes.io/version: "{{ .Values.karpenterVersion }}" - app.kubernetes.io/managed-by: Helm -spec: - replicas: 2 - revisionHistoryLimit: 10 - strategy: - rollingUpdate: - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - template: - metadata: - labels: - app.kubernetes.io/name: karpenter - app.kubernetes.io/instance: karpenter - annotations: - spec: - serviceAccountName: karpenter - securityContext: - fsGroup: 65532 - priorityClassName: "system-cluster-critical" - dnsPolicy: ClusterFirst - schedulerName: "default-scheduler" - containers: - - name: controller - securityContext: - runAsUser: 65532 - runAsGroup: 65532 - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - image: public.ecr.aws/karpenter/controller:{{ .Values.karpenterVersion }} - imagePullPolicy: IfNotPresent - env: - - name: KUBERNETES_MIN_VERSION - value: "1.19.0-0" - - name: KARPENTER_SERVICE - value: karpenter - - name: LOG_LEVEL - value: "info" - - name: LOG_OUTPUT_PATHS - value: "stdout" - - name: LOG_ERROR_OUTPUT_PATHS - value: "stderr" - - name: METRICS_PORT - value: "8080" - - name: HEALTH_PROBE_PORT - value: "8081" - - name: SYSTEM_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MEMORY_LIMIT - valueFrom: - resourceFieldRef: - containerName: controller - divisor: "0" - resource: limits.memory - - name: FEATURE_GATES - value: "SpotToSpotConsolidation=false,NodeRepair=false" - - name: BATCH_MAX_DURATION - value: "10s" - - name: BATCH_IDLE_DURATION - value: "1s" - - name: VM_MEMORY_OVERHEAD_PERCENT - value: "0.075" - - name: RESERVED_ENIS - value: "0" - ports: - - name: http-metrics - containerPort: 8080 - protocol: TCP - - name: http - containerPort: 8081 - protocol: TCP - livenessProbe: - initialDelaySeconds: 30 - timeoutSeconds: 30 - httpGet: - path: /healthz - port: http - readinessProbe: - initialDelaySeconds: 5 - timeoutSeconds: 30 - httpGet: - path: /readyz - port: http - resources: - limits: - cpu: 1 - memory: 1Gi - requests: - cpu: 1 - memory: 1Gi - nodeSelector: - kubernetes.io/os: linux - # The template below patches the .Values.affinity to add a default label selector where not specificed - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: karpenter.sh/nodepool - operator: DoesNotExist - - key: eks.amazonaws.com/nodegroup - operator: In - values: - - {{ .Values.nodegroupName }} - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app.kubernetes.io/instance: karpenter - app.kubernetes.io/name: karpenter - topologyKey: kubernetes.io/hostname - # The template below patches the .Values.topologySpreadConstraints to add a default label selector where not specificed - topologySpreadConstraints: - - labelSelector: - matchLabels: - app.kubernetes.io/instance: karpenter - app.kubernetes.io/name: karpenter - maxSkew: 1 - topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: DoNotSchedule - tolerations: - - key: CriticalAddonsOnly - operator: Exists diff --git a/charts/karpenter/templates/karpenter_iam_roles.yaml b/charts/karpenter/templates/karpenter_iam_roles.yaml deleted file mode 100644 index 2878c2c3..00000000 --- a/charts/karpenter/templates/karpenter_iam_roles.yaml +++ /dev/null @@ -1,152 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: create-karpenter-iam-roles -spec: - template: - spec: - containers: - - name: create-iam-roles - image: amazonlinux:2 - command: - - /bin/sh - - -c - - | - # Install AWS CLI - yum update -y - yum install -y aws-cli - - # Now run the rest of your commands - echo '{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] - }' > node-trust-policy.json - - aws iam create-role --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --assume-role-policy-document file://node-trust-policy.json - - aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKSWorkerNodePolicy" - - aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKS_CNI_Policy" - - aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" - - aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonSSMManagedInstanceCore" - - # Create controller trust policy - cat << EOF > controller-trust-policy.json - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:oidc-provider/${OIDC_ENDPOINT#*//}" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "${OIDC_ENDPOINT#*//}:aud": "sts.amazonaws.com", - "${OIDC_ENDPOINT#*//}:sub": "system:serviceaccount:${KARPENTER_NAMESPACE}:karpenter" - } - } - } - ] - } - EOF - - aws iam create-role --role-name "KarpenterControllerRole-${CLUSTER_NAME}" \ - --assume-role-policy-document file://controller-trust-policy.json - - # Create controller policy - cat << EOF > controller-policy.json - { - "Statement": [ - { - "Action": [ - "ssm:GetParameter", - "ec2:DescribeImages", - "ec2:RunInstances", - "ec2:DescribeSubnets", - "ec2:DescribeSecurityGroups", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeInstances", - "ec2:DescribeInstanceTypes", - "ec2:DescribeInstanceTypeOfferings", - "ec2:DeleteLaunchTemplate", - "ec2:CreateTags", - "ec2:CreateLaunchTemplate", - "ec2:CreateFleet", - "ec2:DescribeSpotPriceHistory", - "pricing:GetProducts" - ], - "Effect": "Allow", - "Resource": "*", - "Sid": "Karpenter" - }, - { - "Action": "ec2:TerminateInstances", - "Condition": { - "StringLike": { - "ec2:ResourceTag/karpenter.sh/nodepool": "*" - } - }, - "Effect": "Allow", - "Resource": "*", - "Sid": "ConditionalEC2Termination" - }, - { - "Effect": "Allow", - "Action": "iam:PassRole", - "Resource": "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}", - "Sid": "PassNodeIAMRole" - }, - { - "Effect": "Allow", - "Action": "eks:DescribeCluster", - "Resource": "arn:${AWS_PARTITION}:eks:${AWS_REGION}:${AWS_ACCOUNT_ID}:cluster/${CLUSTER_NAME}", - "Sid": "EKSClusterEndpointLookup" - } - ], - "Version": "2012-10-17" - } - EOF - - aws iam put-role-policy --role-name "KarpenterControllerRole-${CLUSTER_NAME}" \ - --policy-name "KarpenterControllerPolicy-${CLUSTER_NAME}" \ - --policy-document file://controller-policy.json - env: - - name: AWS_REGION - value: "{{ .Values.awsRegion }}" - - name: CLUSTER_NAME - value: "{{ .Values.clusterName }}" - - name: AWS_PARTITION - value: "{{ .Values.awsPartition }}" - - name: AWS_ACCOUNT_ID - value: "{{ .Values.awsAccountId }}" - - name: OIDC_ENDPOINT - value: "{{ .Values.oidcEndpoint }}" - - name: KARPENTER_NAMESPACE - value: "{{ .Values.karpenterNamespace }}" - - name: K8S_VERSION - value: "{{ .Values.k8sVersion }}" - - name: ARM_AMI_ID - value: "{{ .Values.amiIds.arm }}" - - name: AMD_AMI_ID - value: "{{ .Values.amiIds.amd }}" - - name: GPU_AMI_ID - value: "{{ .Values.amiIds.gpu }}" - restartPolicy: Never - diff --git a/charts/karpenter/templates/rbac.yaml b/charts/karpenter/templates/rbac.yaml deleted file mode 100644 index b0ebb769..00000000 --- a/charts/karpenter/templates/rbac.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# rbac.yaml ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: updater-sa - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: update-aws-auth -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "patch", "update"] - resourceNames: ["aws-auth"] # Restrict to only this ConfigMap ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: update-aws-auth-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: update-aws-auth -subjects: - - kind: ServiceAccount - name: updater-sa - namespace: kube-system # Namespace where aws-auth resides - diff --git a/charts/karpenter/templates/tagging.yaml b/charts/karpenter/templates/tagging.yaml deleted file mode 100644 index d5a216b4..00000000 --- a/charts/karpenter/templates/tagging.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: karpenter-tagging-job -spec: - template: - spec: - containers: - - name: karpenter-tagging - image: amazonlinux:2 - imagePullPolicy: IfNotPresent - env: - - name: AWS_REGION - value: "{{ .Values.awsRegion }}" # AWS Region from values.yaml - - name: CLUSTER_NAME - value: "{{ .Values.clusterName }}" # EKS Cluster Name from values.yaml - command: - - "/bin/sh" - - "-c" - - | - #!/bin/bash - set -e - - # Install AWS CLI - yum update -y - yum install -y aws-cli - - # Export environment variables for AWS CLI - export AWS_REGION="${AWS_REGION}" - export CLUSTER_NAME="${CLUSTER_NAME}" - - # Add tags to subnets for all nodegroups - for NODEGROUP in $(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" --region "${AWS_REGION}" --query 'nodegroups' --output text); do - aws ec2 create-tags \ - --region "${AWS_REGION}" \ - --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources $(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" --nodegroup-name "${NODEGROUP}" \ - --region "${AWS_REGION}" --query 'nodegroup.subnets' --output text ) - done - - # Add tags to security groups for the first nodegroup - NODEGROUP=$(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" --region "${AWS_REGION}" --query 'nodegroups[0]' --output text) - - LAUNCH_TEMPLATE=$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" --nodegroup-name "${NODEGROUP}" \ - --region "${AWS_REGION}" --query 'nodegroup.launchTemplate.{id:id,version:version}' --output text | tr -s "\t" ",") - - # If using Cluster security group - SECURITY_GROUPS=$(aws eks describe-cluster --name "${CLUSTER_NAME}" --region "${AWS_REGION}" \ - --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) - - # If using Launch template security groups - SECURITY_GROUPS="$(aws ec2 describe-launch-template-versions --launch-template-id "${LAUNCH_TEMPLATE%,*}" \ - --versions "${LAUNCH_TEMPLATE#*,}" --region "${AWS_REGION}" \ - --query 'LaunchTemplateVersions[0].LaunchTemplateData.[NetworkInterfaces[0].Groups||SecurityGroupIds]' --output text)" - - aws ec2 create-tags \ - --region "${AWS_REGION}" \ - --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources "${SECURITY_GROUPS}" - restartPolicy: Never - backoffLimit: 4 - diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 8f06c529..88a89bc8 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -1,15 +1,23 @@ -# values.yaml -clusterName: "demo-cluster" # Replace with your actual cluster name -karpenterNamespace: "kube-system" # Default namespace for Karpenter -karpenterVersion: "1.1.1" -awsPartition: "aws" # Change to aws-cn or aws-us-gov if applicable -ec2PrivateDNSName: "{{EC2PrivateDNSName}}" -awsRegion: "ap-south-1" # This will be populated from the AWS CLI -oidcEndpoint: "https://oidc.eks.ap-south-1.amazonaws.com/id/89026BA50DBC3476D1DA2DA155EBB574" # This will be populated from the AWS CLI -awsAccountId: "084828584148" # This will be populated from the AWS CLI -k8sVersion: "1.30" # Kubernetes version you're using -amiIds: - arm: "ami-0e3f1069f5a85fc29" # Placeholder for ARM AMI ID - amd: "ami-0308b0e27f09b0b25" # Placeholder for AMD AMI ID - gpu: "ami-0761bb90cfee48660" # Placeholder for GPU AMI ID +# Custom values for your chart +clusterName: test-demo-cluster +awsPartition: aws +awsAccountId: 3849 +karpenterVersion: 1.1.1 + +# Karpenter chart overrides +karpenter: + settings: + clusterName: test-demo-cluster + # interruptionQueue: test-demo-cluster + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::3849:role/KarpenterControllerRole-test-demo-cluster + controller: + resources: + requests: + cpu: "1" + memory: "1Gi" + limits: + cpu: "1" + memory: "1Gi" From 11144c0e1f26d1661ad9f71d7667b19d3f8ffaca Mon Sep 17 00:00:00 2001 From: Hitesh Makol Date: Thu, 26 Dec 2024 16:58:50 +0530 Subject: [PATCH 4/6] Added template folder --- charts/karpenter/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 charts/karpenter/.gitkeep diff --git a/charts/karpenter/.gitkeep b/charts/karpenter/.gitkeep new file mode 100644 index 00000000..e69de29b From 5df5d49b7094270ff32ac69c1302b626598a66d9 Mon Sep 17 00:00:00 2001 From: Hitesh Makol Date: Thu, 26 Dec 2024 17:01:32 +0530 Subject: [PATCH 5/6] Removed extra Files --- charts/karpenter/{ => templates}/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename charts/karpenter/{ => templates}/.gitkeep (100%) diff --git a/charts/karpenter/.gitkeep b/charts/karpenter/templates/.gitkeep similarity index 100% rename from charts/karpenter/.gitkeep rename to charts/karpenter/templates/.gitkeep From 5581497f72f7f6530c76648e6d804bd6d2650a96 Mon Sep 17 00:00:00 2001 From: Hitesh Makol Date: Fri, 27 Dec 2024 13:27:54 +0530 Subject: [PATCH 6/6] Incorporated Review Comments --- charts/karpenter/Chart.yaml | 3 +-- charts/karpenter/values.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml index 7cae808f..3914ec08 100644 --- a/charts/karpenter/Chart.yaml +++ b/charts/karpenter/Chart.yaml @@ -1,8 +1,7 @@ apiVersion: v2 -name: my-karpenter-chart +name: ot-karpenter version: 0.1.0 dependencies: - name: karpenter version: 1.1.1 repository: oci://public.ecr.aws/karpenter - diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 88a89bc8..04915ef8 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -1,13 +1,13 @@ # Custom values for your chart -clusterName: test-demo-cluster -awsPartition: aws +clusterName: "" +awsPartition: "" awsAccountId: 3849 -karpenterVersion: 1.1.1 +karpenterVersion: # Karpenter chart overrides karpenter: settings: - clusterName: test-demo-cluster + clusterName: "" # interruptionQueue: test-demo-cluster serviceAccount: annotations: