feat: redis-cluster add podAntiAffinity(#1174 ) (#1180)

* feat: redis-cluster add podAntiAffinity Signed-off-by: xiaozhuang <[email protected]> * chore: lint Signed-off-by: xiaozhuang <[email protected]> * chore: webhook add annotation Signed-off-by: xiaozhuang <[email protected]> * chore: lint Signed-off-by: xiaozhuang <[email protected]> * chore: lint Signed-off-by: xiaozhuang <[email protected]> * chore: redis-operator chart add MutatingWebhookConfiguration Signed-off-by: xiaozhuang <[email protected]> * chore: revert kustomization for webhook Signed-off-by: xiaozhuang <[email protected]> * chore: podAntiAffinity control by annotation Signed-off-by: xiaozhuang <[email protected]> * chore: webhook add testing Signed-off-by: xiaozhuang <[email protected]> * chore: lint Signed-off-by: xiaozhuang <[email protected]> * chore: lint Signed-off-by: xiaozhuang <[email protected]> * chore: MutatingWebhookConfiguration remove namespace filter Signed-off-by: xiaozhuang <[email protected]> * change key Signed-off-by: drivebyer <[email protected]> * add example Signed-off-by: drivebyer <[email protected]> --------- Signed-off-by: xiaozhuang <[email protected]> Signed-off-by: drivebyer <[email protected]> Co-authored-by: xiaozhuang <[email protected]> Co-authored-by: drivebyer <[email protected]>
OT-CONTAINER-KIT · Dec 27, 2024 · bf1c79f · bf1c79f
1 parent c02c7c9
commit bf1c79f
Show file tree

Hide file tree

Showing 7 changed files with 386 additions and 0 deletions.
diff --git a/PROJECT b/PROJECT
@@ -88,4 +88,11 @@ resources:
   kind: RedisSentinel
   path: redis-operator/api/v1beta1
   version: v1beta1
+- group: core
+  kind: Pod
+  path: k8s.io/api/core/v1
+  version: v1
+  webhooks:
+    defaulting: true
+    webhookVersion: v1
 version: "3"
diff --git a/charts/redis-operator/templates/mutating-webhook-configuration.yaml b/charts/redis-operator/templates/mutating-webhook-configuration.yaml
@@ -0,0 +1,34 @@
+{{ if .Values.redisOperator.webhook }}
+
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/serving-cert
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: {{ .Release.Namespace }}
+      path: /mutate-core-v1-pod
+  failurePolicy: Fail
+  name: ot-mutate-pod.opstree.com
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    resources:
+    - pods
+  sideEffects: None
+  objectSelector:
+    matchExpressions:
+      - key: redis_setup_type
+        operator: Exists
+
+{{ end }}
diff --git a/charts/redis-operator/values.yaml b/charts/redis-operator/values.yaml
@@ -17,6 +17,7 @@ redisOperator:
   # When not specified, the operator will watch all namespaces. It can be set to a specific namespace or multiple namespaces separated by commas.
   watchNamespace: ""
   env: []
+  # If you want to enable masterSlaveAntiAffinity, you need to set webhook to true.
   webhook: false
   automountServiceAccountToken: true
 

diff --git a/example/v1beta2/redis-cluster-deploy/role-anti-affinity.yaml b/example/v1beta2/redis-cluster-deploy/role-anti-affinity.yaml
@@ -0,0 +1,55 @@
+---
+# Redis Cluster Custom Resource Definition
+# This configuration file defines a Redis Cluster setup with anti-affinity rules
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisCluster
+metadata:
+  name: redis-cluster
+  annotations:
+    # Enable pod anti-affinity between leader and follower pods
+    # This ensures leader and follower pods are scheduled on different nodes
+    # for better high availability
+    # PS: you should enable operator webhook for this to work
+    redisclusters.redis.redis.opstreelabs.in/role-anti-affinity: "true"
+spec:
+  clusterSize: 3
+  clusterVersion: v7
+  persistenceEnabled: true
+  podSecurityContext:
+    runAsUser: 1000
+    fsGroup: 1000
+  kubernetesConfig:
+    image: quay.io/opstree/redis:v7.0.12
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 101m
+        memory: 128Mi
+      limits:
+        cpu: 101m
+        memory: 128Mi
+  redisExporter:
+    enabled: false
+    image: quay.io/opstree/redis-exporter:v1.44.0
+    imagePullPolicy: Always
+    resources:
+      requests:
+        cpu: 100m
+        memory: 128Mi
+      limits:
+        cpu: 100m
+        memory: 128Mi
+  storage:
+    volumeClaimTemplate:
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi
+    nodeConfVolume: false
+    nodeConfVolumeClaimTemplate:
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 1Gi
diff --git a/main.go b/main.go
@@ -29,6 +29,7 @@ import (
 	"github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllers/redissentinel"
 	intctrlutil "github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllerutil"
 	"github.com/OT-CONTAINER-KIT/redis-operator/pkg/k8sutils"
+	coreWebhook "github.com/OT-CONTAINER-KIT/redis-operator/pkg/webhook"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -39,6 +40,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 )
 
 var (
@@ -172,6 +174,11 @@ func main() {
 			setupLog.Error(err, "unable to create webhook", "webhook", "RedisSentinel")
 			os.Exit(1)
 		}
+
+		wblog := ctrl.Log.WithName("webhook").WithName("PodAffiniytMutate")
+		mgr.GetWebhookServer().Register("/mutate-core-v1-pod", &webhook.Admission{
+			Handler: coreWebhook.NewPodAffiniytMutate(mgr.GetClient(), admission.NewDecoder(scheme), wblog),
+		})
 	}
 	// +kubebuilder:scaffold:builder
 

diff --git a/pkg/webhook/pod_webhook.go b/pkg/webhook/pod_webhook.go
@@ -0,0 +1,169 @@
+/*
+Copyright 2020 Opstree Solutions.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package webhook
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"reflect"
+	"strings"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+//+kubebuilder:webhook:path=/mutate-core-v1-pod,mutating=true,failurePolicy=fail,sideEffects=None,groups=core,resources=pods,verbs=create,versions=v1,name=mpod.kb.io,admissionReviewVersions=v1
+
+// PodAntiAffiniytMutate mutate Pods
+type PodAntiAffiniytMutate struct {
+	Client  client.Client
+	decoder *admission.Decoder
+	logger  logr.Logger
+}
+
+func NewPodAffiniytMutate(c client.Client, d *admission.Decoder, log logr.Logger) admission.Handler {
+	return &PodAntiAffiniytMutate{
+		Client:  c,
+		decoder: d,
+		logger:  log,
+	}
+}
+
+const (
+	podAnnotationsRedisClusterApp = "redis.opstreelabs.instance"
+	podLabelsPodName              = "statefulset.kubernetes.io/pod-name"
+	podLabelsRedisType            = "redis_setup_type"
+)
+
+const annotationKeyEnablePodAntiAffinity = "redisclusters.redis.redis.opstreelabs.in/role-anti-affinity"
+
+func (v *PodAntiAffiniytMutate) Handle(ctx context.Context, req admission.Request) admission.Response {
+	logger := v.logger.WithValues("Request.Namespace", req.Namespace, "Request.Name", req.Name)
+
+	pod := &corev1.Pod{}
+	err := v.decoder.Decode(req, pod)
+	if err != nil {
+		return admission.Errored(http.StatusBadRequest, err)
+	}
+
+	// only mutate pods that belong to redis cluster
+	if !v.isRedisClusterPod(pod) {
+		return admission.Allowed("")
+	}
+	// check if the pod anti-affinity is enabled
+	annotations := pod.GetAnnotations()
+	if annotations == nil {
+		return admission.Allowed("")
+	}
+	if enable, ok := annotations[annotationKeyEnablePodAntiAffinity]; !ok || enable != "true" {
+		logger.V(1).Info("pod anti-affinity is not enabled")
+		return admission.Allowed("")
+	}
+
+	old := pod.DeepCopy()
+
+	v.AddPodAntiAffinity(pod)
+	if !reflect.DeepEqual(old, pod) {
+		marshaledPod, err := json.Marshal(pod)
+		if err != nil {
+			return admission.Errored(http.StatusInternalServerError, err)
+		}
+
+		logger.Info("mutate pod with anti-affinity")
+		return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod)
+	}
+
+	return admission.Allowed("")
+}
+
+// PodAntiAffiniytMutate implements admission.DecoderInjector.
+// A decoder will be automatically injected.
+
+// InjectDecoder injects the decoder.
+func (v *PodAntiAffiniytMutate) InjectDecoder(d *admission.Decoder) error {
+	v.decoder = d
+	return nil
+}
+
+func (m *PodAntiAffiniytMutate) InjectLogger(l logr.Logger) error {
+	m.logger = l
+	return nil
+}
+
+func (v *PodAntiAffiniytMutate) AddPodAntiAffinity(pod *corev1.Pod) {
+	podName := pod.ObjectMeta.Name
+	antiLabelValue := v.getAntiAffinityValue(podName)
+
+	if pod.Spec.Affinity == nil {
+		pod.Spec.Affinity = &corev1.Affinity{}
+	}
+	if pod.Spec.Affinity.PodAntiAffinity == nil {
+		pod.Spec.Affinity.PodAntiAffinity = &corev1.PodAntiAffinity{}
+	}
+	if pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
+		pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = make([]corev1.PodAffinityTerm, 0)
+	}
+	addAntiAffinity := corev1.PodAffinityTerm{
+		LabelSelector: &metav1.LabelSelector{
+			MatchExpressions: []metav1.LabelSelectorRequirement{
+				{
+					Key:      podLabelsPodName,
+					Operator: metav1.LabelSelectorOpIn,
+					Values:   []string{antiLabelValue},
+				},
+			},
+		},
+		TopologyKey: "kubernetes.io/hostname",
+	}
+
+	pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, addAntiAffinity)
+}
+
+func (v *PodAntiAffiniytMutate) getPodAnnotations(pod *corev1.Pod) map[string]string {
+	if pod.Annotations == nil {
+		pod.Annotations = make(map[string]string)
+	}
+	return pod.Annotations
+}
+
+func (v *PodAntiAffiniytMutate) isRedisClusterPod(pod *corev1.Pod) bool {
+	annotations := v.getPodAnnotations(pod)
+	if _, ok := annotations[podAnnotationsRedisClusterApp]; !ok {
+		return false
+	}
+
+	labels := pod.GetLabels()
+	if _, ok := labels[podLabelsRedisType]; !ok {
+		return false
+	}
+
+	return true
+}
+
+func (v *PodAntiAffiniytMutate) getAntiAffinityValue(podName string) string {
+	if strings.Contains(podName, "follower") {
+		return strings.Replace(podName, "follower", "leader", -1)
+	}
+	if strings.Contains(podName, "leader") {
+		return strings.Replace(podName, "leader", "follower", -1)
+	}
+	return ""
+}