fix: ping instead of checking pod readiness

api/common_types.go

@@ -17,6 +17,7 @@ type KubernetesConfig struct {
 	Service                *ServiceConfig                   `json:"service,omitempty"`
 	IgnoreAnnotations      []string                         `json:"ignoreAnnotations,omitempty"`
 	MinReadySeconds        *int32                           `json:"minReadySeconds,omitempty"`
+	PodManagementPolicy    appsv1.PodManagementPolicyType   `json:"podManagementPolicy,omitempty"`
 }
 
 // ServiceConfig define the type of service to be created and its annotations

config/crd/bases/redis.redis.opstreelabs.in_redis.yaml

@@ -1110,6 +1110,10 @@ spec:
                   minReadySeconds:
                     format: int32
                     type: integer
+                  podManagementPolicy:
+                    description: PodManagementPolicyType defines the policy for creating
+                      pods under a stateful set.
+                    type: string
                   redisSecret:
                     description: ExistingPasswordSecret is the struct to access the
                       existing secret
@@ -6139,6 +6143,10 @@ spec:
                   minReadySeconds:
                     format: int32
                     type: integer
+                  podManagementPolicy:
+                    description: PodManagementPolicyType defines the policy for creating
+                      pods under a stateful set.
+                    type: string
                   redisSecret:
                     description: ExistingPasswordSecret is the struct to access the
                       existing secret

config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml

@@ -162,6 +162,10 @@ spec:
                   minReadySeconds:
                     format: int32
                     type: integer
+                  podManagementPolicy:
+                    description: PodManagementPolicyType defines the policy for creating
+                      pods under a stateful set.
+                    type: string
                   redisSecret:
                     description: ExistingPasswordSecret is the struct to access the
                       existing secret
@@ -6755,6 +6759,10 @@ spec:
                   minReadySeconds:
                     format: int32
                     type: integer
+                  podManagementPolicy:
+                    description: PodManagementPolicyType defines the policy for creating
+                      pods under a stateful set.
+                    type: string
                   redisSecret:
                     description: ExistingPasswordSecret is the struct to access the
                       existing secret

config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml

@@ -1112,6 +1112,10 @@ spec:
                   minReadySeconds:
                     format: int32
                     type: integer
+                  podManagementPolicy:
+                    description: PodManagementPolicyType defines the policy for creating
+                      pods under a stateful set.
+                    type: string
                   redisSecret:
                     description: ExistingPasswordSecret is the struct to access the
                       existing secret
@@ -6147,6 +6151,10 @@ spec:
                   minReadySeconds:
                     format: int32
                     type: integer
+                  podManagementPolicy:
+                    description: PodManagementPolicyType defines the policy for creating
+                      pods under a stateful set.
+                    type: string
                   redisSecret:
                     description: ExistingPasswordSecret is the struct to access the
                       existing secret

config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml

@@ -1113,6 +1113,10 @@ spec:
                   minReadySeconds:
                     format: int32
                     type: integer
+                  podManagementPolicy:
+                    description: PodManagementPolicyType defines the policy for creating
+                      pods under a stateful set.
+                    type: string
                   redisSecret:
                     description: ExistingPasswordSecret is the struct to access the
                       existing secret
@@ -3601,6 +3605,10 @@ spec:
                   minReadySeconds:
                     format: int32
                     type: integer
+                  podManagementPolicy:
+                    description: PodManagementPolicyType defines the policy for creating
+                      pods under a stateful set.
+                    type: string
                   redisSecret:
                     description: ExistingPasswordSecret is the struct to access the
                       existing secret

controllers/rediscluster_controller.go

@@ -121,7 +121,7 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return intctrlutil.RequeueWithError(err, reqLogger, "")
 	}
 
-	if r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") {
+	if r.IsStatefulSetReady(ctx, r.K8sClient, instance, instance.Namespace, instance.Name+"-leader") {
 		// Mark the cluster status as initializing if there are no follower nodes
 		if (instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0) ||
 			instance.Status.ReadyFollowerReplicas != followerReplicas {
@@ -147,8 +147,8 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		}
 	}
 
-	if !(r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") && r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-follower")) {
-		return intctrlutil.Reconciled()
+	if !(r.IsStatefulSetReady(ctx, r.K8sClient, instance, instance.Namespace, instance.Name+"-leader") && r.IsStatefulSetReady(ctx, r.K8sClient, instance, instance.Namespace, instance.Name+"-follower")) {
+		return intctrlutil.RequeueAfter(reqLogger, time.Second*10, "Redis cluster is not ready")
 	}
 
 	// Mark the cluster status as bootstrapping if all the leader and follower nodes are ready

controllers/redisreplication_controller.go

@@ -57,8 +57,8 @@ func (r *RedisReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	if err != nil {
 		return intctrlutil.RequeueWithError(err, reqLogger, "")
 	}
-	if !r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name) {
-		return intctrlutil.Reconciled()
+	if !r.IsStatefulSetReady(ctx, r.K8sClient, instance, instance.Namespace, instance.Name) {
+		return intctrlutil.RequeueAfter(reqLogger, time.Second*10, "")
 	}
 
 	var realMaster string

docs/content/en/docs/CRD Reference/Redis API/_index.md

@@ -56,6 +56,7 @@ _Appears in:_
 | `redisSecret` _[ExistingPasswordSecret](#existingpasswordsecret)_ |  |
 | `imagePullSecrets` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ |  |
 | `updateStrategy` _[StatefulSetUpdateStrategy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#statefulsetupdatestrategy-v1-apps)_ |  |
+| `podManagementPolicy` _[PodManagementPolicyType](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies)_ |  |
 
 #### VolumeMount
 

k8sutils/cluster-scaling.go

@@ -9,6 +9,7 @@
 	redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
 	"github.com/go-logr/logr"
 	redis "github.com/redis/go-redis/v9"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -106,6 +107,31 @@
 	return strconv.Itoa(totalSlots)
 }
 
+// pingRedisNode will ping the redis node to check if it is up and running
+func pingRedisNode(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr runtime.Object, pod RedisDetails) bool {
+	var redisClient *redis.Client
+
+	switch cr := cr.(type) {
+	case *redisv1beta2.RedisCluster:
+		redisClient = configureRedisClient(client, logger, cr, pod.PodName)
+	case *redisv1beta2.RedisReplication:
+		redisClient = configureRedisReplicationClient(client, logger, cr, pod.PodName)
+	default:
+		logger.Error(nil, "Unknown CR type")
+		return false
+	}
+
+	defer redisClient.Close()
+
+	pong, err := redisClient.Ping(ctx).Result()
+	if err != nil || pong != "PONG" {
+		logger.Error(err, "Failed to ping Redis server")
+		return false
+	}
+
+	return true
+}
+
 // getRedisNodeID would return nodeID of a redis node by passing pod
 func getRedisNodeID(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisCluster, pod RedisDetails) string {
 	redisClient := configureRedisClient(client, logger, cr, pod.PodName)

k8sutils/redis-cluster.go

@@ -52,6 +52,7 @@ func generateRedisClusterParams(cr *redisv1beta2.RedisCluster, replicas int32, e
 		IgnoreAnnotations:             cr.Spec.KubernetesConfig.IgnoreAnnotations,
 		HostNetwork:                   cr.Spec.HostNetwork,
 		MinReadySeconds:               minreadyseconds,
+		PodManagementPolicy:           cr.Spec.KubernetesConfig.PodManagementPolicy,
 	}
 	if cr.Spec.RedisExporter != nil {
 		res.EnableMetrics = cr.Spec.RedisExporter.Enabled

k8sutils/redis-replication.go

@@ -99,6 +99,7 @@ func generateRedisReplicationParams(cr *redisv1beta2.RedisReplication) statefulS
 		UpdateStrategy:                cr.Spec.KubernetesConfig.UpdateStrategy,
 		IgnoreAnnotations:             cr.Spec.KubernetesConfig.IgnoreAnnotations,
 		MinReadySeconds:               minreadyseconds,
+		PodManagementPolicy:           cr.Spec.KubernetesConfig.PodManagementPolicy,
 	}
 	if cr.Spec.KubernetesConfig.ImagePullSecrets != nil {
 		res.ImagePullSecrets = cr.Spec.KubernetesConfig.ImagePullSecrets

k8sutils/redis-sentinel.go

@@ -104,6 +104,7 @@ func generateRedisSentinelParams(cr *redisv1beta2.RedisSentinel, replicas int32,
 		UpdateStrategy:                cr.Spec.KubernetesConfig.UpdateStrategy,
 		IgnoreAnnotations:             cr.Spec.KubernetesConfig.IgnoreAnnotations,
 		MinReadySeconds:               minreadyseconds,
+		PodManagementPolicy:           cr.Spec.KubernetesConfig.PodManagementPolicy,
 	}
 
 	if cr.Spec.KubernetesConfig.ImagePullSecrets != nil {

k8sutils/redis-standalone.go

@@ -94,6 +94,7 @@ func generateRedisStandaloneParams(cr *redisv1beta2.Redis) statefulSetParameters
 		UpdateStrategy:                cr.Spec.KubernetesConfig.UpdateStrategy,
 		IgnoreAnnotations:             cr.Spec.KubernetesConfig.IgnoreAnnotations,
 		MinReadySeconds:               minreadyseconds,
+		PodManagementPolicy:           cr.Spec.KubernetesConfig.PodManagementPolicy,
 	}
 	if cr.Spec.KubernetesConfig.ImagePullSecrets != nil {
 		res.ImagePullSecrets = cr.Spec.KubernetesConfig.ImagePullSecrets

k8sutils/services.go

@@ -55,6 +55,7 @@ func generateServiceDef(serviceMeta metav1.ObjectMeta, epp exporterPortProvider,
 	}
 	if headless {
 		service.Spec.ClusterIP = "None"
+		service.Spec.PublishNotReadyAddresses = true
 	}
 	if exporterPort, ok := epp(); ok {
 		redisExporterService := enableMetricsPort(exporterPort)

k8sutils/services_test.go

@@ -102,9 +102,10 @@ func TestGenerateServiceDef(t *testing.T) {
 							Protocol:   corev1.ProtocolTCP,
 						},
 					},
-					Selector:  map[string]string{"role": "sentinel"},
-					ClusterIP: "None",
-					Type:      corev1.ServiceTypeClusterIP,
+					Selector:                 map[string]string{"role": "sentinel"},
+					ClusterIP:                "None",
+					Type:                     corev1.ServiceTypeClusterIP,
+					PublishNotReadyAddresses: true,
 				},
 			},
 		},
@@ -184,9 +185,10 @@ func TestGenerateServiceDef(t *testing.T) {
 							Protocol:   corev1.ProtocolTCP,
 						},
 					},
-					Selector:  map[string]string{"role": "redis"},
-					ClusterIP: "None",
-					Type:      corev1.ServiceTypeClusterIP,
+					Selector:                 map[string]string{"role": "redis"},
+					ClusterIP:                "None",
+					Type:                     corev1.ServiceTypeClusterIP,
+					PublishNotReadyAddresses: true,
 				},
 			},
 		},

k8sutils/statefulset.go

@@ -19,13 +19,14 @@
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/utils/env"
 	"k8s.io/utils/ptr"
 )
 
 type StatefulSet interface {
-	IsStatefulSetReady(ctx context.Context, namespace, name string) bool
+	IsStatefulSetReady(ctx context.Context, client kubernetes.Interface, cr runtime.Object, namespace, name string) bool
 }
 
 type StatefulSetService struct {
@@ -41,7 +42,7 @@
 	}
 }
 
-func (s *StatefulSetService) IsStatefulSetReady(ctx context.Context, namespace, name string) bool {
+func (s *StatefulSetService) IsStatefulSetReady(ctx context.Context, client kubernetes.Interface, cr runtime.Object, namespace, name string) bool {
 	var (
 		partition = 0
 		replicas  = 1
@@ -74,10 +75,19 @@
 		logger.V(1).Info("StatefulSet is not ready", "Status.ObservedGeneration", sts.Status.ObservedGeneration, "ObjectMeta.Generation", sts.ObjectMeta.Generation)
 		return false
 	}
-	if int(sts.Status.ReadyReplicas) != replicas {
-		logger.V(1).Info("StatefulSet is not ready", "Status.ReadyReplicas", sts.Status.ReadyReplicas, "Replicas", replicas)
-		return false
+
+	for i := 0; i < replicas; i++ {
+		pod := RedisDetails{
+			PodName:   fmt.Sprintf("%s-%d", name, i),
+			Namespace: namespace,
+		}
+
+		if !pingRedisNode(ctx, client, logger, cr, pod) {
+			logger.V(1).Info("StatefulSet is not ready", "PingRedisNode", pod.PodName)
+			return false
+		}
 	}
+
 	return true
 }
 
@@ -108,6 +118,7 @@
 	IgnoreAnnotations             []string
 	HostNetwork                   bool
 	MinReadySeconds               int32
+	PodManagementPolicy           appsv1.PodManagementPolicyType
 }
 
 // containerParameters will define container input params
@@ -280,11 +291,12 @@
 		TypeMeta:   generateMetaInformation("StatefulSet", "apps/v1"),
 		ObjectMeta: stsMeta,
 		Spec: appsv1.StatefulSetSpec{
-			Selector:        LabelSelectors(stsMeta.GetLabels()),
-			ServiceName:     fmt.Sprintf("%s-headless", stsMeta.Name),
-			Replicas:        params.Replicas,
-			UpdateStrategy:  params.UpdateStrategy,
-			MinReadySeconds: params.MinReadySeconds,
+			Selector:            LabelSelectors(stsMeta.GetLabels()),
+			ServiceName:         fmt.Sprintf("%s-headless", stsMeta.Name),
+			Replicas:            params.Replicas,
+			UpdateStrategy:      params.UpdateStrategy,
+			MinReadySeconds:     params.MinReadySeconds,
+			PodManagementPolicy: params.PodManagementPolicy,
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      stsMeta.GetLabels(),