diff --git a/api/common_types.go b/api/common_types.go index c0212cf31..5e9ae247a 100644 --- a/api/common_types.go +++ b/api/common_types.go @@ -17,6 +17,7 @@ type KubernetesConfig struct { Service *ServiceConfig `json:"service,omitempty"` IgnoreAnnotations []string `json:"ignoreAnnotations,omitempty"` MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` + PodManagementPolicy appsv1.PodManagementPolicyType `json:"podManagementPolicy,omitempty"` } // ServiceConfig define the type of service to be created and its annotations diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml index 3ad451862..b86ea48d3 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml @@ -1110,6 +1110,10 @@ spec: minReadySeconds: format: int32 type: integer + podManagementPolicy: + description: PodManagementPolicyType defines the policy for creating + pods under a stateful set. + type: string redisSecret: description: ExistingPasswordSecret is the struct to access the existing secret @@ -6139,6 +6143,10 @@ spec: minReadySeconds: format: int32 type: integer + podManagementPolicy: + description: PodManagementPolicyType defines the policy for creating + pods under a stateful set. + type: string redisSecret: description: ExistingPasswordSecret is the struct to access the existing secret diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml index a5a596988..1244f4869 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml @@ -162,6 +162,10 @@ spec: minReadySeconds: format: int32 type: integer + podManagementPolicy: + description: PodManagementPolicyType defines the policy for creating + pods under a stateful set. + type: string redisSecret: description: ExistingPasswordSecret is the struct to access the existing secret @@ -6755,6 +6759,10 @@ spec: minReadySeconds: format: int32 type: integer + podManagementPolicy: + description: PodManagementPolicyType defines the policy for creating + pods under a stateful set. + type: string redisSecret: description: ExistingPasswordSecret is the struct to access the existing secret diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml index 8cf6c32eb..35de896c1 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml @@ -1112,6 +1112,10 @@ spec: minReadySeconds: format: int32 type: integer + podManagementPolicy: + description: PodManagementPolicyType defines the policy for creating + pods under a stateful set. + type: string redisSecret: description: ExistingPasswordSecret is the struct to access the existing secret @@ -6147,6 +6151,10 @@ spec: minReadySeconds: format: int32 type: integer + podManagementPolicy: + description: PodManagementPolicyType defines the policy for creating + pods under a stateful set. + type: string redisSecret: description: ExistingPasswordSecret is the struct to access the existing secret diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml index 4599c070b..10ca7ec32 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml @@ -1113,6 +1113,10 @@ spec: minReadySeconds: format: int32 type: integer + podManagementPolicy: + description: PodManagementPolicyType defines the policy for creating + pods under a stateful set. + type: string redisSecret: description: ExistingPasswordSecret is the struct to access the existing secret @@ -3601,6 +3605,10 @@ spec: minReadySeconds: format: int32 type: integer + podManagementPolicy: + description: PodManagementPolicyType defines the policy for creating + pods under a stateful set. + type: string redisSecret: description: ExistingPasswordSecret is the struct to access the existing secret diff --git a/controllers/rediscluster_controller.go b/controllers/rediscluster_controller.go index 969d44291..2e4e6b489 100644 --- a/controllers/rediscluster_controller.go +++ b/controllers/rediscluster_controller.go @@ -121,7 +121,7 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return intctrlutil.RequeueWithError(err, reqLogger, "") } - if r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") { + if r.IsStatefulSetReady(ctx, r.K8sClient, instance, instance.Namespace, instance.Name+"-leader") { // Mark the cluster status as initializing if there are no follower nodes if (instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0) || instance.Status.ReadyFollowerReplicas != followerReplicas { @@ -147,8 +147,8 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } } - if !(r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") && r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-follower")) { - return intctrlutil.Reconciled() + if !(r.IsStatefulSetReady(ctx, r.K8sClient, instance, instance.Namespace, instance.Name+"-leader") && r.IsStatefulSetReady(ctx, r.K8sClient, instance, instance.Namespace, instance.Name+"-follower")) { + return intctrlutil.RequeueAfter(reqLogger, time.Second*10, "Redis cluster is not ready") } // Mark the cluster status as bootstrapping if all the leader and follower nodes are ready diff --git a/controllers/redisreplication_controller.go b/controllers/redisreplication_controller.go index 335d0606e..76e9d020b 100644 --- a/controllers/redisreplication_controller.go +++ b/controllers/redisreplication_controller.go @@ -57,8 +57,8 @@ func (r *RedisReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Req if err != nil { return intctrlutil.RequeueWithError(err, reqLogger, "") } - if !r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name) { - return intctrlutil.Reconciled() + if !r.IsStatefulSetReady(ctx, r.K8sClient, instance, instance.Namespace, instance.Name) { + return intctrlutil.RequeueAfter(reqLogger, time.Second*10, "") } var realMaster string diff --git a/docs/content/en/docs/CRD Reference/Redis API/_index.md b/docs/content/en/docs/CRD Reference/Redis API/_index.md index f0331d16c..46295d935 100644 --- a/docs/content/en/docs/CRD Reference/Redis API/_index.md +++ b/docs/content/en/docs/CRD Reference/Redis API/_index.md @@ -56,6 +56,7 @@ _Appears in:_ | `redisSecret` _[ExistingPasswordSecret](#existingpasswordsecret)_ | | | `imagePullSecrets` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | | | `updateStrategy` _[StatefulSetUpdateStrategy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#statefulsetupdatestrategy-v1-apps)_ | | +| `podManagementPolicy` _[PodManagementPolicyType](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies)_ | | #### VolumeMount diff --git a/k8sutils/cluster-scaling.go b/k8sutils/cluster-scaling.go index 4b6edec9e..46e4338b1 100644 --- a/k8sutils/cluster-scaling.go +++ b/k8sutils/cluster-scaling.go @@ -9,6 +9,7 @@ import ( redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" "github.com/go-logr/logr" redis "github.com/redis/go-redis/v9" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" ) @@ -106,6 +107,31 @@ func getRedisClusterSlots(ctx context.Context, redisClient *redis.Client, logger return strconv.Itoa(totalSlots) } +// pingRedisNode will ping the redis node to check if it is up and running +func pingRedisNode(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr runtime.Object, pod RedisDetails) bool { + var redisClient *redis.Client + + switch cr := cr.(type) { + case *redisv1beta2.RedisCluster: + redisClient = configureRedisClient(client, logger, cr, pod.PodName) + case *redisv1beta2.RedisReplication: + redisClient = configureRedisReplicationClient(client, logger, cr, pod.PodName) + default: + logger.Error(nil, "Unknown CR type") + return false + } + + defer redisClient.Close() + + pong, err := redisClient.Ping(ctx).Result() + if err != nil || pong != "PONG" { + logger.Error(err, "Failed to ping Redis server") + return false + } + + return true +} + // getRedisNodeID would return nodeID of a redis node by passing pod func getRedisNodeID(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisCluster, pod RedisDetails) string { redisClient := configureRedisClient(client, logger, cr, pod.PodName) diff --git a/k8sutils/redis-cluster.go b/k8sutils/redis-cluster.go index 9cc63ecd3..399a50bcd 100644 --- a/k8sutils/redis-cluster.go +++ b/k8sutils/redis-cluster.go @@ -52,6 +52,7 @@ func generateRedisClusterParams(cr *redisv1beta2.RedisCluster, replicas int32, e IgnoreAnnotations: cr.Spec.KubernetesConfig.IgnoreAnnotations, HostNetwork: cr.Spec.HostNetwork, MinReadySeconds: minreadyseconds, + PodManagementPolicy: cr.Spec.KubernetesConfig.PodManagementPolicy, } if cr.Spec.RedisExporter != nil { res.EnableMetrics = cr.Spec.RedisExporter.Enabled diff --git a/k8sutils/redis-replication.go b/k8sutils/redis-replication.go index 66c066492..847603da9 100644 --- a/k8sutils/redis-replication.go +++ b/k8sutils/redis-replication.go @@ -99,6 +99,7 @@ func generateRedisReplicationParams(cr *redisv1beta2.RedisReplication) statefulS UpdateStrategy: cr.Spec.KubernetesConfig.UpdateStrategy, IgnoreAnnotations: cr.Spec.KubernetesConfig.IgnoreAnnotations, MinReadySeconds: minreadyseconds, + PodManagementPolicy: cr.Spec.KubernetesConfig.PodManagementPolicy, } if cr.Spec.KubernetesConfig.ImagePullSecrets != nil { res.ImagePullSecrets = cr.Spec.KubernetesConfig.ImagePullSecrets diff --git a/k8sutils/redis-sentinel.go b/k8sutils/redis-sentinel.go index 873b05e1c..83a557690 100644 --- a/k8sutils/redis-sentinel.go +++ b/k8sutils/redis-sentinel.go @@ -104,6 +104,7 @@ func generateRedisSentinelParams(cr *redisv1beta2.RedisSentinel, replicas int32, UpdateStrategy: cr.Spec.KubernetesConfig.UpdateStrategy, IgnoreAnnotations: cr.Spec.KubernetesConfig.IgnoreAnnotations, MinReadySeconds: minreadyseconds, + PodManagementPolicy: cr.Spec.KubernetesConfig.PodManagementPolicy, } if cr.Spec.KubernetesConfig.ImagePullSecrets != nil { diff --git a/k8sutils/redis-standalone.go b/k8sutils/redis-standalone.go index 4fed7d135..934a57f68 100644 --- a/k8sutils/redis-standalone.go +++ b/k8sutils/redis-standalone.go @@ -94,6 +94,7 @@ func generateRedisStandaloneParams(cr *redisv1beta2.Redis) statefulSetParameters UpdateStrategy: cr.Spec.KubernetesConfig.UpdateStrategy, IgnoreAnnotations: cr.Spec.KubernetesConfig.IgnoreAnnotations, MinReadySeconds: minreadyseconds, + PodManagementPolicy: cr.Spec.KubernetesConfig.PodManagementPolicy, } if cr.Spec.KubernetesConfig.ImagePullSecrets != nil { res.ImagePullSecrets = cr.Spec.KubernetesConfig.ImagePullSecrets diff --git a/k8sutils/services.go b/k8sutils/services.go index 627117277..24b7f74e6 100644 --- a/k8sutils/services.go +++ b/k8sutils/services.go @@ -55,6 +55,7 @@ func generateServiceDef(serviceMeta metav1.ObjectMeta, epp exporterPortProvider, } if headless { service.Spec.ClusterIP = "None" + service.Spec.PublishNotReadyAddresses = true } if exporterPort, ok := epp(); ok { redisExporterService := enableMetricsPort(exporterPort) diff --git a/k8sutils/services_test.go b/k8sutils/services_test.go index 5906d61db..e7503f691 100644 --- a/k8sutils/services_test.go +++ b/k8sutils/services_test.go @@ -102,9 +102,10 @@ func TestGenerateServiceDef(t *testing.T) { Protocol: corev1.ProtocolTCP, }, }, - Selector: map[string]string{"role": "sentinel"}, - ClusterIP: "None", - Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{"role": "sentinel"}, + ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, + PublishNotReadyAddresses: true, }, }, }, @@ -184,9 +185,10 @@ func TestGenerateServiceDef(t *testing.T) { Protocol: corev1.ProtocolTCP, }, }, - Selector: map[string]string{"role": "redis"}, - ClusterIP: "None", - Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{"role": "redis"}, + ClusterIP: "None", + Type: corev1.ServiceTypeClusterIP, + PublishNotReadyAddresses: true, }, }, }, diff --git a/k8sutils/statefulset.go b/k8sutils/statefulset.go index bc31d4975..e28cc303f 100644 --- a/k8sutils/statefulset.go +++ b/k8sutils/statefulset.go @@ -19,13 +19,14 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/utils/env" "k8s.io/utils/ptr" ) type StatefulSet interface { - IsStatefulSetReady(ctx context.Context, namespace, name string) bool + IsStatefulSetReady(ctx context.Context, client kubernetes.Interface, cr runtime.Object, namespace, name string) bool } type StatefulSetService struct { @@ -41,7 +42,7 @@ func NewStatefulSetService(kubeClient kubernetes.Interface, log logr.Logger) *St } } -func (s *StatefulSetService) IsStatefulSetReady(ctx context.Context, namespace, name string) bool { +func (s *StatefulSetService) IsStatefulSetReady(ctx context.Context, client kubernetes.Interface, cr runtime.Object, namespace, name string) bool { var ( partition = 0 replicas = 1 @@ -74,10 +75,19 @@ func (s *StatefulSetService) IsStatefulSetReady(ctx context.Context, namespace, logger.V(1).Info("StatefulSet is not ready", "Status.ObservedGeneration", sts.Status.ObservedGeneration, "ObjectMeta.Generation", sts.ObjectMeta.Generation) return false } - if int(sts.Status.ReadyReplicas) != replicas { - logger.V(1).Info("StatefulSet is not ready", "Status.ReadyReplicas", sts.Status.ReadyReplicas, "Replicas", replicas) - return false + + for i := 0; i < replicas; i++ { + pod := RedisDetails{ + PodName: fmt.Sprintf("%s-%d", name, i), + Namespace: namespace, + } + + if !pingRedisNode(ctx, client, logger, cr, pod) { + logger.V(1).Info("StatefulSet is not ready", "PingRedisNode", pod.PodName) + return false + } } + return true } @@ -108,6 +118,7 @@ type statefulSetParameters struct { IgnoreAnnotations []string HostNetwork bool MinReadySeconds int32 + PodManagementPolicy appsv1.PodManagementPolicyType } // containerParameters will define container input params @@ -280,11 +291,12 @@ func generateStatefulSetsDef(stsMeta metav1.ObjectMeta, params statefulSetParame TypeMeta: generateMetaInformation("StatefulSet", "apps/v1"), ObjectMeta: stsMeta, Spec: appsv1.StatefulSetSpec{ - Selector: LabelSelectors(stsMeta.GetLabels()), - ServiceName: fmt.Sprintf("%s-headless", stsMeta.Name), - Replicas: params.Replicas, - UpdateStrategy: params.UpdateStrategy, - MinReadySeconds: params.MinReadySeconds, + Selector: LabelSelectors(stsMeta.GetLabels()), + ServiceName: fmt.Sprintf("%s-headless", stsMeta.Name), + Replicas: params.Replicas, + UpdateStrategy: params.UpdateStrategy, + MinReadySeconds: params.MinReadySeconds, + PodManagementPolicy: params.PodManagementPolicy, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: stsMeta.GetLabels(),