From 276ea3ed979947d7cdd4b3d708862245ddcd8883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Tyczy=C5=84ski?= Date: Tue, 7 Dec 2021 16:38:32 +0100 Subject: [PATCH] Remove support for Endpoints and ConfigMaps lock from leader election Kubernetes-commit: 29d9683cd0fb3cc81810a8b39715e5eb4c68b00e --- tools/leaderelection/leaderelection_test.go | 92 ++----------------- .../resourcelock/configmaplock.go | 14 +-- .../resourcelock/endpointslock.go | 14 +-- .../leaderelection/resourcelock/interface.go | 86 +++++++++++++++-- 4 files changed, 99 insertions(+), 107 deletions(-) diff --git a/tools/leaderelection/leaderelection_test.go b/tools/leaderelection/leaderelection_test.go index 8fba412ceb..9691c8e02e 100644 --- a/tools/leaderelection/leaderelection_test.go +++ b/tools/leaderelection/leaderelection_test.go @@ -66,11 +66,6 @@ func createLockObject(t *testing.T, objectType, namespace, name string, record * return } -// Will test leader election using endpoints as the resource -func TestTryAcquireOrRenewEndpoints(t *testing.T) { - testTryAcquireOrRenew(t, "endpoints") -} - type Reactor struct { verb string objectType string @@ -259,24 +254,14 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { }) switch objectType { - case "endpoints": - lock = &rl.EndpointsLock{ - EndpointsMeta: objectMeta, - LockConfig: resourceLockConfig, - Client: c.CoreV1(), - } - case "configmaps": - lock = &rl.ConfigMapLock{ - ConfigMapMeta: objectMeta, - LockConfig: resourceLockConfig, - Client: c.CoreV1(), - } case "leases": lock = &rl.LeaseLock{ LeaseMeta: objectMeta, LockConfig: resourceLockConfig, Client: c.CoordinationV1(), } + default: + t.Fatalf("Unknown objectType: %v", objectType) } lec := LeaderElectionConfig{ @@ -325,11 +310,6 @@ func testTryAcquireOrRenew(t *testing.T, objectType string) { } } -// Will test leader election using configmap as the resource -func TestTryAcquireOrRenewConfigMaps(t *testing.T) { - testTryAcquireOrRenew(t, "configmaps") -} - // Will test leader election using lease as the resource func TestTryAcquireOrRenewLeases(t *testing.T) { testTryAcquireOrRenew(t, "leases") @@ -364,9 +344,9 @@ func TestLeaseSpecToLeaderElectionRecordRoundTrip(t *testing.T) { func multiLockType(t *testing.T, objectType string) (primaryType, secondaryType string) { switch objectType { case rl.EndpointsLeasesResourceLock: - return rl.EndpointsResourceLock, rl.LeasesResourceLock + return "endpoints", rl.LeasesResourceLock case rl.ConfigMapsLeasesResourceLock: - return rl.ConfigMapsResourceLock, rl.LeasesResourceLock + return "configmaps", rl.LeasesResourceLock default: t.Fatal("unexpected objType:" + objectType) } @@ -818,9 +798,7 @@ func testTryAcquireOrRenewMultiLock(t *testing.T, objectType string) { var wg sync.WaitGroup wg.Add(1) var reportedLeader string - var lock rl.Interface - objectMeta := metav1.ObjectMeta{Namespace: "foo", Name: "bar"} resourceLockConfig := rl.ResourceLockConfig{ Identity: "baz", EventRecorder: &record.FakeRecorder{}, @@ -834,33 +812,9 @@ func testTryAcquireOrRenewMultiLock(t *testing.T, objectType string) { return true, nil, fmt.Errorf("unreachable action") }) - switch objectType { - case rl.EndpointsLeasesResourceLock: - lock = &rl.MultiLock{ - Primary: &rl.EndpointsLock{ - EndpointsMeta: objectMeta, - LockConfig: resourceLockConfig, - Client: c.CoreV1(), - }, - Secondary: &rl.LeaseLock{ - LeaseMeta: objectMeta, - LockConfig: resourceLockConfig, - Client: c.CoordinationV1(), - }, - } - case rl.ConfigMapsLeasesResourceLock: - lock = &rl.MultiLock{ - Primary: &rl.ConfigMapLock{ - ConfigMapMeta: objectMeta, - LockConfig: resourceLockConfig, - Client: c.CoreV1(), - }, - Secondary: &rl.LeaseLock{ - LeaseMeta: objectMeta, - LockConfig: resourceLockConfig, - Client: c.CoordinationV1(), - }, - } + lock, err := rl.New(objectType, "foo", "bar", c.CoreV1(), c.CoordinationV1(), resourceLockConfig) + if err != nil { + t.Fatalf("Couldn't create lock: %v", err) } lec := LeaderElectionConfig{ @@ -983,24 +937,14 @@ func testReleaseLease(t *testing.T, objectType string) { }) switch objectType { - case "endpoints": - lock = &rl.EndpointsLock{ - EndpointsMeta: objectMeta, - LockConfig: resourceLockConfig, - Client: c.CoreV1(), - } - case "configmaps": - lock = &rl.ConfigMapLock{ - ConfigMapMeta: objectMeta, - LockConfig: resourceLockConfig, - Client: c.CoreV1(), - } case "leases": lock = &rl.LeaseLock{ LeaseMeta: objectMeta, LockConfig: resourceLockConfig, Client: c.CoordinationV1(), } + default: + t.Fatalf("Unknown objectType: %v", objectType) } lec := LeaderElectionConfig{ @@ -1058,29 +1002,11 @@ func testReleaseLease(t *testing.T, objectType string) { } } -// Will test leader election using endpoints as the resource -func TestReleaseLeaseEndpoints(t *testing.T) { - testReleaseLease(t, "endpoints") -} - -// Will test leader election using endpoints as the resource -func TestReleaseLeaseConfigMaps(t *testing.T) { - testReleaseLease(t, "configmaps") -} - // Will test leader election using endpoints as the resource func TestReleaseLeaseLeases(t *testing.T) { testReleaseLease(t, "leases") } -func TestReleaseOnCancellation_Endpoints(t *testing.T) { - testReleaseOnCancellation(t, "endpoints") -} - -func TestReleaseOnCancellation_ConfigMaps(t *testing.T) { - testReleaseOnCancellation(t, "configmaps") -} - func TestReleaseOnCancellation_Leases(t *testing.T) { testReleaseOnCancellation(t, "leases") } diff --git a/tools/leaderelection/resourcelock/configmaplock.go b/tools/leaderelection/resourcelock/configmaplock.go index 552280d2d7..5702728982 100644 --- a/tools/leaderelection/resourcelock/configmaplock.go +++ b/tools/leaderelection/resourcelock/configmaplock.go @@ -32,7 +32,7 @@ import ( // and use ConfigMaps as the means to pass that configuration // data we will likely move to deprecate the Endpoints lock. -type ConfigMapLock struct { +type configMapLock struct { // ConfigMapMeta should contain a Name and a Namespace of a // ConfigMapMeta object that the LeaderElector will attempt to lead. ConfigMapMeta metav1.ObjectMeta @@ -42,7 +42,7 @@ type ConfigMapLock struct { } // Get returns the election record from a ConfigMap Annotation -func (cml *ConfigMapLock) Get(ctx context.Context) (*LeaderElectionRecord, []byte, error) { +func (cml *configMapLock) Get(ctx context.Context) (*LeaderElectionRecord, []byte, error) { var record LeaderElectionRecord var err error cml.cm, err = cml.Client.ConfigMaps(cml.ConfigMapMeta.Namespace).Get(ctx, cml.ConfigMapMeta.Name, metav1.GetOptions{}) @@ -63,7 +63,7 @@ func (cml *ConfigMapLock) Get(ctx context.Context) (*LeaderElectionRecord, []byt } // Create attempts to create a LeaderElectionRecord annotation -func (cml *ConfigMapLock) Create(ctx context.Context, ler LeaderElectionRecord) error { +func (cml *configMapLock) Create(ctx context.Context, ler LeaderElectionRecord) error { recordBytes, err := json.Marshal(ler) if err != nil { return err @@ -81,7 +81,7 @@ func (cml *ConfigMapLock) Create(ctx context.Context, ler LeaderElectionRecord) } // Update will update an existing annotation on a given resource. -func (cml *ConfigMapLock) Update(ctx context.Context, ler LeaderElectionRecord) error { +func (cml *configMapLock) Update(ctx context.Context, ler LeaderElectionRecord) error { if cml.cm == nil { return errors.New("configmap not initialized, call get or create first") } @@ -102,7 +102,7 @@ func (cml *ConfigMapLock) Update(ctx context.Context, ler LeaderElectionRecord) } // RecordEvent in leader election while adding meta-data -func (cml *ConfigMapLock) RecordEvent(s string) { +func (cml *configMapLock) RecordEvent(s string) { if cml.LockConfig.EventRecorder == nil { return } @@ -116,11 +116,11 @@ func (cml *ConfigMapLock) RecordEvent(s string) { // Describe is used to convert details on current resource lock // into a string -func (cml *ConfigMapLock) Describe() string { +func (cml *configMapLock) Describe() string { return fmt.Sprintf("%v/%v", cml.ConfigMapMeta.Namespace, cml.ConfigMapMeta.Name) } // Identity returns the Identity of the lock -func (cml *ConfigMapLock) Identity() string { +func (cml *configMapLock) Identity() string { return cml.LockConfig.Identity } diff --git a/tools/leaderelection/resourcelock/endpointslock.go b/tools/leaderelection/resourcelock/endpointslock.go index dcd2026efd..af3fa16269 100644 --- a/tools/leaderelection/resourcelock/endpointslock.go +++ b/tools/leaderelection/resourcelock/endpointslock.go @@ -27,7 +27,7 @@ import ( corev1client "k8s.io/client-go/kubernetes/typed/core/v1" ) -type EndpointsLock struct { +type endpointsLock struct { // EndpointsMeta should contain a Name and a Namespace of an // Endpoints object that the LeaderElector will attempt to lead. EndpointsMeta metav1.ObjectMeta @@ -37,7 +37,7 @@ type EndpointsLock struct { } // Get returns the election record from a Endpoints Annotation -func (el *EndpointsLock) Get(ctx context.Context) (*LeaderElectionRecord, []byte, error) { +func (el *endpointsLock) Get(ctx context.Context) (*LeaderElectionRecord, []byte, error) { var record LeaderElectionRecord var err error el.e, err = el.Client.Endpoints(el.EndpointsMeta.Namespace).Get(ctx, el.EndpointsMeta.Name, metav1.GetOptions{}) @@ -58,7 +58,7 @@ func (el *EndpointsLock) Get(ctx context.Context) (*LeaderElectionRecord, []byte } // Create attempts to create a LeaderElectionRecord annotation -func (el *EndpointsLock) Create(ctx context.Context, ler LeaderElectionRecord) error { +func (el *endpointsLock) Create(ctx context.Context, ler LeaderElectionRecord) error { recordBytes, err := json.Marshal(ler) if err != nil { return err @@ -76,7 +76,7 @@ func (el *EndpointsLock) Create(ctx context.Context, ler LeaderElectionRecord) e } // Update will update and existing annotation on a given resource. -func (el *EndpointsLock) Update(ctx context.Context, ler LeaderElectionRecord) error { +func (el *endpointsLock) Update(ctx context.Context, ler LeaderElectionRecord) error { if el.e == nil { return errors.New("endpoint not initialized, call get or create first") } @@ -97,7 +97,7 @@ func (el *EndpointsLock) Update(ctx context.Context, ler LeaderElectionRecord) e } // RecordEvent in leader election while adding meta-data -func (el *EndpointsLock) RecordEvent(s string) { +func (el *endpointsLock) RecordEvent(s string) { if el.LockConfig.EventRecorder == nil { return } @@ -111,11 +111,11 @@ func (el *EndpointsLock) RecordEvent(s string) { // Describe is used to convert details on current resource lock // into a string -func (el *EndpointsLock) Describe() string { +func (el *endpointsLock) Describe() string { return fmt.Sprintf("%v/%v", el.EndpointsMeta.Namespace, el.EndpointsMeta.Name) } // Identity returns the Identity of the lock -func (el *EndpointsLock) Identity() string { +func (el *endpointsLock) Identity() string { return el.LockConfig.Identity } diff --git a/tools/leaderelection/resourcelock/interface.go b/tools/leaderelection/resourcelock/interface.go index bc77c2eda8..c6e23bda16 100644 --- a/tools/leaderelection/resourcelock/interface.go +++ b/tools/leaderelection/resourcelock/interface.go @@ -31,11 +31,77 @@ import ( const ( LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader" - EndpointsResourceLock = "endpoints" - ConfigMapsResourceLock = "configmaps" + endpointsResourceLock = "endpoints" + configMapsResourceLock = "configmaps" LeasesResourceLock = "leases" - EndpointsLeasesResourceLock = "endpointsleases" - ConfigMapsLeasesResourceLock = "configmapsleases" + // When using EndpointsLeasesResourceLock, you need to ensure that + // API Priority & Fairness is configured with non-default flow-schema + // that will catch the necessary operations on leader-election related + // endpoint objects. + // + // The example of such flow scheme could look like this: + // apiVersion: flowcontrol.apiserver.k8s.io/v1beta2 + // kind: FlowSchema + // metadata: + // name: my-leader-election + // spec: + // distinguisherMethod: + // type: ByUser + // matchingPrecedence: 200 + // priorityLevelConfiguration: + // name: leader-election # reference the PL + // rules: + // - resourceRules: + // - apiGroups: + // - "" + // namespaces: + // - '*' + // resources: + // - endpoints + // verbs: + // - get + // - create + // - update + // subjects: + // - kind: ServiceAccount + // serviceAccount: + // name: '*' + // namespace: kube-system + EndpointsLeasesResourceLock = "endpointsleases" + // When using EndpointsLeasesResourceLock, you need to ensure that + // API Priority & Fairness is configured with non-default flow-schema + // that will catch the necessary operations on leader-election related + // configmap objects. + // + // The example of such flow scheme could look like this: + // apiVersion: flowcontrol.apiserver.k8s.io/v1beta2 + // kind: FlowSchema + // metadata: + // name: my-leader-election + // spec: + // distinguisherMethod: + // type: ByUser + // matchingPrecedence: 200 + // priorityLevelConfiguration: + // name: leader-election # reference the PL + // rules: + // - resourceRules: + // - apiGroups: + // - "" + // namespaces: + // - '*' + // resources: + // - configmaps + // verbs: + // - get + // - create + // - update + // subjects: + // - kind: ServiceAccount + // serviceAccount: + // name: '*' + // namespace: kube-system + ConfigMapsLeasesResourceLock = "configmapsleases" ) // LeaderElectionRecord is the record that is stored in the leader election annotation. @@ -98,7 +164,7 @@ type Interface interface { // Manufacture will create a lock of a given type according to the input parameters func New(lockType string, ns string, name string, coreClient corev1.CoreV1Interface, coordinationClient coordinationv1.CoordinationV1Interface, rlc ResourceLockConfig) (Interface, error) { - endpointsLock := &EndpointsLock{ + endpointsLock := &endpointsLock{ EndpointsMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, @@ -106,7 +172,7 @@ func New(lockType string, ns string, name string, coreClient corev1.CoreV1Interf Client: coreClient, LockConfig: rlc, } - configmapLock := &ConfigMapLock{ + configmapLock := &configMapLock{ ConfigMapMeta: metav1.ObjectMeta{ Namespace: ns, Name: name, @@ -123,10 +189,10 @@ func New(lockType string, ns string, name string, coreClient corev1.CoreV1Interf LockConfig: rlc, } switch lockType { - case EndpointsResourceLock: - return endpointsLock, nil - case ConfigMapsResourceLock: - return configmapLock, nil + case endpointsResourceLock: + return nil, fmt.Errorf("endpoints lock is removed, migrate to %s", EndpointsLeasesResourceLock) + case configMapsResourceLock: + return nil, fmt.Errorf("configmaps lock is removed, migrate to %s", ConfigMapsLeasesResourceLock) case LeasesResourceLock: return leaseLock, nil case EndpointsLeasesResourceLock: