diff --git a/internal/controller/etcdcluster_controller.go b/internal/controller/etcdcluster_controller.go index 1c852bec..8566c67a 100644 --- a/internal/controller/etcdcluster_controller.go +++ b/internal/controller/etcdcluster_controller.go @@ -226,8 +226,9 @@ func (r *EtcdClusterReconciler) validateSpec(ctx context.Context, s *reconcileSt if currentVersion != targetVersion { canParse, err := validateEtcdUpgradePath(etcdversions.AllVersions, currentVersion, targetVersion) if !canParse { - logger.Info("error when parsing reconcile versions; it is your responsibility "+ - "to validate if the upgrade path is supported", + logger.Info( + "error when parsing reconcile versions; it is your responsibility "+ + "to validate if the upgrade path is supported", "current", currentVersion, "target", targetVersion, "error", err, @@ -235,7 +236,8 @@ func (r *EtcdClusterReconciler) validateSpec(ctx context.Context, s *reconcileSt return nil } if err != nil { - logger.Error(err, "unsupported upgrade path between current and target versions", + logger.Error( + err, "unsupported upgrade path between current and target versions", "current", currentVersion, "target", targetVersion, ) @@ -416,14 +418,63 @@ func (r *EtcdClusterReconciler) promoteLearner(ctx context.Context, s *reconcile // updateConfig compares each Pod's running configuration against // EtcdCluster.Spec and recreates the first Pod whose config has drifted. // -// TODO: not implemented yet. Per the workflow diagram's "Update config" -// phase, this should hash EtcdCluster.Spec, compare it against each Pod's +// This hashes the current EtcdCluster.Spec, compare it against each Pod's // recorded config hash, and recreate mismatched Pods one at a time, starting // with the highest ordinal and working down. If the Pod being replaced is // the leader, move leadership to another member (the one with the lowest // ordinal) first. func (r *EtcdClusterReconciler) updateConfig(ctx context.Context, s *reconcileState) (ctrl.Result, error) { - return ctrl.Result{}, nil + logger := log.FromContext(ctx) + + desiredHash := EtcdClusterHash(s.cluster) + configDrifted := findConfigDriftedPod(s.pods, s.cluster.Name, desiredHash) + if configDrifted == nil { + return ctrl.Result{}, nil + } + + configDriftedOrdinal := podOrdinal(configDrifted.Name, s.cluster.Name) + logger.Info("pod config drifted from spec, recreating it", + "pod", configDrifted.Name, + "desiredHash", desiredHash, + "podHash", configDrifted.Annotations[HashMetadataKey]) + + leaderID, leaderOrdinal := findLeader(s.memberHealth, s.memberListResp, s.cluster.Name, logger) + if leaderOrdinal == configDriftedOrdinal { + r.moveLeadership(ctx, s, leaderID, leaderOrdinal) + } + + if err := r.Delete(ctx, configDrifted); err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: requeueDuration}, nil +} + +// moveLeadership transfers etcd leadership away from the member at +// leaderOrdinal, whose Pod is about to be deleted, to the member returned by +// nextLeaderCandidate. +func (r *EtcdClusterReconciler) moveLeadership(ctx context.Context, s *reconcileState, leaderID uint64, leaderOrdinal int) { + logger := log.FromContext(ctx) + + candidateID, candidateOrdinal := nextLeaderCandidate(s.pods, s.memberListResp, s.cluster.Name, leaderID, leaderOrdinal) + if candidateID == leaderID { + logger.Info("No other member to take over leadership, proceeding without transferring it", + "leaderID", candidateID, "leaderOrdinal", candidateOrdinal) + return + } + + leaderEndpoint := clientEndpointForOrdinal(s.cluster.Name, s.cluster.Namespace, leaderOrdinal, clusterTLSEnabled(s.cluster)) + logger.Info("Transferring etcd leadership away from the member being acted on", + "leaderID", leaderID, "leaderOrdinal", leaderOrdinal, + "newLeaderID", candidateID, "newLeaderOrdinal", candidateOrdinal) + + if err := moveLeader(etcdutils.ClientConfig{Endpoints: []string{leaderEndpoint}, TLS: s.tlsConfig}, candidateID); err != nil { + logger.Error(err, "Failed to transfer etcd leadership", + "leaderID", leaderID, "newLeaderID", candidateID) + return + } + logger.Info("Etcd leadership transferred", "newLeaderID", candidateID, "newLeaderOrdinal", candidateOrdinal) } // scaleCluster compares the desired cluster size with the observed Pod count @@ -596,7 +647,8 @@ func (r *EtcdClusterReconciler) updateConditions(s *reconcileState) { } else { availableCondition.Message = fmt.Sprintf( "Etcd cluster has %d/%d healthy members, quorum requires %d", - healthyCount, len(s.memberListResp.Members), quorum) + healthyCount, len(s.memberListResp.Members), quorum, + ) } } diff --git a/internal/controller/etcdcluster_controller_test.go b/internal/controller/etcdcluster_controller_test.go index 5cda4b34..50e7d9cb 100644 --- a/internal/controller/etcdcluster_controller_test.go +++ b/internal/controller/etcdcluster_controller_test.go @@ -17,6 +17,9 @@ limitations under the License. package controller import ( + "context" + "errors" + "slices" "testing" "github.com/stretchr/testify/assert" @@ -25,11 +28,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/internal/etcdutils" + "go.etcd.io/etcd/api/v3/etcdserverpb" + clientv3 "go.etcd.io/etcd/client/v3" ) // TestFetchAndValidateState verifies the fetchAndValidateState helper across @@ -373,3 +381,231 @@ func TestBootstrapCluster(t *testing.T) { assert.Equal(t, "None", svc.Spec.ClusterIP) }) } + +type moveLeaderActivity struct { + calls int + endpoints []string + memberID uint64 +} + +func NewFakeMoveLeader(t *testing.T, err error) *moveLeaderActivity { + t.Helper() + + ml := &moveLeaderActivity{} + original := moveLeader + t.Cleanup(func() { moveLeader = original }) + + moveLeader = func(cfg etcdutils.ClientConfig, memberID uint64) error { + ml.calls++ + ml.endpoints = cfg.Endpoints + ml.memberID = memberID + return err + } + return ml +} + +const ( + updateConfigCluster = "etcd" + updateConfigNamespace = "default" + + staleConfigHash = "000000000000" +) + +func getMemberID(ordinal int) uint64 { return uint64(100 * (ordinal + 1)) } + +func clientEndpoint(ordinal int) string { + return clientEndpointForOrdinal(updateConfigCluster, updateConfigNamespace, ordinal, false) +} + +func newCluster(t *testing.T, size int) *ecv1alpha1.EtcdCluster { + t.Helper() + ec := &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: updateConfigCluster, Namespace: updateConfigNamespace}, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: size, + Version: "3.5.17", + ImageRegistry: DefaultImageRegistry, + }, + } + require.NoError(t, k8sClient.Create(t.Context(), ec)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), ec) + }) + return ec +} + +func newMemberPods(t *testing.T, ec *ecv1alpha1.EtcdCluster, size int, staleOrdinals ...int) { + t.Helper() + desiredHash := EtcdClusterHash(ec) + + for ordinal := range size { + pod := buildMemberPod(ec, memberPodName(ec.Name, ordinal), etcdClusterStateExisting, "ignored") + pod.Annotations[HashMetadataKey] = desiredHash + if slices.Contains(staleOrdinals, ordinal) { + pod.Annotations[HashMetadataKey] = staleConfigHash + } + require.NoError(t, controllerutil.SetControllerReference(ec, pod, scheme.Scheme)) + require.NoError(t, k8sClient.Create(t.Context(), pod)) + t.Cleanup(func() { + _ = k8sClient.Delete(context.Background(), pod, client.GracePeriodSeconds(0)) + }) + } +} + +func setupReconcileState(t *testing.T, ec *ecv1alpha1.EtcdCluster, leaderOrdinal int) *reconcileState { + t.Helper() + pods, err := listOwnedPods(t.Context(), k8sClient, ec) + require.NoError(t, err) + + state := &reconcileState{ + cluster: ec, + pods: pods, + memberListResp: &clientv3.MemberListResponse{}, + } + for _, pod := range pods { + ordinal := podOrdinal(pod.Name, ec.Name) + state.memberListResp.Members = append(state.memberListResp.Members, + &etcdserverpb.Member{ID: getMemberID(ordinal), Name: pod.Name}) + state.memberHealth = append(state.memberHealth, etcdutils.EpHealth{ + Ep: clientEndpoint(ordinal), + Health: true, + Status: &clientv3.StatusResponse{ + Header: &etcdserverpb.ResponseHeader{MemberId: getMemberID(ordinal)}, + Leader: getMemberID(leaderOrdinal), + }, + }) + } + return state +} + +func movedLeadership(from, to int) moveLeaderActivity { + return moveLeaderActivity{ + calls: 1, + memberID: getMemberID(to), + endpoints: []string{clientEndpoint(from)}, + } +} + +func remainingOrdinals(t *testing.T, ec *ecv1alpha1.EtcdCluster) []int { + t.Helper() + pods, err := listOwnedPods(t.Context(), k8sClient, ec) + require.NoError(t, err) + + ordinals := make([]int, 0, len(pods)) + for _, pod := range pods { + ordinals = append(ordinals, podOrdinal(pod.Name, ec.Name)) + } + slices.Sort(ordinals) + return ordinals +} + +func TestUpdateConfig(t *testing.T) { + tests := []struct { + name string + size int + leaderOrdinal int + staleOrdinals []int + moveLeaderErr error + wantRemaining []int + wantMoveLeader moveLeaderActivity + }{ + { + name: "no drift falls through to the next phase", + size: 3, + leaderOrdinal: 1, + wantRemaining: []int{0, 1, 2}, + }, + { + name: "every pod drifted recreates the highest ordinal only", + size: 3, + leaderOrdinal: 0, + staleOrdinals: []int{0, 1, 2}, + wantRemaining: []int{0, 1}, + }, + { + name: "drifted pod that is not the leader keeps leadership in place", + size: 3, + leaderOrdinal: 0, + staleOrdinals: []int{2}, + wantRemaining: []int{0, 1}, + }, + { + name: "drifted leader hands leadership to the lowest ordinal", + size: 3, + leaderOrdinal: 2, + staleOrdinals: []int{2}, + wantRemaining: []int{0, 1}, + wantMoveLeader: movedLeadership(2, 0), + }, + { + name: "drifted leader on the lowest ordinal hands leadership to the next one", + size: 3, + leaderOrdinal: 0, + staleOrdinals: []int{0}, + wantRemaining: []int{1, 2}, + wantMoveLeader: movedLeadership(0, 1), + }, + { + name: "drifted lowest ordinal that is not the leader keeps leadership in place", + size: 3, + leaderOrdinal: 1, + staleOrdinals: []int{0}, + wantRemaining: []int{1, 2}, + }, + { + name: "a failed transfer still recreates the pod", + size: 3, + leaderOrdinal: 2, + staleOrdinals: []int{2}, + moveLeaderErr: errors.New("etcdserver: request timed out"), + wantRemaining: []int{0, 1}, + wantMoveLeader: movedLeadership(2, 0), + }, + { + name: "single member cluster has nowhere to move leadership to", + size: 1, + leaderOrdinal: 0, + staleOrdinals: []int{0}, + wantRemaining: []int{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ec := newCluster(t, tt.size) + newMemberPods(t, ec, tt.size, tt.staleOrdinals...) + state := setupReconcileState(t, ec, tt.leaderOrdinal) + require.Len(t, state.pods, tt.size) + + r := &EtcdClusterReconciler{Client: k8sClient, Scheme: scheme.Scheme} + moveLeader := NewFakeMoveLeader(t, tt.moveLeaderErr) + + res, err := r.updateConfig(t.Context(), state) + require.NoError(t, err) + + wantResult := ctrl.Result{} + if len(tt.wantRemaining) != tt.size { + wantResult = ctrl.Result{RequeueAfter: requeueDuration} + } + assert.Equal(t, wantResult, res) + assert.Equal(t, tt.wantRemaining, remainingOrdinals(t, ec)) + assert.Equal(t, tt.wantMoveLeader, *moveLeader) + }) + } +} + +func TestUpdateConfigOnAlreadyGonePods(t *testing.T) { + ec := newCluster(t, 3) + newMemberPods(t, ec, 3, 2) + state := setupReconcileState(t, ec, 0) + + r := &EtcdClusterReconciler{Client: k8sClient, Scheme: scheme.Scheme} + NewFakeMoveLeader(t, nil) + + require.NoError(t, k8sClient.Delete(t.Context(), state.pods[2], client.GracePeriodSeconds(0))) + + res, err := r.updateConfig(t.Context(), state) + require.NoError(t, err) + assert.Equal(t, ctrl.Result{RequeueAfter: requeueDuration}, res) + assert.Equal(t, []int{0, 1}, remainingOrdinals(t, ec)) +} diff --git a/internal/controller/pods.go b/internal/controller/pods.go index 2f2bf7c0..2ba3a820 100644 --- a/internal/controller/pods.go +++ b/internal/controller/pods.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "maps" + "slices" "sort" "strconv" "strings" @@ -40,9 +41,14 @@ import ( ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" "go.etcd.io/etcd-operator/internal/etcdutils" + "go.etcd.io/etcd/api/v3/etcdserverpb" clientv3 "go.etcd.io/etcd/client/v3" ) +// moveLeader performs the etcd MoveLeader RPC. It is a package-level variable +// so unit tests can stub out the network call. +var moveLeader = etcdutils.MoveLeader + // memberPodName returns the deterministic name for an etcd member pod. // The naming convention mirrors StatefulSet so that headless-service DNS is identical. func memberPodName(clusterName string, ordinal int) string { @@ -74,7 +80,8 @@ func nextPodOrdinal(currentOrdinals []int, expectedReplica int) int { // sorted in ascending ordinal order. func listOwnedPods(ctx context.Context, c client.Client, ec *ecv1alpha1.EtcdCluster) ([]*corev1.Pod, error) { podList := &corev1.PodList{} - if err := c.List(ctx, podList, + if err := c.List( + ctx, podList, client.InNamespace(ec.Namespace), client.MatchingLabels(etcdClusterLabels(ec)), ); err != nil { @@ -104,6 +111,76 @@ func isPodReady(pod *corev1.Pod) bool { return false } +func clonePods(pods []*corev1.Pod, clusterName string) []*corev1.Pod { + sorted := slices.Clone(pods) + sort.Slice(sorted, func(i, j int) bool { + return podOrdinal(sorted[i].Name, clusterName) < podOrdinal(sorted[j].Name, clusterName) + }) + return sorted +} + +// findConfigDriftedPod returns the member Pod with the highest ordinal whose +// recorded config hash no longer matches desiredHash, or nil when every Pod is +// already running the desired config. +func findConfigDriftedPod(pods []*corev1.Pod, clusterName, desiredHash string) *corev1.Pod { + podsCopy := clonePods(pods, clusterName) + slices.Reverse(podsCopy) + + for _, pod := range podsCopy { + if podOrdinal(pod.Name, clusterName) < 0 { + continue // not a member Pod of this cluster, so never delete it + } + if pod.Annotations[HashMetadataKey] != desiredHash { + return pod + } + } + return nil +} + +// getMember returns the etcd member served by the given Pod. +func getMember(memberListResp *clientv3.MemberListResponse, podName string) (*etcdserverpb.Member, bool) { + for _, member := range memberListResp.Members { + if member.Name == podName { + return member, true + } + } + return nil, false +} + +// findLeader returns the member ID of the current etcd leader and the ordinal +// of the Pod serving it. +func findLeader(memberHealth []etcdutils.EpHealth, + memberListResp *clientv3.MemberListResponse, + clusterName string, + logger logr.Logger, +) (leaderID uint64, leaderOrdinal int) { + leaderID, _ = etcdutils.FindLeaderStatus(memberHealth, logger) + if leaderID == 0 { + return 0, -1 + } + for _, member := range memberListResp.Members { + if member.ID == leaderID { + return leaderID, podOrdinal(member.Name, clusterName) + } + } + return leaderID, -1 +} + +// nextLeaderCandidate returns the member that etcd leadership should be moved +// to before the current leader's Pod is removed or recreated, along with that +// member's Pod ordinal: the existing member with the lowest ordinal, excluding +// the current leader itself. +func nextLeaderCandidate(pods []*corev1.Pod, memberListResp *clientv3.MemberListResponse, clusterName string, currentLeaderID uint64, currentLeaderOrdinal int) (memberID uint64, ordinal int) { + for _, pod := range clonePods(pods, clusterName) { + member, ok := getMember(memberListResp, pod.Name) + if !ok || member.ID == currentLeaderID { + continue + } + return member.ID, podOrdinal(pod.Name, clusterName) + } + return currentLeaderID, currentLeaderOrdinal +} + // waitForPodReady polls until the given Pod has its Ready condition set to True, // using an exponential back-off. It is provided as a utility; the primary // reconcile paths do not block on it, relying on natural requeueing instead. @@ -243,7 +320,8 @@ const ( func defaultArgs(name string, tlsEnabled bool) []string { scheme := clusterScheme(tlsEnabled) args := make([]string, 0, 13) - args = append(args, + args = append( + args, "--name=$(POD_NAME)", fmt.Sprintf("--listen-peer-urls=%s://0.0.0.0:2380", scheme), fmt.Sprintf("--listen-client-urls=%s://0.0.0.0:2379", scheme), @@ -253,7 +331,8 @@ func defaultArgs(name string, tlsEnabled bool) []string { if !tlsEnabled { return args } - return append(args, + return append( + args, "--cert-file="+serverCertFile, "--key-file="+serverKeyFile, "--trusted-ca-file="+serverTrustedCAFile, @@ -363,7 +442,8 @@ func buildMemberPod(ec *ecv1alpha1.EtcdCluster, podName string, initialClusterSt // setup TLS certificate volumes. Both the pod Volume and VolumeMount are // declared together so the etcd TLS args file paths resolve at runtime. if ec.Spec.TLS != nil { - podSpec.Volumes = append(podSpec.Volumes, + podSpec.Volumes = append( + podSpec.Volumes, corev1.Volume{ Name: "server-secret", VolumeSource: corev1.VolumeSource{ @@ -379,7 +459,8 @@ func buildMemberPod(ec *ecv1alpha1.EtcdCluster, podName string, initialClusterSt ) etcdContainer := &podSpec.Containers[0] - etcdContainer.VolumeMounts = append(etcdContainer.VolumeMounts, + etcdContainer.VolumeMounts = append( + etcdContainer.VolumeMounts, corev1.VolumeMount{ Name: "server-secret", MountPath: serverCertMountDir, diff --git a/internal/controller/pods_test.go b/internal/controller/pods_test.go index b06c9c1b..4da17a47 100644 --- a/internal/controller/pods_test.go +++ b/internal/controller/pods_test.go @@ -863,3 +863,260 @@ func TestBuildMemberPodTLSVolumes(t *testing.T) { assert.Contains(t, args, "--listen-client-urls=http://0.0.0.0:2379") }) } + +const testClusterName = "etcd" + +func podNames(pods []*corev1.Pod) []string { + names := make([]string, 0, len(pods)) + for _, pod := range pods { + names = append(names, pod.Name) + } + return names +} + +func createPods(ordinals ...int) []*corev1.Pod { + pods := make([]*corev1.Pod, 0, len(ordinals)) + for _, ordinal := range ordinals { + pods = append(pods, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: memberPodName(testClusterName, ordinal), + Namespace: "default", + }, + }) + } + return pods +} + +func TestClonePods(t *testing.T) { + tests := []struct { + name string + ordinals []int + want []string + }{ + { + name: "sorts by ordinal, not by name", + ordinals: []int{2, 10, 0}, + want: []string{"etcd-0", "etcd-2", "etcd-10"}, + }, + { + name: "no pods", + ordinals: nil, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pods := createPods(tt.ordinals...) + asPassedIn := podNames(pods) + + assert.Equal(t, tt.want, podNames(clonePods(pods, testClusterName))) + assert.Equal(t, asPassedIn, podNames(pods)) + }) + } +} + +func TestFindConfigDriftedPod(t *testing.T) { + const ( + clusterName = "etcd" + desiredHash = "abc123def456" + staleHash = "000000000000" + ) + + withHash := func(pods []*corev1.Pod, hashes map[string]string) []*corev1.Pod { + for _, pod := range pods { + if hash, ok := hashes[pod.Name]; ok { + pod.Annotations = map[string]string{HashMetadataKey: hash} + } + } + return pods + } + + tests := []struct { + name string + pods []*corev1.Pod + want string // expected pod name, "" for nil + }{ + { + name: "no pods", + pods: nil, + want: "", + }, + { + name: "every pod is up to date", + pods: withHash(createPods(0, 1, 2), map[string]string{ + "etcd-0": desiredHash, "etcd-1": desiredHash, "etcd-2": desiredHash, + }), + want: "", + }, + { + name: "single drifted pod", + pods: withHash(createPods(0, 1, 2), map[string]string{ + "etcd-0": desiredHash, "etcd-1": staleHash, "etcd-2": desiredHash, + }), + want: "etcd-1", + }, + { + name: "several drifted pods returns the highest ordinal", + pods: withHash(createPods(0, 1, 2), map[string]string{ + "etcd-0": staleHash, "etcd-1": staleHash, "etcd-2": staleHash, + }), + want: "etcd-2", + }, + { + name: "a pod with no hash annotation counts as drifted", + pods: withHash(createPods(0, 1), map[string]string{ + "etcd-0": desiredHash, + }), + want: "etcd-1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := findConfigDriftedPod(tt.pods, clusterName, desiredHash) + if tt.want == "" { + assert.Nil(t, got) + return + } + require.NotNil(t, got) + assert.Equal(t, tt.want, got.Name) + }) + } +} + +func TestFindLeader(t *testing.T) { + memberList := &clientv3.MemberListResponse{ + Members: []*etcdserverpb.Member{ + {ID: 100, Name: "etcd-0"}, + {ID: 200, Name: "etcd-1"}, + {ID: 300, Name: "etcd-2"}, + }, + } + + health := func(leaderID uint64, memberIDs ...uint64) []etcdutils.EpHealth { + infos := make([]etcdutils.EpHealth, 0, len(memberIDs)) + for _, id := range memberIDs { + infos = append(infos, etcdutils.EpHealth{ + Health: true, + Status: &clientv3.StatusResponse{ + Header: &etcdserverpb.ResponseHeader{MemberId: id}, + Leader: leaderID, + }, + }) + } + return infos + } + + tests := []struct { + name string + memberHealth []etcdutils.EpHealth + wantLeaderID uint64 + wantOrdinal int + }{ + { + name: "leader on the lowest ordinal", + memberHealth: health(100, 100, 200, 300), + wantLeaderID: 100, + wantOrdinal: 0, + }, + { + name: "leader in the middle", + memberHealth: health(200, 100, 200, 300), + wantLeaderID: 200, + wantOrdinal: 1, + }, + { + name: "leader on the highest ordinal", + memberHealth: health(300, 100, 200, 300), + wantLeaderID: 300, + wantOrdinal: 2, + }, + { + name: "leader reported by a member that is not itself the leader", + memberHealth: health(300, 100, 200, 300)[:2], + wantLeaderID: 0, + wantOrdinal: -1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + leaderID, leaderOrdinal := findLeader(tt.memberHealth, memberList, testClusterName, log.Log) + + assert.Equal(t, tt.wantLeaderID, leaderID) + assert.Equal(t, tt.wantOrdinal, leaderOrdinal) + }) + } +} + +func TestNextLeaderCandidate(t *testing.T) { + const clusterName = "etcd" + + memberList := func(members ...*etcdserverpb.Member) *clientv3.MemberListResponse { + return &clientv3.MemberListResponse{Members: members} + } + member := func(id uint64, name string) *etcdserverpb.Member { + return &etcdserverpb.Member{ID: id, Name: name} + } + + threeMembers := memberList( + member(100, "etcd-0"), + member(200, "etcd-1"), + member(300, "etcd-2"), + ) + + tests := []struct { + name string + pods []*corev1.Pod + memberListResp *clientv3.MemberListResponse + currentLeaderID uint64 + currentLeaderOrdinal int + wantID uint64 + wantOrdinal int + }{ + { + name: "picks the lowest ordinal member", + pods: createPods(0, 1, 2), + memberListResp: threeMembers, + currentLeaderID: 300, + currentLeaderOrdinal: 2, + wantID: 100, + wantOrdinal: 0, + }, + { + name: "skips the current leader even when it is the lowest", + pods: createPods(0, 1, 2), + memberListResp: threeMembers, + currentLeaderID: 100, + currentLeaderOrdinal: 0, + wantID: 200, + wantOrdinal: 1, + }, + { + name: "unsorted pods still yield the lowest ordinal", + pods: createPods(2, 0, 1), + memberListResp: threeMembers, + currentLeaderID: 300, + currentLeaderOrdinal: 2, + wantID: 100, + wantOrdinal: 0, + }, + { + name: "single member cluster keeps its leader", + pods: createPods(0), + memberListResp: memberList(member(100, "etcd-0")), + currentLeaderID: 100, + currentLeaderOrdinal: 0, + wantID: 100, + wantOrdinal: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotID, gotOrdinal := nextLeaderCandidate(tt.pods, tt.memberListResp, clusterName, tt.currentLeaderID, tt.currentLeaderOrdinal) + assert.Equal(t, tt.wantID, gotID) + assert.Equal(t, tt.wantOrdinal, gotOrdinal) + }) + } +}