feat: quorum-gated one-pod-at-a-time version upgrades via OnDelete

config/rbac/role.yaml

@@ -28,6 +28,13 @@ rules:
   - list
   - patch
   - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - delete
+  - get
 - apiGroups:
   - apps
   resources:

docs/upgrade.md

@@ -13,3 +13,24 @@ However, if you are using custom etcd image tags that are not
 compatible with [Semantic Versioning](https://semver.org/),
 no validation will be performed and you have to manually ensure
 that the upgrade path is supported.
+
+## Upgrade mechanics
+
+The StatefulSet uses the `OnDelete` update strategy, so a version change
+re-renders the pod template without restarting anything. The operator then
+replaces one pod per reconcile, and only while all members are healthy, a
+leader exists, and every member's revision is within 90% of the leader's.
+The leader's pod is replaced last. Persistent storage (`spec.storageSpec`)
+is strongly recommended: a deleted pod without a PVC loses its data
+directory and cannot rejoin the cluster cleanly.
+
+## Failed upgrades and rollback
+
+If a replaced pod cannot run the new version (nonexistent tag, incompatible
+flags), fix the spec — reverting `spec.version` to the version the members
+still report is accepted, since upgrade-path validation runs against the
+observed cluster version. While the remaining members hold quorum the
+operator keeps syncing the template and replaces the broken pod
+automatically. If quorum is lost it stops deleting pods; recover manually by
+fixing the spec and deleting the affected pods
+(`kubectl delete pod <cluster>-<ordinal>`).

internal/controller/etcdcluster_controller.go

@@ -47,7 +47,10 @@ const (
 // EtcdClusterReconciler reconciles a EtcdCluster object
 type EtcdClusterReconciler struct {
 	client.Client
-	Scheme        *runtime.Scheme
+	Scheme *runtime.Scheme
+	// PodReader reads pods straight from the API server. Reading pods through
+	// the cached client would lazily start a cluster-wide pod informer.
+	PodReader     client.Reader
 	Recorder      events.EventRecorder
 	ImageRegistry string
 }
@@ -69,6 +72,7 @@ type reconcileState struct {
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch;get;list;update
+// +kubebuilder:rbac:groups="",resources=pods,verbs=get;delete
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;patch;update;delete
 // +kubebuilder:rbac:groups="cert-manager.io",resources=certificates,verbs=get;list;watch;create;patch;update;delete
 // +kubebuilder:rbac:groups="cert-manager.io",resources=clusterissuers,verbs=get;list;watch
@@ -106,6 +110,12 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	if err = r.performHealthChecks(ctx, state); err != nil {
+		// A pod replaced during an upgrade may never come back healthy;
+		// without this the failed health check would keep template and pod
+		// convergence unreachable forever.
+		if res, handled := r.recoverDegradedUpgrade(ctx, state); handled {
+			return res, nil
+		}
 		return ctrl.Result{}, err
 	}
 
@@ -180,6 +190,12 @@ func (r *EtcdClusterReconciler) fetchAndValidateState(ctx context.Context, req c
 			return &reconcileState{cluster: ec, sts: sts}, ctrl.Result{}, nil
 		}
 		currentVersion := stsImage[idx+1:]
+		// Prefer the observed cluster version: the template tag may name an
+		// image that never ran (e.g. a nonexistent patch release), which would
+		// make every rollback look like a downgrade and wedge the cluster.
+		if ec.Status.CurrentVersion != "" {
+			currentVersion = ec.Status.CurrentVersion
+		}
 		targetVersion := ec.Spec.Version
 
 		// Only handle cases when there is a version change.
@@ -337,8 +353,7 @@ func (r *EtcdClusterReconciler) reconcileClusterState(ctx context.Context, s *re
 	}
 
 	if targetReplica == int32(s.cluster.Spec.Size) {
-		logger.Info("EtcdCluster is already up-to-date")
-		return ctrl.Result{}, nil
+		return r.reconcileVersionUpgrade(ctx, s)
 	}
 
 	eps := clientEndpointsFromStatefulsets(s.sts)
@@ -580,6 +595,9 @@ func isCertManagerCRDPresent(mgr ctrl.Manager) bool {
 // SetupWithManager sets up the controller with the Manager.
 func (r *EtcdClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	r.Recorder = mgr.GetEventRecorder("etcdcluster-controller")
+	if r.PodReader == nil {
+		r.PodReader = mgr.GetAPIReader()
+	}
 	setupLog := ctrl.Log.WithName("setup")
 
 	builder := ctrl.NewControllerManagedBy(mgr).

internal/controller/etcdcluster_controller_test.go

@@ -177,6 +177,51 @@ func TestFetchAndValidateState(t *testing.T) {
 				assert.Equal(t, ctrl.Result{}, res)
 			},
 		},
+		{
+			// The template may name a version that never ran (phantom patch
+			// tag); validation runs against the observed cluster version so
+			// reverting spec.version is not misread as a downgrade.
+			name: "Rollback to observed version allowed despite bad template tag",
+			ec: &ecv1alpha1.EtcdCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "etcd",
+					Namespace: "default",
+					UID:       "2",
+				},
+				Spec:   ecv1alpha1.EtcdClusterSpec{Size: 1, Version: "3.5.17"},
+				Status: ecv1alpha1.EtcdClusterStatus{CurrentVersion: "3.5.17"},
+			},
+			sts: &appsv1.StatefulSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "etcd",
+					Namespace: "default",
+					OwnerReferences: []metav1.OwnerReference{
+						{
+							APIVersion: ecv1alpha1.GroupVersion.String(),
+							Kind:       "EtcdCluster",
+							Name:       "etcd",
+							UID:        "2",
+							Controller: pointerToBool(true),
+						},
+					},
+				},
+				Spec: appsv1.StatefulSetSpec{
+					Template: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{Image: "gcr.io/etcd-development/etcd:3.6.99"},
+							},
+						},
+					},
+				},
+			},
+			req: ctrl.Request{NamespacedName: types.NamespacedName{Name: "etcd", Namespace: "default"}},
+			assert: func(t *testing.T, state *reconcileState, res ctrl.Result, err error, ec *ecv1alpha1.EtcdCluster, sts *appsv1.StatefulSet) {
+				require.NotNil(t, state)
+				assert.NoError(t, err)
+				assert.Equal(t, ctrl.Result{}, res)
+			},
+		},
 		{
 			name: "Cannot parse StatefulSet image tag",
 			ec: &ecv1alpha1.EtcdCluster{