From 7ac40ac144758d028df70d72dd1871248443797a Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Wed, 1 Jul 2026 12:17:27 -0400 Subject: [PATCH 1/5] feat(etcdmirror): add EtcdMirror CRD Go types and CEL validation Introduces the EtcdMirror CRD (schema only, no controller/agent) that will describe a continuous one-way key-range sync from a source etcd cluster to a target etcd cluster. Adds EtcdMirrorSpec/Status, the EtcdMirrorEndpoint/TLS/Auth/Sync/Checkpoint/Reconciliation sub-specs, phase and condition constants (including TargetThrottled and ReplicationLagExceeded), printer columns, and CEL XValidation rules for the destPrefix/noDestPrefix mutual exclusion, the endpointList/serviceRef oneOf, and the insecureSkipVerify/ insecureSkipVerifyAcknowledgeRisk companion-field requirement. Registers EtcdMirror/EtcdMirrorList in the scheme, regenerates zz_generated.deepcopy.go and the CRD manifest via controller-gen, and adds a realistic sample CR. CEL rules are covered by a table-driven envtest suite in etcdmirror_cel_test.go, following the tls_cel_test.go pattern, exercised against a real envtest apiserver. The design's Metrics field (reusing EtcdClusterSpec's MetricsSpec) is intentionally omitted: MetricsSpec does not exist on this branch yet and will land with pr/domain-metrics; it will be added in a later PR once that type is importable. No controller or agent logic in this PR -- types, generated code, and CRD schema only. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Xavier Lange --- api/v1alpha1/etcdmirror_types.go | 548 ++++++ api/v1alpha1/groupversion_info.go | 2 + api/v1alpha1/zz_generated.deepcopy.go | 333 +++- .../bases/operator.etcd.io_etcdmirrors.yaml | 1716 +++++++++++++++++ .../samples/operator_v1alpha1_etcdmirror.yaml | 50 + internal/controller/etcdmirror_cel_test.go | 326 ++++ 6 files changed, 2974 insertions(+), 1 deletion(-) create mode 100644 api/v1alpha1/etcdmirror_types.go create mode 100644 config/crd/bases/operator.etcd.io_etcdmirrors.yaml create mode 100644 config/samples/operator_v1alpha1_etcdmirror.yaml create mode 100644 internal/controller/etcdmirror_cel_test.go diff --git a/api/v1alpha1/etcdmirror_types.go b/api/v1alpha1/etcdmirror_types.go new file mode 100644 index 00000000..2bf64789 --- /dev/null +++ b/api/v1alpha1/etcdmirror_types.go @@ -0,0 +1,548 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EtcdMirrorSpec defines the desired state of an EtcdMirror. +// +// +kubebuilder:validation:XValidation:rule="!(has(self.sync) && has(self.sync.destPrefix) && size(self.sync.destPrefix) > 0 && has(self.sync.noDestPrefix) && self.sync.noDestPrefix)",message="sync.destPrefix and sync.noDestPrefix are mutually exclusive" +type EtcdMirrorSpec struct { + // Source is the etcd cluster keys are read from. EtcdMirror never writes + // back to Source; the agent's source-side client is only ever used for + // Get/Watch, never Put/Delete/Txn. + Source EtcdMirrorEndpoint `json:"source"` + + // Target is the etcd cluster keys are written into. + // + // SECURITY PREREQUISITE: Target.Auth's credential (and/or the target + // client certificate's associated etcd RBAC role) MUST be an etcd RBAC + // user restricted via a range-scoped grant-permission to Target.Prefix (or + // Sync.DestPrefix's resulting range) -- never a cluster-admin-equivalent + // credential. The agent's client-side rewriteKey/prefix logic is defense + // against bugs in the agent's own code; it is NOT a security boundary. A + // compromised agent process, a buggy rewriteKey, or an empty/typo'd prefix + // with an over-privileged credential can write or delete anywhere in the + // target cluster's entire keyspace. This is a deployment prerequisite the + // operator (human) must configure in etcd itself (via `etcdctl role + // grant-permission --prefix=true readwrite `) before + // pointing an EtcdMirror at it; the controller cannot create or enforce + // etcd-native RBAC roles itself, but it does check for and surface an + // unrestricted-looking target credential where etcd's auth API makes that + // detectable (see Safety Guards). + Target EtcdMirrorEndpoint `json:"target"` + + // ExpectEmptyPrefix, when true, makes the controller verify the target's + // effective destination prefix has zero keys before the agent's FIRST-EVER + // SyncBase begins (i.e. no valid checkpoint exists yet for this source + // cluster identity), refusing to proceed (Phase -> Failed, condition + // EmptyTargetViolation) if it is already non-empty. Analogous to + // EtcdRestore's assertClusterEmpty guard. + // + // SCOPE, STATED PLAINLY: this is single-shot, genesis-only protection. It + // is checked exactly once, at the moment a fresh (or cluster-identity- + // invalidated, see Checkpoint Lifecycle) InitialSync begins. It does NOT + // re-arm on a compaction-triggered forced resync later in the CR's life + // (that resync reuses the already-established target prefix ownership), + // and it does NOT protect against a foreign writer landing keys under the + // same target prefix at any point AFTER the first successful sync -- there + // is no key-tagging/provenance marker in v1 that would let the agent + // distinguish "my own prior output" from "someone else's write" on a + // later re-check (see Non-Goals). Operators who need ongoing enforcement + // that nothing else writes to the target prefix must arrange that via + // etcd RBAC (grant only this mirror's target credential write access to + // the prefix) rather than relying on ExpectEmptyPrefix, which is a + // bring-up guard, not a standing invariant. + // Defaults to false; operators standing up a NEW mirror onto a fresh + // prefix should set this true. + // +optional + ExpectEmptyPrefix bool `json:"expectEmptyPrefix,omitempty"` + + // Sync tunes runtime sync behavior (batching, rate limiting, prefix + // rewrite, backoff). + // +optional + Sync EtcdMirrorSyncSpec `json:"sync,omitempty"` + + // Checkpoint configures the agent's local durable progress checkpoint. + // +optional + Checkpoint *EtcdMirrorCheckpointSpec `json:"checkpoint,omitempty"` + + // Reconciliation optionally enables a periodic full diff-and-repair pass + // layered on top of the continuous watch-based mirror. + // +optional + Reconciliation *EtcdMirrorReconciliationSpec `json:"reconciliation,omitempty"` + + // PodTemplate carries scheduling/affinity/labels/annotations for the agent + // pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. + // +optional + PodTemplate *PodTemplate `json:"podTemplate,omitempty"` + + // Paused, when true, tells the controller to scale the agent StatefulSet to + // zero replicas without deleting the CR or its checkpoint PVC. For planned + // maintenance windows on either cluster without losing sync position. The + // agent's own retry/backoff loop handles transient interruptions on its + // own; Paused is for deliberate, operator-initiated stops. + // +optional + Paused bool `json:"paused,omitempty"` +} + +// EtcdMirrorEndpoint describes how to reach, authenticate to, and scope one +// side (source or target) of a mirror. Both sides need an identical shape +// (address resolution + prefix + TLS + auth), so one type serves both roles, +// the same way BackupDestination is reused verbatim between EtcdBackup and +// EtcdRestore rather than forked into near-duplicate per-role types. +// +// Exactly one of EndpointList or ServiceRef must be set. +// +// +kubebuilder:validation:XValidation:rule="(has(self.endpointList) && size(self.endpointList) > 0) != has(self.serviceRef)",message="exactly one of endpointList or serviceRef must be set" +type EtcdMirrorEndpoint struct { + // EndpointList is a raw set of etcd client-URL host:port (or + // scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". + // This is the ONLY supported mechanism for a cluster external to this + // Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB) -- + // there is deliberately no "tunnel" or "port-forward" mode. A + // `kubectl port-forward` is a client-attached, ephemeral process tied to a + // human's terminal; it does not survive this mirror pod restarting, + // rescheduling, or the operator itself restarting, which defeats the + // entire point of a supervised, restart-tolerant workload. If you need a + // persistent tunnel, terminate it yourself (VPN, Interconnect, NLB) + // upstream of this CR and hand EtcdMirror the resulting stable + // endpoint(s); network reachability itself is out of scope for this CRD. + // + // IP-LITERAL ENDPOINTS: if any entry here is a bare IP literal (plausible + // for an NLB endpoint with no DNS name), Go's TLS stack requires an IP SAN + // (not a DNS SAN) on the peer certificate for verification to succeed. + // Either set the corresponding EtcdMirrorTLS.ServerName to a hostname that + // IS present as a DNS SAN on the certificate, or ensure the certificate + // carries an IP SAN matching the literal. Operators who hit a verification + // failure here should fix the SAN/ServerName mismatch, not reach for + // InsecureSkipVerify to work around it. + // +optional + EndpointList []string `json:"endpointList,omitempty"` + + // ServiceRef points at a Kubernetes Service in this cluster whose DNS name + // resolves the etcd client endpoint(s). The rarer same-cluster or + // co-located case (e.g. mirroring between two EtcdClusters both running in + // this GKE cluster). Namespace defaults to the EtcdMirror's own namespace + // when empty. + // +optional + ServiceRef *EtcdMirrorServiceRef `json:"serviceRef,omitempty"` + + // Prefix is the etcd key prefix on THIS side. On Source, only keys under + // this prefix are synced; empty means the whole keyspace. On Target, this + // is the prefix under which the agent writes mirrored keys after any + // Sync.DestPrefix remap is applied. + // +optional + Prefix string `json:"prefix,omitempty"` + + // TLS configures the agent's client TLS identity/trust for THIS side. Nil + // means the agent dials this side in cleartext. + // +optional + TLS *EtcdMirrorTLS `json:"tls,omitempty"` + + // Auth configures etcd username/password (RBAC) auth for THIS side, + // ambient-or-secretRef per the objectstore_creds.go pattern. + // +optional + Auth *EtcdMirrorAuth `json:"auth,omitempty"` +} + +// EtcdMirrorServiceRef points at a Service and the client port on it to dial. +type EtcdMirrorServiceRef struct { + // Name is the Service name. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Namespace defaults to the EtcdMirror's own namespace when empty. + // +optional + Namespace string `json:"namespace,omitempty"` + + // Port is the Service port name or number exposing etcd's client API. + // Defaults to "client" when empty, matching PodMonitorSpec.Port's + // convention elsewhere in this API group. + // +optional + Port string `json:"port,omitempty"` +} + +// EtcdMirrorTLS configures the mirror agent's client TLS identity for one side +// of a mirror. Plain secretRef, not a reuse of EtcdClusterTLS/TLSSurface -- +// the mirror agent is always a client to clusters it does not own, so the +// issuer-selection machinery in TLSSurface/ProviderCertManagerConfig doesn't apply. +// +// +kubebuilder:validation:XValidation:rule="!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk",message="insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk to also be true" +// +kubebuilder:validation:XValidation:rule="has(self.secretRef.name) && size(self.secretRef.name) > 0",message="secretRef.name is required" +type EtcdMirrorTLS struct { + // SecretRef names a Secret (in the EtcdMirror's namespace) holding this + // side's TLS material in the standard kubernetes.io/tls-compatible shape: + // - ca.crt: PEM CA bundle used to verify the peer's server certificate. + // Required for TLS unless InsecureSkipVerify is true. + // - tls.crt / tls.key: PEM client certificate + key, for mTLS. Optional -- + // omit both for server-auth-only TLS (verify the peer, authenticate to + // it via Auth instead, or not at all). + // This is the same key layout a cert-manager Certificate's spec.secretName + // Secret uses, so a cert-manager Certificate (or `kubectl create secret + // tls`, or any other issuance mechanism) can populate this Secret with zero + // coupling from this CRD to cert-manager. + SecretRef corev1.LocalObjectReference `json:"secretRef"` + + // InsecureSkipVerify disables server certificate verification. Strongly + // discouraged, especially for a source reached over the public internet. + // Requires InsecureSkipVerifyAcknowledgeRisk to also be set true + // (CEL-enforced on this type). The controller additionally emits a + // standing Warning event whenever this is true. + // +optional + // +kubebuilder:default=false + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` + + // InsecureSkipVerifyAcknowledgeRisk must independently be set true + // whenever InsecureSkipVerify is true (CEL-enforced companion field). Its + // only purpose is to require a deliberate, separate, reviewable line in + // the manifest diff before disabling TLS verification. + // +optional + // +kubebuilder:default=false + InsecureSkipVerifyAcknowledgeRisk bool `json:"insecureSkipVerifyAcknowledgeRisk,omitempty"` + + // ServerName overrides the TLS ServerName (SNI) used for verification, for + // cases where the dialed endpoint's address doesn't match a SAN on the + // certificate (e.g. dialing an NLB IP directly -- see EndpointList's + // IP-literal guidance). + // +optional + ServerName string `json:"serverName,omitempty"` +} + +// EtcdMirrorAuth configures etcd RBAC username/password auth for one side. +// +// +kubebuilder:validation:XValidation:rule="has(self.secretRef.name) && size(self.secretRef.name) > 0",message="secretRef.name is required" +type EtcdMirrorAuth struct { + // SecretRef names a Secret holding "username" and "password" keys. If this + // whole Auth block is nil, the agent does not call etcd's Authenticate() + // at all. There is no ambient fallback for etcd username/password auth + // (unlike objectstore's IRSA/Workload Identity). + SecretRef corev1.LocalObjectReference `json:"secretRef"` +} + +// EtcdMirrorSyncSpec tunes the mirror's runtime sync behavior. Defaults are +// chosen to match etcdctl make-mirror's own defaults where one exists (e.g. +// MaxTxnOps=128). +type EtcdMirrorSyncSpec struct { + // DestPrefix rewrites Source.Prefix to a different prefix on the target, + // via an anchored strip-and-reprefix (strings.TrimPrefix(key, + // source.Prefix), then destPrefix + rest) -- never make-mirror's naive + // first-occurrence strings.Replace. Mutually exclusive with NoDestPrefix. + // Empty (and NoDestPrefix false) means the source prefix is reused + // verbatim on the target, landing keys under Target.Prefix + the stripped + // key remainder. + // +optional + DestPrefix string `json:"destPrefix,omitempty"` + + // NoDestPrefix strips Source.Prefix entirely rather than remapping it, + // mirroring make-mirror's --no-dest-prefix. Mutually exclusive with a + // non-empty DestPrefix (rejected by EtcdMirrorSpec's XValidation rule). + // +optional + NoDestPrefix bool `json:"noDestPrefix,omitempty"` + + // MaxTxnOps bounds how many put/delete operations the agent batches into a + // single destination Txn, applied uniformly to BOTH the initial SyncBase + // phase and the watch-driven SyncUpdates phase. Defaults to 128 when unset. + // +optional + // +kubebuilder:validation:Minimum=1 + MaxTxnOps int32 `json:"maxTxnOps,omitempty"` + + // MaxOpsPerSecond rate-limits the agent's destination write rate (a simple + // token bucket over puts+deletes/sec), applied to BOTH InitialSync's Txn + // stream and SyncUpdates. Zero (default) means unlimited. + // +optional + // +kubebuilder:validation:Minimum=0 + MaxOpsPerSecond int32 `json:"maxOpsPerSecond,omitempty"` + + // ReconnectBackoff bounds the retry/backoff loop wrapping every Syncer call + // and every destination Txn call. Defaults to exponential backoff from 1s + // to 30s, uncapped in attempt count, when unset. + // +optional + ReconnectBackoff *EtcdMirrorBackoffSpec `json:"reconnectBackoff,omitempty"` + + // DialTimeout bounds how long the agent waits to establish the initial + // client connection to each side. Defaults to 10s when unset. + // +optional + DialTimeout *metav1.Duration `json:"dialTimeout,omitempty"` +} + +type EtcdMirrorBackoffSpec struct { + // +optional + InitialDelay *metav1.Duration `json:"initialDelay,omitempty"` + // +optional + MaxDelay *metav1.Duration `json:"maxDelay,omitempty"` +} + +// EtcdMirrorCheckpointSpec configures the agent's local PVC-backed checkpoint. +type EtcdMirrorCheckpointSpec struct { + // StorageSpec requests persistent storage for the checkpoint file, reusing + // EtcdClusterSpec's StorageSpec shape. When nil the controller mounts an + // emptyDir instead. StorageSpec should be set for any production mirror; a + // small size (e.g. 64Mi) is sufficient. + // +optional + StorageSpec *StorageSpec `json:"storageSpec,omitempty"` + + // SyncInterval controls how often the agent flushes its in-memory + // last-applied revision to the checkpoint file (atomic write: temp file + + // fsync + rename). Defaults to 5s when unset. + // +optional + SyncInterval *metav1.Duration `json:"syncInterval,omitempty"` +} + +// EtcdMirrorReconciliationSpec configures an OPTIONAL periodic full +// reconciliation pass layered on top of the continuous watch-based mirror. +type EtcdMirrorReconciliationSpec struct { + // Enabled toggles the periodic full reconciliation pass. Defaults to + // false: it is a diff of the full prefix contents on both sides + // (O(keyspace size)), so it is opt-in. + // +optional + Enabled bool `json:"enabled,omitempty"` + + // Interval between reconciliation passes. Defaults to 1h when Enabled and + // unset. + // +optional + Interval *metav1.Duration `json:"interval,omitempty"` + + // DeleteOrphans, when true, allows reconciliation to DELETE target keys + // under the destination prefix that have no corresponding source key. + // Defaults to false. + // +optional + DeleteOrphans bool `json:"deleteOrphans,omitempty"` +} + +// EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. +// Unlike BackupPhase/RestorePhase, most phases here are NOT terminal -- a +// healthy mirror spends its life cycling between Syncing and (briefly, on +// transient errors) Degraded; there is no "Completed" state. +type EtcdMirrorPhase string + +const ( + // EtcdMirrorPhasePending means the EtcdMirror has been accepted but the + // agent workload has not been created yet. + EtcdMirrorPhasePending EtcdMirrorPhase = "Pending" + // EtcdMirrorPhaseConnecting means the agent pod is running and establishing + // client connections to both Source and Target. + EtcdMirrorPhaseConnecting EtcdMirrorPhase = "Connecting" + // EtcdMirrorPhaseInitialSync means the agent is running SyncBase: the + // paginated, revision-pinned full range scan establishing the base + // revision. Entered on a genesis start (no valid checkpoint), a + // cluster-identity-invalidated checkpoint, or an agent-initiated forced + // resync after compaction (see EtcdMirrorConditionCompacted). + EtcdMirrorPhaseInitialSync EtcdMirrorPhase = "InitialSync" + // EtcdMirrorPhaseSyncing is the steady-state: SyncBase has completed (or + // was skipped via checkpoint resume) and SyncUpdates is watching and + // applying live changes. A healthy mirror spends effectively all its time + // here. + EtcdMirrorPhaseSyncing EtcdMirrorPhase = "Syncing" + // EtcdMirrorPhaseDegraded means the agent hit a recoverable condition + // (reconnect backoff to either side, target throttling backoff, or an + // in-progress compaction-forced resync) and is retrying/self-healing. + // Non-terminal: expected to return to Syncing/InitialSync automatically + // without operator action. + EtcdMirrorPhaseDegraded EtcdMirrorPhase = "Degraded" + // EtcdMirrorPhasePaused means spec.paused is true; the agent StatefulSet is + // scaled to zero. The checkpoint is retained so resuming picks up from the + // last-applied revision (subject to the same cluster-identity check on + // resume as any other restart). + EtcdMirrorPhasePaused EtcdMirrorPhase = "Paused" + // EtcdMirrorPhaseFailed means the mirror hit a terminal, non-recoverable + // error (ExpectEmptyPrefix violated at genesis, malformed cert material, + // unresolvable spec misconfiguration) requiring operator intervention. + EtcdMirrorPhaseFailed EtcdMirrorPhase = "Failed" +) + +// Condition types reported on EtcdMirror status. +const ( + // EtcdMirrorConditionAvailable is True only when the agent pod is running, + // in the Syncing phase, AND the loop-liveness check shows forward progress + // within the configured staleness threshold. A wedged main loop that keeps + // answering /statusz with a stale-but-structurally-valid response is + // explicitly NOT allowed to read as Available=True; see + // ReplicationLagExceeded below for the companion signal. + EtcdMirrorConditionAvailable = "Available" + // EtcdMirrorConditionSourceReachable is True when the agent's last attempt + // to reach Source succeeded. Split from TargetReachable because in the + // primary use case (RKE1 source over the public internet) source + // reachability is the most likely persistent failure mode. + EtcdMirrorConditionSourceReachable = "SourceReachable" + // EtcdMirrorConditionTargetReachable is the target-side analogue. + EtcdMirrorConditionTargetReachable = "TargetReachable" + // EtcdMirrorConditionTargetThrottled is True while the agent is backing + // off from throttling-class errors on Target (etcd ErrTooManyRequests / + // gRPC ResourceExhausted / quota-exhausted), handled distinctly from a + // plain connection drop (a more conservative backoff curve). Kept separate + // from TargetReachable=False because "target is up but rejecting my write + // rate" is a different, actionable signal than "target is unreachable." + EtcdMirrorConditionTargetThrottled = "TargetThrottled" + // EtcdMirrorConditionInitialSyncComplete is True once SyncBase has + // completed at least once against the currently-checkpointed source + // cluster identity. Durable -- does not flip back to False on a + // compaction-forced resync (Compacted covers that). Reset to False if the + // checkpoint is invalidated by a cluster-identity mismatch. + EtcdMirrorConditionInitialSyncComplete = "InitialSyncComplete" + // EtcdMirrorConditionCompacted is True (Reason "ForcedResync") while the + // agent is auto-healing a forced fresh SyncBase after detecting + // source-side compaction raced the watch (a WatchResponse with + // Canceled=true and CompactRevision != 0), rather than crash-looping the + // way etcdctl make-mirror does. Reverts to False once the forced resync + // completes and steady-state watching resumes. + EtcdMirrorConditionCompacted = "Compacted" + // EtcdMirrorConditionReplicationLagExceeded is True when SourceRevision - + // LastAppliedRevision (or, more precisely, the agent's own "time since + // last successful destination apply" loop-liveness measure) has exceeded a + // threshold for a sustained duration D. A deadlocked target-write retry + // loop can keep the process alive and /statusz technically responding, + // but it cannot keep making progress, and this condition is derived from + // progress, not liveness. Threshold and duration D are agent-internal + // constants in v1 (not a spec knob). + EtcdMirrorConditionReplicationLagExceeded = "ReplicationLagExceeded" + // EtcdMirrorConditionDriftDetected is True when the last reconciliation + // pass (if spec.reconciliation.enabled) found a nonzero number of + // orphaned/missing keys. Carries counts in Message. Present only when + // reconciliation is enabled; sticky until the next pass reports clean. + EtcdMirrorConditionDriftDetected = "DriftDetected" + // EtcdMirrorConditionEmptyTargetViolation is True (and terminal, Phase -> + // Failed) when ExpectEmptyPrefix was set and the destination prefix was + // found non-empty before the first-ever InitialSync began. + EtcdMirrorConditionEmptyTargetViolation = "EmptyTargetViolation" +) + +// EtcdMirrorStatus defines the observed state of an EtcdMirror. Progress +// fields are synced periodically (not per-op), so they are a coarse, +// point-in-time mirror of the agent's authoritative local checkpoint, useful +// for kubectl/dashboards, not an audit log. +type EtcdMirrorStatus struct { + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // +optional + Phase EtcdMirrorPhase `json:"phase,omitempty"` + + // LastAppliedRevision is the source etcd revision through which the target + // is known to be caught up, as of the last periodic status sync. The + // agent's local checkpoint file is the authoritative, hot-path copy; this + // is a periodically-synced mirror of it for observability. + // +optional + LastAppliedRevision int64 `json:"lastAppliedRevision,omitempty"` + + // SourceRevision is the source cluster's revision as of the last status + // sync, for computing an approximate replication lag (SourceRevision - + // LastAppliedRevision) at a glance. + // +optional + SourceRevision int64 `json:"sourceRevision,omitempty"` + + // SourceClusterID is the source etcd cluster's cluster ID, as observed on + // the response header of the agent's most recent successful call against + // Source. Surfaced for operator debugging of checkpoint-invalidation + // events -- a changed SourceClusterID across reconciles is the visible + // symptom of "source endpoint now points at a different cluster than the + // checkpoint was taken against." + // +optional + SourceClusterID string `json:"sourceClusterID,omitempty"` + + // InitialSyncKeyCount, InitialSyncStartTime, InitialSyncCompletionTime + // track the base-sync phase for observability. + // +optional + InitialSyncKeyCount int64 `json:"initialSyncKeyCount,omitempty"` + // +optional + InitialSyncStartTime *metav1.Time `json:"initialSyncStartTime,omitempty"` + // +optional + InitialSyncCompletionTime *metav1.Time `json:"initialSyncCompletionTime,omitempty"` + + // ForcedResyncCount counts how many times the agent has auto-healed from a + // source-compaction-raced-the-watch error, OR a cluster-identity-mismatch + // checkpoint invalidation, by re-running SyncBase. Monotonically + // increasing, never reset. + // +optional + ForcedResyncCount int32 `json:"forcedResyncCount,omitempty"` + + // LastReconciliationTime and LastReconciliationDrift record the most + // recent periodic reconciliation pass, when spec.reconciliation.enabled. + // +optional + LastReconciliationTime *metav1.Time `json:"lastReconciliationTime,omitempty"` + // +optional + LastReconciliationDrift *EtcdMirrorDriftInfo `json:"lastReconciliationDrift,omitempty"` + + // LastStatusSyncTime is when status was last refreshed from the agent, so + // staleness (e.g. a wedged agent that stopped responding but hasn't + // crashed) is directly observable. + // +optional + LastStatusSyncTime *metav1.Time `json:"lastStatusSyncTime,omitempty"` + + // LastProgressTime is when the agent last recorded ANY successful + // destination apply (a completed Txn during InitialSync, SyncUpdates, or + // reconciliation). Distinct from LastStatusSyncTime: /statusz can keep + // responding on-time from a wedged loop that has stopped applying writes, + // so this field is what ReplicationLagExceeded is actually derived from. + // +optional + LastProgressTime *metav1.Time `json:"lastProgressTime,omitempty"` + + // AgentPod is the name of the current agent pod (the sole pod of the + // size-1 StatefulSet), for convenient kubectl logs/exec. + // +optional + AgentPod string `json:"agentPod,omitempty"` + + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +type EtcdMirrorDriftInfo struct { + MissingKeys int64 `json:"missingKeys,omitempty"` + OrphanKeys int64 `json:"orphanKeys,omitempty"` + Repaired bool `json:"repaired,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Source",type=string,JSONPath=`.spec.source.prefix` +// +kubebuilder:printcolumn:name="Target",type=string,JSONPath=`.spec.target.prefix` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Revision",type=integer,JSONPath=`.status.lastAppliedRevision` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// EtcdMirror is the Schema for the etcdmirrors API. It describes a continuous, +// one-way key-range sync from a source etcd cluster to a target etcd cluster, +// run as a single supervised pod (a size-1 StatefulSet) in this cluster. +// +// EtcdMirror is deliberately one-way only: clientv3/mirror.Syncer has no +// bidirectional primitive, and etcd itself has no concept of "this write +// originated from a mirror, don't re-mirror it back." Two opposite-direction +// EtcdMirrors pointed at each other create an unbounded write-ping-pong with no +// conflict resolution; see Non-Goals. +type EtcdMirror struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec EtcdMirrorSpec `json:"spec,omitempty"` + Status EtcdMirrorStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// EtcdMirrorList contains a list of EtcdMirror. +type EtcdMirrorList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EtcdMirror `json:"items"` +} diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go index 9a178715..54ef002f 100644 --- a/api/v1alpha1/groupversion_info.go +++ b/api/v1alpha1/groupversion_info.go @@ -40,6 +40,8 @@ func addKnownTypes(s *runtime.Scheme) error { s.AddKnownTypes(GroupVersion, &EtcdCluster{}, &EtcdClusterList{}, + &EtcdMirror{}, + &EtcdMirrorList{}, ) metav1.AddToGroupVersion(s, GroupVersion) return nil diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9db6ca12..7cb29323 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" netx "net" ) @@ -200,6 +200,337 @@ func (in *EtcdClusterStatus) DeepCopy() *EtcdClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirror) DeepCopyInto(out *EtcdMirror) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirror. +func (in *EtcdMirror) DeepCopy() *EtcdMirror { + if in == nil { + return nil + } + out := new(EtcdMirror) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdMirror) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorAuth) DeepCopyInto(out *EtcdMirrorAuth) { + *out = *in + out.SecretRef = in.SecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorAuth. +func (in *EtcdMirrorAuth) DeepCopy() *EtcdMirrorAuth { + if in == nil { + return nil + } + out := new(EtcdMirrorAuth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorBackoffSpec) DeepCopyInto(out *EtcdMirrorBackoffSpec) { + *out = *in + if in.InitialDelay != nil { + in, out := &in.InitialDelay, &out.InitialDelay + *out = new(metav1.Duration) + **out = **in + } + if in.MaxDelay != nil { + in, out := &in.MaxDelay, &out.MaxDelay + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorBackoffSpec. +func (in *EtcdMirrorBackoffSpec) DeepCopy() *EtcdMirrorBackoffSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorBackoffSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorCheckpointSpec) DeepCopyInto(out *EtcdMirrorCheckpointSpec) { + *out = *in + if in.StorageSpec != nil { + in, out := &in.StorageSpec, &out.StorageSpec + *out = new(StorageSpec) + (*in).DeepCopyInto(*out) + } + if in.SyncInterval != nil { + in, out := &in.SyncInterval, &out.SyncInterval + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorCheckpointSpec. +func (in *EtcdMirrorCheckpointSpec) DeepCopy() *EtcdMirrorCheckpointSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorCheckpointSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorDriftInfo) DeepCopyInto(out *EtcdMirrorDriftInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorDriftInfo. +func (in *EtcdMirrorDriftInfo) DeepCopy() *EtcdMirrorDriftInfo { + if in == nil { + return nil + } + out := new(EtcdMirrorDriftInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorEndpoint) DeepCopyInto(out *EtcdMirrorEndpoint) { + *out = *in + if in.EndpointList != nil { + in, out := &in.EndpointList, &out.EndpointList + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ServiceRef != nil { + in, out := &in.ServiceRef, &out.ServiceRef + *out = new(EtcdMirrorServiceRef) + **out = **in + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(EtcdMirrorTLS) + **out = **in + } + if in.Auth != nil { + in, out := &in.Auth, &out.Auth + *out = new(EtcdMirrorAuth) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorEndpoint. +func (in *EtcdMirrorEndpoint) DeepCopy() *EtcdMirrorEndpoint { + if in == nil { + return nil + } + out := new(EtcdMirrorEndpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorList) DeepCopyInto(out *EtcdMirrorList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EtcdMirror, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorList. +func (in *EtcdMirrorList) DeepCopy() *EtcdMirrorList { + if in == nil { + return nil + } + out := new(EtcdMirrorList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdMirrorList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorReconciliationSpec) DeepCopyInto(out *EtcdMirrorReconciliationSpec) { + *out = *in + if in.Interval != nil { + in, out := &in.Interval, &out.Interval + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorReconciliationSpec. +func (in *EtcdMirrorReconciliationSpec) DeepCopy() *EtcdMirrorReconciliationSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorReconciliationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorServiceRef) DeepCopyInto(out *EtcdMirrorServiceRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorServiceRef. +func (in *EtcdMirrorServiceRef) DeepCopy() *EtcdMirrorServiceRef { + if in == nil { + return nil + } + out := new(EtcdMirrorServiceRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorSpec) DeepCopyInto(out *EtcdMirrorSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + in.Target.DeepCopyInto(&out.Target) + in.Sync.DeepCopyInto(&out.Sync) + if in.Checkpoint != nil { + in, out := &in.Checkpoint, &out.Checkpoint + *out = new(EtcdMirrorCheckpointSpec) + (*in).DeepCopyInto(*out) + } + if in.Reconciliation != nil { + in, out := &in.Reconciliation, &out.Reconciliation + *out = new(EtcdMirrorReconciliationSpec) + (*in).DeepCopyInto(*out) + } + if in.PodTemplate != nil { + in, out := &in.PodTemplate, &out.PodTemplate + *out = new(PodTemplate) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorSpec. +func (in *EtcdMirrorSpec) DeepCopy() *EtcdMirrorSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorStatus) DeepCopyInto(out *EtcdMirrorStatus) { + *out = *in + if in.InitialSyncStartTime != nil { + in, out := &in.InitialSyncStartTime, &out.InitialSyncStartTime + *out = (*in).DeepCopy() + } + if in.InitialSyncCompletionTime != nil { + in, out := &in.InitialSyncCompletionTime, &out.InitialSyncCompletionTime + *out = (*in).DeepCopy() + } + if in.LastReconciliationTime != nil { + in, out := &in.LastReconciliationTime, &out.LastReconciliationTime + *out = (*in).DeepCopy() + } + if in.LastReconciliationDrift != nil { + in, out := &in.LastReconciliationDrift, &out.LastReconciliationDrift + *out = new(EtcdMirrorDriftInfo) + **out = **in + } + if in.LastStatusSyncTime != nil { + in, out := &in.LastStatusSyncTime, &out.LastStatusSyncTime + *out = (*in).DeepCopy() + } + if in.LastProgressTime != nil { + in, out := &in.LastProgressTime, &out.LastProgressTime + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorStatus. +func (in *EtcdMirrorStatus) DeepCopy() *EtcdMirrorStatus { + if in == nil { + return nil + } + out := new(EtcdMirrorStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorSyncSpec) DeepCopyInto(out *EtcdMirrorSyncSpec) { + *out = *in + if in.ReconnectBackoff != nil { + in, out := &in.ReconnectBackoff, &out.ReconnectBackoff + *out = new(EtcdMirrorBackoffSpec) + (*in).DeepCopyInto(*out) + } + if in.DialTimeout != nil { + in, out := &in.DialTimeout, &out.DialTimeout + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorSyncSpec. +func (in *EtcdMirrorSyncSpec) DeepCopy() *EtcdMirrorSyncSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorSyncSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorTLS) DeepCopyInto(out *EtcdMirrorTLS) { + *out = *in + out.SecretRef = in.SecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorTLS. +func (in *EtcdMirrorTLS) DeepCopy() *EtcdMirrorTLS { + if in == nil { + return nil + } + out := new(EtcdMirrorTLS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemberStatus) DeepCopyInto(out *MemberStatus) { *out = *in diff --git a/config/crd/bases/operator.etcd.io_etcdmirrors.yaml b/config/crd/bases/operator.etcd.io_etcdmirrors.yaml new file mode 100644 index 00000000..0cf10bed --- /dev/null +++ b/config/crd/bases/operator.etcd.io_etcdmirrors.yaml @@ -0,0 +1,1716 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: etcdmirrors.operator.etcd.io +spec: + group: operator.etcd.io + names: + kind: EtcdMirror + listKind: EtcdMirrorList + plural: etcdmirrors + singular: etcdmirror + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.source.prefix + name: Source + type: string + - jsonPath: .spec.target.prefix + name: Target + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.lastAppliedRevision + name: Revision + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + EtcdMirror is the Schema for the etcdmirrors API. It describes a continuous, + one-way key-range sync from a source etcd cluster to a target etcd cluster, + run as a single supervised pod (a size-1 StatefulSet) in this cluster. + + EtcdMirror is deliberately one-way only: clientv3/mirror.Syncer has no + bidirectional primitive, and etcd itself has no concept of "this write + originated from a mirror, don't re-mirror it back." Two opposite-direction + EtcdMirrors pointed at each other create an unbounded write-ping-pong with no + conflict resolution; see Non-Goals. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EtcdMirrorSpec defines the desired state of an EtcdMirror. + properties: + checkpoint: + description: Checkpoint configures the agent's local durable progress + checkpoint. + properties: + storageSpec: + description: |- + StorageSpec requests persistent storage for the checkpoint file, reusing + EtcdClusterSpec's StorageSpec shape. When nil the controller mounts an + emptyDir instead. StorageSpec should be set for any production mirror; a + small size (e.g. 64Mi) is sufficient. + properties: + accessModes: + type: string + pvcName: + type: string + storageClassName: + type: string + volumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + volumeSizeRequest: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - volumeSizeRequest + type: object + syncInterval: + description: |- + SyncInterval controls how often the agent flushes its in-memory + last-applied revision to the checkpoint file (atomic write: temp file + + fsync + rename). Defaults to 5s when unset. + type: string + type: object + expectEmptyPrefix: + description: |- + ExpectEmptyPrefix, when true, makes the controller verify the target's + effective destination prefix has zero keys before the agent's FIRST-EVER + SyncBase begins (i.e. no valid checkpoint exists yet for this source + cluster identity), refusing to proceed (Phase -> Failed, condition + EmptyTargetViolation) if it is already non-empty. Analogous to + EtcdRestore's assertClusterEmpty guard. + + SCOPE, STATED PLAINLY: this is single-shot, genesis-only protection. It + is checked exactly once, at the moment a fresh (or cluster-identity- + invalidated, see Checkpoint Lifecycle) InitialSync begins. It does NOT + re-arm on a compaction-triggered forced resync later in the CR's life + (that resync reuses the already-established target prefix ownership), + and it does NOT protect against a foreign writer landing keys under the + same target prefix at any point AFTER the first successful sync -- there + is no key-tagging/provenance marker in v1 that would let the agent + distinguish "my own prior output" from "someone else's write" on a + later re-check (see Non-Goals). Operators who need ongoing enforcement + that nothing else writes to the target prefix must arrange that via + etcd RBAC (grant only this mirror's target credential write access to + the prefix) rather than relying on ExpectEmptyPrefix, which is a + bring-up guard, not a standing invariant. + Defaults to false; operators standing up a NEW mirror onto a fresh + prefix should set this true. + type: boolean + paused: + description: |- + Paused, when true, tells the controller to scale the agent StatefulSet to + zero replicas without deleting the CR or its checkpoint PVC. For planned + maintenance windows on either cluster without losing sync position. The + agent's own retry/backoff loop handles transient interruptions on its + own; Paused is for deliberate, operator-initiated stops. + type: boolean + podTemplate: + description: |- + PodTemplate carries scheduling/affinity/labels/annotations for the agent + pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. + properties: + metadata: + description: Metadata is the metadata to add to the pod. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + affinity: + description: Affinity is a group of affinity scheduling rules. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in the + range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + type: object + reconciliation: + description: |- + Reconciliation optionally enables a periodic full diff-and-repair pass + layered on top of the continuous watch-based mirror. + properties: + deleteOrphans: + description: |- + DeleteOrphans, when true, allows reconciliation to DELETE target keys + under the destination prefix that have no corresponding source key. + Defaults to false. + type: boolean + enabled: + description: |- + Enabled toggles the periodic full reconciliation pass. Defaults to + false: it is a diff of the full prefix contents on both sides + (O(keyspace size)), so it is opt-in. + type: boolean + interval: + description: |- + Interval between reconciliation passes. Defaults to 1h when Enabled and + unset. + type: string + type: object + source: + description: |- + Source is the etcd cluster keys are read from. EtcdMirror never writes + back to Source; the agent's source-side client is only ever used for + Get/Watch, never Put/Delete/Txn. + properties: + auth: + description: |- + Auth configures etcd username/password (RBAC) auth for THIS side, + ambient-or-secretRef per the objectstore_creds.go pattern. + properties: + secretRef: + description: |- + SecretRef names a Secret holding "username" and "password" keys. If this + whole Auth block is nil, the agent does not call etcd's Authenticate() + at all. There is no ambient fallback for etcd username/password auth + (unlike objectstore's IRSA/Workload Identity). + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + x-kubernetes-validations: + - message: secretRef.name is required + rule: has(self.secretRef.name) && size(self.secretRef.name) + > 0 + endpointList: + description: |- + EndpointList is a raw set of etcd client-URL host:port (or + scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". + This is the ONLY supported mechanism for a cluster external to this + Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB) -- + there is deliberately no "tunnel" or "port-forward" mode. A + `kubectl port-forward` is a client-attached, ephemeral process tied to a + human's terminal; it does not survive this mirror pod restarting, + rescheduling, or the operator itself restarting, which defeats the + entire point of a supervised, restart-tolerant workload. If you need a + persistent tunnel, terminate it yourself (VPN, Interconnect, NLB) + upstream of this CR and hand EtcdMirror the resulting stable + endpoint(s); network reachability itself is out of scope for this CRD. + + IP-LITERAL ENDPOINTS: if any entry here is a bare IP literal (plausible + for an NLB endpoint with no DNS name), Go's TLS stack requires an IP SAN + (not a DNS SAN) on the peer certificate for verification to succeed. + Either set the corresponding EtcdMirrorTLS.ServerName to a hostname that + IS present as a DNS SAN on the certificate, or ensure the certificate + carries an IP SAN matching the literal. Operators who hit a verification + failure here should fix the SAN/ServerName mismatch, not reach for + InsecureSkipVerify to work around it. + items: + type: string + type: array + prefix: + description: |- + Prefix is the etcd key prefix on THIS side. On Source, only keys under + this prefix are synced; empty means the whole keyspace. On Target, this + is the prefix under which the agent writes mirrored keys after any + Sync.DestPrefix remap is applied. + type: string + serviceRef: + description: |- + ServiceRef points at a Kubernetes Service in this cluster whose DNS name + resolves the etcd client endpoint(s). The rarer same-cluster or + co-located case (e.g. mirroring between two EtcdClusters both running in + this GKE cluster). Namespace defaults to the EtcdMirror's own namespace + when empty. + properties: + name: + description: Name is the Service name. + minLength: 1 + type: string + namespace: + description: Namespace defaults to the EtcdMirror's own namespace + when empty. + type: string + port: + description: |- + Port is the Service port name or number exposing etcd's client API. + Defaults to "client" when empty, matching PodMonitorSpec.Port's + convention elsewhere in this API group. + type: string + required: + - name + type: object + tls: + description: |- + TLS configures the agent's client TLS identity/trust for THIS side. Nil + means the agent dials this side in cleartext. + properties: + insecureSkipVerify: + default: false + description: |- + InsecureSkipVerify disables server certificate verification. Strongly + discouraged, especially for a source reached over the public internet. + Requires InsecureSkipVerifyAcknowledgeRisk to also be set true + (CEL-enforced on this type). The controller additionally emits a + standing Warning event whenever this is true. + type: boolean + insecureSkipVerifyAcknowledgeRisk: + default: false + description: |- + InsecureSkipVerifyAcknowledgeRisk must independently be set true + whenever InsecureSkipVerify is true (CEL-enforced companion field). Its + only purpose is to require a deliberate, separate, reviewable line in + the manifest diff before disabling TLS verification. + type: boolean + secretRef: + description: |- + SecretRef names a Secret (in the EtcdMirror's namespace) holding this + side's TLS material in the standard kubernetes.io/tls-compatible shape: + - ca.crt: PEM CA bundle used to verify the peer's server certificate. + Required for TLS unless InsecureSkipVerify is true. + - tls.crt / tls.key: PEM client certificate + key, for mTLS. Optional -- + omit both for server-auth-only TLS (verify the peer, authenticate to + it via Auth instead, or not at all). + This is the same key layout a cert-manager Certificate's spec.secretName + Secret uses, so a cert-manager Certificate (or `kubectl create secret + tls`, or any other issuance mechanism) can populate this Secret with zero + coupling from this CRD to cert-manager. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + serverName: + description: |- + ServerName overrides the TLS ServerName (SNI) used for verification, for + cases where the dialed endpoint's address doesn't match a SAN on the + certificate (e.g. dialing an NLB IP directly -- see EndpointList's + IP-literal guidance). + type: string + required: + - secretRef + type: object + x-kubernetes-validations: + - message: insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk + to also be true + rule: '!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk' + - message: secretRef.name is required + rule: has(self.secretRef.name) && size(self.secretRef.name) + > 0 + type: object + x-kubernetes-validations: + - message: exactly one of endpointList or serviceRef must be set + rule: (has(self.endpointList) && size(self.endpointList) > 0) != + has(self.serviceRef) + sync: + description: |- + Sync tunes runtime sync behavior (batching, rate limiting, prefix + rewrite, backoff). + properties: + destPrefix: + description: |- + DestPrefix rewrites Source.Prefix to a different prefix on the target, + via an anchored strip-and-reprefix (strings.TrimPrefix(key, + source.Prefix), then destPrefix + rest) -- never make-mirror's naive + first-occurrence strings.Replace. Mutually exclusive with NoDestPrefix. + Empty (and NoDestPrefix false) means the source prefix is reused + verbatim on the target, landing keys under Target.Prefix + the stripped + key remainder. + type: string + dialTimeout: + description: |- + DialTimeout bounds how long the agent waits to establish the initial + client connection to each side. Defaults to 10s when unset. + type: string + maxOpsPerSecond: + description: |- + MaxOpsPerSecond rate-limits the agent's destination write rate (a simple + token bucket over puts+deletes/sec), applied to BOTH InitialSync's Txn + stream and SyncUpdates. Zero (default) means unlimited. + format: int32 + minimum: 0 + type: integer + maxTxnOps: + description: |- + MaxTxnOps bounds how many put/delete operations the agent batches into a + single destination Txn, applied uniformly to BOTH the initial SyncBase + phase and the watch-driven SyncUpdates phase. Defaults to 128 when unset. + format: int32 + minimum: 1 + type: integer + noDestPrefix: + description: |- + NoDestPrefix strips Source.Prefix entirely rather than remapping it, + mirroring make-mirror's --no-dest-prefix. Mutually exclusive with a + non-empty DestPrefix (rejected by EtcdMirrorSpec's XValidation rule). + type: boolean + reconnectBackoff: + description: |- + ReconnectBackoff bounds the retry/backoff loop wrapping every Syncer call + and every destination Txn call. Defaults to exponential backoff from 1s + to 30s, uncapped in attempt count, when unset. + properties: + initialDelay: + type: string + maxDelay: + type: string + type: object + type: object + target: + description: |- + Target is the etcd cluster keys are written into. + + SECURITY PREREQUISITE: Target.Auth's credential (and/or the target + client certificate's associated etcd RBAC role) MUST be an etcd RBAC + user restricted via a range-scoped grant-permission to Target.Prefix (or + Sync.DestPrefix's resulting range) -- never a cluster-admin-equivalent + credential. The agent's client-side rewriteKey/prefix logic is defense + against bugs in the agent's own code; it is NOT a security boundary. A + compromised agent process, a buggy rewriteKey, or an empty/typo'd prefix + with an over-privileged credential can write or delete anywhere in the + target cluster's entire keyspace. This is a deployment prerequisite the + operator (human) must configure in etcd itself (via `etcdctl role + grant-permission --prefix=true readwrite `) before + pointing an EtcdMirror at it; the controller cannot create or enforce + etcd-native RBAC roles itself, but it does check for and surface an + unrestricted-looking target credential where etcd's auth API makes that + detectable (see Safety Guards). + properties: + auth: + description: |- + Auth configures etcd username/password (RBAC) auth for THIS side, + ambient-or-secretRef per the objectstore_creds.go pattern. + properties: + secretRef: + description: |- + SecretRef names a Secret holding "username" and "password" keys. If this + whole Auth block is nil, the agent does not call etcd's Authenticate() + at all. There is no ambient fallback for etcd username/password auth + (unlike objectstore's IRSA/Workload Identity). + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + x-kubernetes-validations: + - message: secretRef.name is required + rule: has(self.secretRef.name) && size(self.secretRef.name) + > 0 + endpointList: + description: |- + EndpointList is a raw set of etcd client-URL host:port (or + scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". + This is the ONLY supported mechanism for a cluster external to this + Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB) -- + there is deliberately no "tunnel" or "port-forward" mode. A + `kubectl port-forward` is a client-attached, ephemeral process tied to a + human's terminal; it does not survive this mirror pod restarting, + rescheduling, or the operator itself restarting, which defeats the + entire point of a supervised, restart-tolerant workload. If you need a + persistent tunnel, terminate it yourself (VPN, Interconnect, NLB) + upstream of this CR and hand EtcdMirror the resulting stable + endpoint(s); network reachability itself is out of scope for this CRD. + + IP-LITERAL ENDPOINTS: if any entry here is a bare IP literal (plausible + for an NLB endpoint with no DNS name), Go's TLS stack requires an IP SAN + (not a DNS SAN) on the peer certificate for verification to succeed. + Either set the corresponding EtcdMirrorTLS.ServerName to a hostname that + IS present as a DNS SAN on the certificate, or ensure the certificate + carries an IP SAN matching the literal. Operators who hit a verification + failure here should fix the SAN/ServerName mismatch, not reach for + InsecureSkipVerify to work around it. + items: + type: string + type: array + prefix: + description: |- + Prefix is the etcd key prefix on THIS side. On Source, only keys under + this prefix are synced; empty means the whole keyspace. On Target, this + is the prefix under which the agent writes mirrored keys after any + Sync.DestPrefix remap is applied. + type: string + serviceRef: + description: |- + ServiceRef points at a Kubernetes Service in this cluster whose DNS name + resolves the etcd client endpoint(s). The rarer same-cluster or + co-located case (e.g. mirroring between two EtcdClusters both running in + this GKE cluster). Namespace defaults to the EtcdMirror's own namespace + when empty. + properties: + name: + description: Name is the Service name. + minLength: 1 + type: string + namespace: + description: Namespace defaults to the EtcdMirror's own namespace + when empty. + type: string + port: + description: |- + Port is the Service port name or number exposing etcd's client API. + Defaults to "client" when empty, matching PodMonitorSpec.Port's + convention elsewhere in this API group. + type: string + required: + - name + type: object + tls: + description: |- + TLS configures the agent's client TLS identity/trust for THIS side. Nil + means the agent dials this side in cleartext. + properties: + insecureSkipVerify: + default: false + description: |- + InsecureSkipVerify disables server certificate verification. Strongly + discouraged, especially for a source reached over the public internet. + Requires InsecureSkipVerifyAcknowledgeRisk to also be set true + (CEL-enforced on this type). The controller additionally emits a + standing Warning event whenever this is true. + type: boolean + insecureSkipVerifyAcknowledgeRisk: + default: false + description: |- + InsecureSkipVerifyAcknowledgeRisk must independently be set true + whenever InsecureSkipVerify is true (CEL-enforced companion field). Its + only purpose is to require a deliberate, separate, reviewable line in + the manifest diff before disabling TLS verification. + type: boolean + secretRef: + description: |- + SecretRef names a Secret (in the EtcdMirror's namespace) holding this + side's TLS material in the standard kubernetes.io/tls-compatible shape: + - ca.crt: PEM CA bundle used to verify the peer's server certificate. + Required for TLS unless InsecureSkipVerify is true. + - tls.crt / tls.key: PEM client certificate + key, for mTLS. Optional -- + omit both for server-auth-only TLS (verify the peer, authenticate to + it via Auth instead, or not at all). + This is the same key layout a cert-manager Certificate's spec.secretName + Secret uses, so a cert-manager Certificate (or `kubectl create secret + tls`, or any other issuance mechanism) can populate this Secret with zero + coupling from this CRD to cert-manager. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + serverName: + description: |- + ServerName overrides the TLS ServerName (SNI) used for verification, for + cases where the dialed endpoint's address doesn't match a SAN on the + certificate (e.g. dialing an NLB IP directly -- see EndpointList's + IP-literal guidance). + type: string + required: + - secretRef + type: object + x-kubernetes-validations: + - message: insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk + to also be true + rule: '!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk' + - message: secretRef.name is required + rule: has(self.secretRef.name) && size(self.secretRef.name) + > 0 + type: object + x-kubernetes-validations: + - message: exactly one of endpointList or serviceRef must be set + rule: (has(self.endpointList) && size(self.endpointList) > 0) != + has(self.serviceRef) + required: + - source + - target + type: object + x-kubernetes-validations: + - message: sync.destPrefix and sync.noDestPrefix are mutually exclusive + rule: '!(has(self.sync) && has(self.sync.destPrefix) && size(self.sync.destPrefix) + > 0 && has(self.sync.noDestPrefix) && self.sync.noDestPrefix)' + status: + description: |- + EtcdMirrorStatus defines the observed state of an EtcdMirror. Progress + fields are synced periodically (not per-op), so they are a coarse, + point-in-time mirror of the agent's authoritative local checkpoint, useful + for kubectl/dashboards, not an audit log. + properties: + agentPod: + description: |- + AgentPod is the name of the current agent pod (the sole pod of the + size-1 StatefulSet), for convenient kubectl logs/exec. + type: string + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + forcedResyncCount: + description: |- + ForcedResyncCount counts how many times the agent has auto-healed from a + source-compaction-raced-the-watch error, OR a cluster-identity-mismatch + checkpoint invalidation, by re-running SyncBase. Monotonically + increasing, never reset. + format: int32 + type: integer + initialSyncCompletionTime: + format: date-time + type: string + initialSyncKeyCount: + description: |- + InitialSyncKeyCount, InitialSyncStartTime, InitialSyncCompletionTime + track the base-sync phase for observability. + format: int64 + type: integer + initialSyncStartTime: + format: date-time + type: string + lastAppliedRevision: + description: |- + LastAppliedRevision is the source etcd revision through which the target + is known to be caught up, as of the last periodic status sync. The + agent's local checkpoint file is the authoritative, hot-path copy; this + is a periodically-synced mirror of it for observability. + format: int64 + type: integer + lastProgressTime: + description: |- + LastProgressTime is when the agent last recorded ANY successful + destination apply (a completed Txn during InitialSync, SyncUpdates, or + reconciliation). Distinct from LastStatusSyncTime: /statusz can keep + responding on-time from a wedged loop that has stopped applying writes, + so this field is what ReplicationLagExceeded is actually derived from. + format: date-time + type: string + lastReconciliationDrift: + properties: + missingKeys: + format: int64 + type: integer + orphanKeys: + format: int64 + type: integer + repaired: + type: boolean + type: object + lastReconciliationTime: + description: |- + LastReconciliationTime and LastReconciliationDrift record the most + recent periodic reconciliation pass, when spec.reconciliation.enabled. + format: date-time + type: string + lastStatusSyncTime: + description: |- + LastStatusSyncTime is when status was last refreshed from the agent, so + staleness (e.g. a wedged agent that stopped responding but hasn't + crashed) is directly observable. + format: date-time + type: string + observedGeneration: + format: int64 + type: integer + phase: + description: |- + EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. + Unlike BackupPhase/RestorePhase, most phases here are NOT terminal -- a + healthy mirror spends its life cycling between Syncing and (briefly, on + transient errors) Degraded; there is no "Completed" state. + type: string + sourceClusterID: + description: |- + SourceClusterID is the source etcd cluster's cluster ID, as observed on + the response header of the agent's most recent successful call against + Source. Surfaced for operator debugging of checkpoint-invalidation + events -- a changed SourceClusterID across reconciles is the visible + symptom of "source endpoint now points at a different cluster than the + checkpoint was taken against." + type: string + sourceRevision: + description: |- + SourceRevision is the source cluster's revision as of the last status + sync, for computing an approximate replication lag (SourceRevision - + LastAppliedRevision) at a glance. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/samples/operator_v1alpha1_etcdmirror.yaml b/config/samples/operator_v1alpha1_etcdmirror.yaml new file mode 100644 index 00000000..94b33159 --- /dev/null +++ b/config/samples/operator_v1alpha1_etcdmirror.yaml @@ -0,0 +1,50 @@ +apiVersion: operator.etcd.io/v1alpha1 +kind: EtcdMirror +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdmirror-sample +spec: + # External RKE1/AWS source reached over a public NLB, over TLS with a + # CA-verified server certificate. + source: + endpointList: + - "https://etcd-rke1.example.com:2379" + prefix: "/registry/" + tls: + secretRef: + name: etcdmirror-sample-source-tls + auth: + secretRef: + name: etcdmirror-sample-source-auth + + # In-cluster EtcdCluster reached via its client Service. + target: + serviceRef: + name: etcdcluster-sample-client + port: client + prefix: "/mirrored/registry/" + tls: + secretRef: + name: etcdmirror-sample-target-tls + + expectEmptyPrefix: true + + sync: + maxTxnOps: 128 + maxOpsPerSecond: 500 + dialTimeout: 10s + reconnectBackoff: + initialDelay: 1s + maxDelay: 30s + + checkpoint: + storageSpec: + volumeSizeRequest: 64Mi + syncInterval: 5s + + reconciliation: + enabled: true + interval: 1h + deleteOrphans: false diff --git a/internal/controller/etcdmirror_cel_test.go b/internal/controller/etcdmirror_cel_test.go new file mode 100644 index 00000000..70fcf476 --- /dev/null +++ b/internal/controller/etcdmirror_cel_test.go @@ -0,0 +1,326 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// validSourceEndpoint and validTargetEndpoint are baseline endpoints that +// satisfy EtcdMirrorEndpoint's own oneOf rule, so each test case only needs +// to perturb the field(s) it's actually exercising. +func validSourceEndpoint() ecv1alpha1.EtcdMirrorEndpoint { + return ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"https://etcd-source.example.com:2379"}, + Prefix: "/registry/", + } +} + +func validTargetEndpoint() ecv1alpha1.EtcdMirrorEndpoint { + return ecv1alpha1.EtcdMirrorEndpoint{ + ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-target-client"}, + Prefix: "/mirrored/", + } +} + +// TestEtcdMirrorSyncPrefixCELValidation drives the sync.destPrefix / +// sync.noDestPrefix mutual-exclusion XValidation rule on EtcdMirrorSpec +// against the real envtest apiserver. +func TestEtcdMirrorSyncPrefixCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + sync ecv1alpha1.EtcdMirrorSyncSpec + wantApply bool + }{ + { + name: "neither destPrefix nor noDestPrefix set accepted", + sync: ecv1alpha1.EtcdMirrorSyncSpec{}, + wantApply: true, + }, + { + name: "destPrefix alone accepted", + sync: ecv1alpha1.EtcdMirrorSyncSpec{DestPrefix: "/other/"}, + wantApply: true, + }, + { + name: "noDestPrefix alone accepted", + sync: ecv1alpha1.EtcdMirrorSyncSpec{NoDestPrefix: true}, + wantApply: true, + }, + { + name: "destPrefix and noDestPrefix together rejected", + sync: ecv1alpha1.EtcdMirrorSyncSpec{DestPrefix: "/other/", NoDestPrefix: true}, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cel-mirror-syncprefix-", + Namespace: "default", + }, + Spec: ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Sync: tt.sync, + }, + } + err := k8sClient.Create(t.Context(), em) + if tt.wantApply { + require.NoError(t, err, "apiserver should accept a valid sync spec") + _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) + } else { + assert.Error(t, err, "apiserver should reject destPrefix+noDestPrefix via CEL") + } + }) + } +} + +// TestEtcdMirrorEndpointOneOfCELValidation drives the endpointList/serviceRef +// exactly-one-of XValidation rule on EtcdMirrorEndpoint, exercised on both +// Source and Target. +func TestEtcdMirrorEndpointOneOfCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + neither := ecv1alpha1.EtcdMirrorEndpoint{Prefix: "/x/"} + both := ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"https://etcd.example.com:2379"}, + ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-client"}, + } + emptyList := ecv1alpha1.EtcdMirrorEndpoint{EndpointList: []string{}} + + tests := []struct { + name string + source ecv1alpha1.EtcdMirrorEndpoint + target ecv1alpha1.EtcdMirrorEndpoint + wantApply bool + }{ + { + name: "valid endpointList source, serviceRef target accepted", + source: validSourceEndpoint(), + target: validTargetEndpoint(), + wantApply: true, + }, + { + name: "source with neither endpointList nor serviceRef rejected", + source: neither, + target: validTargetEndpoint(), + wantApply: false, + }, + { + name: "source with both endpointList and serviceRef rejected", + source: both, + target: validTargetEndpoint(), + wantApply: false, + }, + { + name: "source with empty endpointList and no serviceRef rejected", + source: emptyList, + target: validTargetEndpoint(), + wantApply: false, + }, + { + name: "target with neither endpointList nor serviceRef rejected", + source: validSourceEndpoint(), + target: neither, + wantApply: false, + }, + { + name: "target with both endpointList and serviceRef rejected", + source: validSourceEndpoint(), + target: both, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cel-mirror-endpoint-", + Namespace: "default", + }, + Spec: ecv1alpha1.EtcdMirrorSpec{ + Source: tt.source, + Target: tt.target, + }, + } + err := k8sClient.Create(t.Context(), em) + if tt.wantApply { + require.NoError(t, err, "apiserver should accept exactly-one-of endpointList/serviceRef") + _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) + } else { + assert.Error(t, err, "apiserver should reject endpointList/serviceRef oneOf violation via CEL") + } + }) + } +} + +// TestEtcdMirrorTLSInsecureSkipVerifyCELValidation drives the +// insecureSkipVerify/insecureSkipVerifyAcknowledgeRisk companion-field +// XValidation rule on EtcdMirrorTLS. +func TestEtcdMirrorTLSInsecureSkipVerifyCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + secretRef := corev1.LocalObjectReference{Name: "etcd-mirror-tls"} + + tests := []struct { + name string + tls *ecv1alpha1.EtcdMirrorTLS + wantApply bool + }{ + { + name: "no TLS block accepted", + tls: nil, + wantApply: true, + }, + { + name: "TLS with verification enabled accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: secretRef}, + wantApply: true, + }, + { + name: "insecureSkipVerify with acknowledgement accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: secretRef, + InsecureSkipVerify: true, + InsecureSkipVerifyAcknowledgeRisk: true, + }, + wantApply: true, + }, + { + name: "insecureSkipVerify without acknowledgement rejected", + tls: &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: secretRef, + InsecureSkipVerify: true, + }, + wantApply: false, + }, + { + name: "acknowledgement without insecureSkipVerify accepted (not the risky case)", + tls: &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: secretRef, + InsecureSkipVerifyAcknowledgeRisk: true, + }, + wantApply: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := validSourceEndpoint() + source.TLS = tt.tls + em := &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cel-mirror-tls-", + Namespace: "default", + }, + Spec: ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }, + } + err := k8sClient.Create(t.Context(), em) + if tt.wantApply { + require.NoError(t, err, "apiserver should accept a valid TLS block") + _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) + } else { + assert.Error(t, err, "apiserver should reject insecureSkipVerify without acknowledgement via CEL") + } + }) + } +} + +// TestEtcdMirrorSecretRefNameRequiredCELValidation drives the +// "secretRef.name is required" XValidation rule shared by EtcdMirrorTLS and +// EtcdMirrorAuth, exercised on both the TLS and Auth blocks. +func TestEtcdMirrorSecretRefNameRequiredCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + tls *ecv1alpha1.EtcdMirrorTLS + auth *ecv1alpha1.EtcdMirrorAuth + wantApply bool + }{ + { + name: "TLS secretRef with non-empty name accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: corev1.LocalObjectReference{Name: "etcd-mirror-tls"}}, + wantApply: true, + }, + { + name: "TLS secretRef with empty name rejected", + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: corev1.LocalObjectReference{Name: ""}}, + wantApply: false, + }, + { + name: "Auth secretRef with non-empty name accepted", + auth: &ecv1alpha1.EtcdMirrorAuth{SecretRef: corev1.LocalObjectReference{Name: "etcd-mirror-auth"}}, + wantApply: true, + }, + { + name: "Auth secretRef with empty name rejected", + auth: &ecv1alpha1.EtcdMirrorAuth{SecretRef: corev1.LocalObjectReference{Name: ""}}, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := validSourceEndpoint() + source.TLS = tt.tls + source.Auth = tt.auth + em := &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "cel-mirror-secretref-", + Namespace: "default", + }, + Spec: ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }, + } + err := k8sClient.Create(t.Context(), em) + if tt.wantApply { + require.NoError(t, err, "apiserver should accept a non-empty secretRef.name") + _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) + } else { + assert.Error(t, err, "apiserver should reject an empty secretRef.name via CEL") + } + }) + } +} From 29c646ea589893f345da10d246a88edfa287a19e Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Sat, 4 Jul 2026 17:35:39 -0400 Subject: [PATCH 2/5] feat(etcdmirror): rework API per adversarial cross-cloud review Reshape the EtcdMirror API for the Design-3 engine (checkpoint stored in the target etcd, stateless agent) per the 48-finding cross-cloud review. Deletions: - checkpoint.storageSpec (no PVC; checkpoint lives in the target at a reserved fenced key) - expectEmptyPrefix (superseded by initialSync.mode) - noDestPrefix (redundant; one anchored rewrite formula key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix)) - syncInterval Additions/renames: - spec.mode: Sync|Drain; status.cutover{drainTargetRevision, drainedRevision, verifiedTime, counts, leasedKeyCount} - initialSync.mode: RequireEmpty|Overwrite|OverwriteAndPrune, startRevision - sync: requestTimeout, excludePrefixes, pageKeyLimit, watchBufferBytes; batching flushes only at source-revision boundaries - TLS: secretRef pointerized (nil = system trust roots), caBundleRef - pod resources; configurable reserved checkpoint key - watermark-derived lag/liveness semantics; frozen condition vocabulary (TargetQuotaExhausted, ResyncLoopDetected, CutoverReady, InvariantsHeld, LearnerEndpoint, Prefix/DirectionConflict) and Event Reason constants as API contract - status: source/target versions and cluster IDs, leaseBackedKeyCount, always-on source/target key counts, initialSyncTotalKeyCount - CEL: scheme-vs-TLS rules, prefix/rewrite immutability - docs/etcdmirror.md (fidelity caveats, one-way contract) and regenerated API reference Co-Authored-By: Claude Fable 5 Signed-off-by: Xavier Lange --- api/v1alpha1/etcdmirror_types.go | 912 +++++++++++++----- api/v1alpha1/zz_generated.deepcopy.go | 122 ++- .../bases/operator.etcd.io_etcdmirrors.yaml | 879 +++++++++++------ .../samples/operator_v1alpha1_etcdmirror.yaml | 56 +- docs/api-references/docs.md | 460 +++++++++ docs/etcdmirror.md | 141 +++ internal/controller/etcdmirror_cel_test.go | 679 ++++++++++--- 7 files changed, 2574 insertions(+), 675 deletions(-) create mode 100644 docs/etcdmirror.md diff --git a/api/v1alpha1/etcdmirror_types.go b/api/v1alpha1/etcdmirror_types.go index 2bf64789..ff4b4d7a 100644 --- a/api/v1alpha1/etcdmirror_types.go +++ b/api/v1alpha1/etcdmirror_types.go @@ -18,147 +18,242 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// EtcdMirrorMode selects the mirror's operating mode. +type EtcdMirrorMode string + +const ( + // EtcdMirrorModeSync is normal continuous replication. + EtcdMirrorModeSync EtcdMirrorMode = "Sync" + // EtcdMirrorModeDrain prepares for cutover: the agent records the source + // revision observed when Drain is requested (status.cutover.drainTargetRevision), + // keeps replicating until the checkpoint watermark reaches it, runs a + // verification pass (per-side key counts, lease-backed key count), then + // sets the CutoverReady condition and flips the fence key's role to + // Primary so any straggler apply fails its mod-revision compare loudly. + // Runbook: quiesce source writers -> set mode=Drain -> + // `kubectl wait --for=condition=CutoverReady etcdmirror/` -> + // purge/re-lease lease-backed keys -> repoint clients -> delete the CR. + EtcdMirrorModeDrain EtcdMirrorMode = "Drain" +) + +// EtcdMirrorInitialSyncMode governs how the agent treats pre-existing keys +// under the effective destination prefix at genesis (first-ever sync, or a +// checkpoint invalidated by a cluster-identity mismatch). +type EtcdMirrorInitialSyncMode string + +const ( + // EtcdMirrorInitialSyncRequireEmpty refuses to start if the destination + // prefix already holds any key (Phase -> Failed, condition + // EmptyTargetViolation). The reserved checkpoint key is excluded by exact + // match. + // + // RE-ARM CONTRACT: a source OR target cluster-ID mismatch invalidates + // the checkpoint, forces genesis, and RE-ARMS this check. An ordinary + // forced resync (Compacted) does NOT re-check RequireEmpty: the decoded, + // ownership-validated fence proves the destination data is this link's + // own. + EtcdMirrorInitialSyncRequireEmpty EtcdMirrorInitialSyncMode = "RequireEmpty" + // EtcdMirrorInitialSyncOverwrite scans and writes over whatever is there. + // Keys present on the target but absent on the source are left alone. + EtcdMirrorInitialSyncOverwrite EtcdMirrorInitialSyncMode = "Overwrite" + // EtcdMirrorInitialSyncOverwriteAndPrune is Overwrite plus one mandatory + // orphan-prune pass after the scan: target keys under the destination + // prefix with no source counterpart are deleted. This makes reversal onto + // a previously-populated prefix (failback) a first-class correct + // operation instead of silently resurrecting deleted keys. + EtcdMirrorInitialSyncOverwriteAndPrune EtcdMirrorInitialSyncMode = "OverwriteAndPrune" +) + // EtcdMirrorSpec defines the desired state of an EtcdMirror. // -// +kubebuilder:validation:XValidation:rule="!(has(self.sync) && has(self.sync.destPrefix) && size(self.sync.destPrefix) > 0 && has(self.sync.noDestPrefix) && self.sync.noDestPrefix)",message="sync.destPrefix and sync.noDestPrefix are mutually exclusive" +// Range-defining and rewrite fields are immutable (CEL transition rules +// below): source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes +// and checkpoint.key. Changing what range is mirrored, or where it lands, +// mid-life silently diverges: a restarted agent resumes from its checkpoint +// without a scan, so removing an exclusion never backfills pre-existing keys +// and adding one strands already-mirrored keys as permanent orphans — all +// with every condition green. Endpoints stay mutable — rotating an NLB DNS +// name or adding a member to the same cluster is routine; pointing at a +// different cluster is caught at runtime by the checkpoint's dual-cluster-ID +// binding, not by spec validation. +// +// The transition rules compare VALUES, presence-normalized: for these +// fields the empty value and an absent field are semantically identical +// (prefix "" = whole keyspace, destPrefix "" = strip the source prefix), and +// Go typed clients drop explicit "" through omitempty — presence-based rules +// would reject every typed-client update of a CR created with an explicit +// empty string. +// +// +kubebuilder:validation:XValidation:rule="(has(self.source.prefix) ? self.source.prefix : \"\") == (has(oldSelf.source.prefix) ? oldSelf.source.prefix : \"\")",message="source.prefix is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.target.prefix) ? self.target.prefix : \"\") == (has(oldSelf.target.prefix) ? oldSelf.target.prefix : \"\")",message="target.prefix is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.sync) && has(self.sync.destPrefix) ? self.sync.destPrefix : \"\") == (has(oldSelf.sync) && has(oldSelf.sync.destPrefix) ? oldSelf.sync.destPrefix : \"\")",message="sync.destPrefix is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.sync) && has(self.sync.excludePrefixes) ? self.sync.excludePrefixes : []) == (has(oldSelf.sync) && has(oldSelf.sync.excludePrefixes) ? oldSelf.sync.excludePrefixes : [])",message="sync.excludePrefixes is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.checkpoint) && has(self.checkpoint.key) ? self.checkpoint.key : \"\") == (has(oldSelf.checkpoint) && has(oldSelf.checkpoint.key) ? oldSelf.checkpoint.key : \"\")",message="checkpoint.key is immutable" +// +kubebuilder:validation:XValidation:rule="!has(self.checkpoint) || !has(self.checkpoint.key) || self.checkpoint.key == \"\" || self.checkpoint.key.startsWith((has(self.target.prefix) ? self.target.prefix : \"\") + (has(self.sync) && has(self.sync.destPrefix) ? self.sync.destPrefix : \"\"))",message="checkpoint.key must live under the effective destination prefix (target.prefix + sync.destPrefix)" type EtcdMirrorSpec struct { + // Mode selects continuous replication (Sync, the default) or a cutover + // drain (Drain). See EtcdMirrorModeDrain for the cutover contract. + // +optional + // +kubebuilder:validation:Enum=Sync;Drain + // +kubebuilder:default=Sync + Mode EtcdMirrorMode `json:"mode,omitempty"` + // Source is the etcd cluster keys are read from. EtcdMirror never writes // back to Source; the agent's source-side client is only ever used for // Get/Watch, never Put/Delete/Txn. + // + // VERSION FLOOR: source etcd must be >= 3.4 (probed via maintenance + // Status() at connect; below the floor the mirror goes Failed with reason + // UnsupportedVersion). >= 3.4.25 / 3.5.8 is the recommended floor: below + // it, watch progress notifications are unreliable and the agent cannot + // trust the watermark machinery that drives lag, the checkpoint, and the + // Drain gate. Source EtcdMirrorEndpoint `json:"source"` // Target is the etcd cluster keys are written into. // - // SECURITY PREREQUISITE: Target.Auth's credential (and/or the target - // client certificate's associated etcd RBAC role) MUST be an etcd RBAC - // user restricted via a range-scoped grant-permission to Target.Prefix (or - // Sync.DestPrefix's resulting range) -- never a cluster-admin-equivalent - // credential. The agent's client-side rewriteKey/prefix logic is defense - // against bugs in the agent's own code; it is NOT a security boundary. A - // compromised agent process, a buggy rewriteKey, or an empty/typo'd prefix - // with an over-privileged credential can write or delete anywhere in the - // target cluster's entire keyspace. This is a deployment prerequisite the - // operator (human) must configure in etcd itself (via `etcdctl role - // grant-permission --prefix=true readwrite `) before - // pointing an EtcdMirror at it; the controller cannot create or enforce - // etcd-native RBAC roles itself, but it does check for and surface an - // unrestricted-looking target credential where etcd's auth API makes that - // detectable (see Safety Guards). + // SECURITY PREREQUISITE: Target's credential (etcd RBAC user and/or the + // client certificate's role) MUST be range-scoped to the effective + // destination prefix — never a cluster-admin-equivalent credential. The + // agent's client-side rewrite logic is defense against bugs in its own + // code, NOT a security boundary. The grant must also cover the reserved + // checkpoint key (see Checkpoint). Configure via `etcdctl role + // grant-permission --prefix=true readwrite ` before + // pointing an EtcdMirror at the cluster. + // + // The target must run with auto-compaction enabled: forced-resync churn + // and prune passes march an uncompacted target toward its storage quota + // (2GiB by default), which surfaces as TargetQuotaExhausted. Target EtcdMirrorEndpoint `json:"target"` - // ExpectEmptyPrefix, when true, makes the controller verify the target's - // effective destination prefix has zero keys before the agent's FIRST-EVER - // SyncBase begins (i.e. no valid checkpoint exists yet for this source - // cluster identity), refusing to proceed (Phase -> Failed, condition - // EmptyTargetViolation) if it is already non-empty. Analogous to - // EtcdRestore's assertClusterEmpty guard. - // - // SCOPE, STATED PLAINLY: this is single-shot, genesis-only protection. It - // is checked exactly once, at the moment a fresh (or cluster-identity- - // invalidated, see Checkpoint Lifecycle) InitialSync begins. It does NOT - // re-arm on a compaction-triggered forced resync later in the CR's life - // (that resync reuses the already-established target prefix ownership), - // and it does NOT protect against a foreign writer landing keys under the - // same target prefix at any point AFTER the first successful sync -- there - // is no key-tagging/provenance marker in v1 that would let the agent - // distinguish "my own prior output" from "someone else's write" on a - // later re-check (see Non-Goals). Operators who need ongoing enforcement - // that nothing else writes to the target prefix must arrange that via - // etcd RBAC (grant only this mirror's target credential write access to - // the prefix) rather than relying on ExpectEmptyPrefix, which is a - // bring-up guard, not a standing invariant. - // Defaults to false; operators standing up a NEW mirror onto a fresh - // prefix should set this true. - // +optional - ExpectEmptyPrefix bool `json:"expectEmptyPrefix,omitempty"` - - // Sync tunes runtime sync behavior (batching, rate limiting, prefix - // rewrite, backoff). + // InitialSync governs genesis behavior: how pre-existing destination keys + // are treated (Mode) and optionally where replication starts + // (StartRevision). + // +optional + InitialSync *EtcdMirrorInitialSyncSpec `json:"initialSync,omitempty"` + + // Sync tunes runtime sync behavior (batching, paging, rate limiting, + // prefix rewrite, timeouts, backoff). // +optional Sync EtcdMirrorSyncSpec `json:"sync,omitempty"` - // Checkpoint configures the agent's local durable progress checkpoint. + // Checkpoint configures the reserved checkpoint/fence key on the target. // +optional Checkpoint *EtcdMirrorCheckpointSpec `json:"checkpoint,omitempty"` // Reconciliation optionally enables a periodic full diff-and-repair pass - // layered on top of the continuous watch-based mirror. + // layered on top of the continuous watch-based mirror. Independent of + // this setting, one reconciliation-with-delete pass always runs after any + // forced resync (mark-and-sweep), and as the OverwriteAndPrune genesis + // pass and the Drain verification pass. // +optional Reconciliation *EtcdMirrorReconciliationSpec `json:"reconciliation,omitempty"` - // PodTemplate carries scheduling/affinity/labels/annotations for the agent - // pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. + // PodTemplate carries scheduling/affinity/labels/annotations for the + // agent pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. // +optional PodTemplate *PodTemplate `json:"podTemplate,omitempty"` - // Paused, when true, tells the controller to scale the agent StatefulSet to - // zero replicas without deleting the CR or its checkpoint PVC. For planned - // maintenance windows on either cluster without losing sync position. The - // agent's own retry/backoff loop handles transient interruptions on its - // own; Paused is for deliberate, operator-initiated stops. + // Resources are the agent container's compute resources. The agent's + // memory model is bounded by Sync.PageBytes (single in-flight scan page, + // no unbounded read-ahead); size limits accordingly. + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + + // Paused, when true, scales the agent Deployment to zero without deleting + // the CR or its checkpoint. The checkpoint lives in the target etcd, so + // resume picks up from the last fenced watermark. NOTE: pausing longer + // than the source's compaction retention guarantees a full forced resync + // on resume — there is no free lunch past the retention window. // +optional Paused bool `json:"paused,omitempty"` } +// EtcdMirrorInitialSyncSpec governs the genesis scan. +// +// +kubebuilder:validation:XValidation:rule="!(has(self.startRevision) && self.startRevision > 0 && (!has(self.mode) || self.mode == 'RequireEmpty'))",message="initialSync.startRevision requires initialSync.mode Overwrite or OverwriteAndPrune (a seeded target is not empty)" +type EtcdMirrorInitialSyncSpec struct { + // Mode governs pre-existing destination keys at genesis. Defaults to + // RequireEmpty (refuse a non-empty destination prefix). + // +optional + // +kubebuilder:validation:Enum=RequireEmpty;Overwrite;OverwriteAndPrune + // +kubebuilder:default=RequireEmpty + Mode EtcdMirrorInitialSyncMode `json:"mode,omitempty"` + + // StartRevision, when > 0, skips the genesis scan entirely and starts + // watching from StartRevision+1. For fidelity-preserving seeds: restore + // the target from a source snapshot (`etcdutl snapshot restore + // --bump-revision --mark-compacted`), then mirror only the delta. + // Requires Mode Overwrite or OverwriteAndPrune (CEL-enforced). + // +optional + // +kubebuilder:validation:Minimum=0 + StartRevision int64 `json:"startRevision,omitempty"` +} + // EtcdMirrorEndpoint describes how to reach, authenticate to, and scope one // side (source or target) of a mirror. Both sides need an identical shape // (address resolution + prefix + TLS + auth), so one type serves both roles, // the same way BackupDestination is reused verbatim between EtcdBackup and // EtcdRestore rather than forked into near-duplicate per-role types. // -// Exactly one of EndpointList or ServiceRef must be set. +// Exactly one of EndpointList or ServiceRef must be set. An empty +// endpointList ([]) is treated as unset, per Kubernetes list conventions — +// so `endpointList: []` alongside a serviceRef is accepted. +// +// Endpoint scheme and the TLS block must agree (CEL-enforced both ways): +// http:// endpoints with a tls block would silently drop TLS at dial time; +// https:// endpoints without one would dial with undeclared system-roots +// TLS. The agent derives the dial scheme from the presence of the tls block, +// so the declared contract is true by construction. // // +kubebuilder:validation:XValidation:rule="(has(self.endpointList) && size(self.endpointList) > 0) != has(self.serviceRef)",message="exactly one of endpointList or serviceRef must be set" +// +kubebuilder:validation:XValidation:rule="!(has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, e.startsWith('http://')))",message="http:// endpoints conflict with a tls block: use https:// endpoints or remove tls" +// +kubebuilder:validation:XValidation:rule="!(!has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, e.startsWith('https://')))",message="https:// endpoints require a tls block (an empty tls block selects server-auth TLS against system trust roots)" type EtcdMirrorEndpoint struct { // EndpointList is a raw set of etcd client-URL host:port (or // scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". // This is the ONLY supported mechanism for a cluster external to this - // Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB) -- - // there is deliberately no "tunnel" or "port-forward" mode. A - // `kubectl port-forward` is a client-attached, ephemeral process tied to a - // human's terminal; it does not survive this mirror pod restarting, - // rescheduling, or the operator itself restarting, which defeats the - // entire point of a supervised, restart-tolerant workload. If you need a - // persistent tunnel, terminate it yourself (VPN, Interconnect, NLB) - // upstream of this CR and hand EtcdMirror the resulting stable - // endpoint(s); network reachability itself is out of scope for this CRD. + // Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB); + // there is deliberately no tunnel/port-forward mode — terminate any + // tunnel upstream and hand EtcdMirror the resulting stable endpoint(s). + // + // Prefer listing per-member endpoints over a single load-balancer VIP: a + // TCP-health-checked VIP cannot see etcd quorum, and the client's own + // balancer handles per-member failover. // - // IP-LITERAL ENDPOINTS: if any entry here is a bare IP literal (plausible - // for an NLB endpoint with no DNS name), Go's TLS stack requires an IP SAN - // (not a DNS SAN) on the peer certificate for verification to succeed. - // Either set the corresponding EtcdMirrorTLS.ServerName to a hostname that - // IS present as a DNS SAN on the certificate, or ensure the certificate - // carries an IP SAN matching the literal. Operators who hit a verification - // failure here should fix the SAN/ServerName mismatch, not reach for - // InsecureSkipVerify to work around it. + // IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN) + // to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a + // hostname present as a DNS SAN on the certificate, or ensure the + // certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch; + // do not reach for InsecureSkipVerify. // +optional EndpointList []string `json:"endpointList,omitempty"` - // ServiceRef points at a Kubernetes Service in this cluster whose DNS name - // resolves the etcd client endpoint(s). The rarer same-cluster or - // co-located case (e.g. mirroring between two EtcdClusters both running in - // this GKE cluster). Namespace defaults to the EtcdMirror's own namespace - // when empty. + // ServiceRef points at a Kubernetes Service in this cluster whose DNS + // name resolves the etcd client endpoint(s), for the same-cluster case. + // Namespace defaults to the EtcdMirror's own namespace when empty. // +optional ServiceRef *EtcdMirrorServiceRef `json:"serviceRef,omitempty"` // Prefix is the etcd key prefix on THIS side. On Source, only keys under // this prefix are synced; empty means the whole keyspace. On Target, this - // is the prefix under which the agent writes mirrored keys after any - // Sync.DestPrefix remap is applied. + // is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's + // rewrite formula). Immutable after creation. // +optional Prefix string `json:"prefix,omitempty"` - // TLS configures the agent's client TLS identity/trust for THIS side. Nil - // means the agent dials this side in cleartext. + // TLS configures the agent's client TLS for THIS side. Nil means the + // agent dials this side in cleartext (and https:// endpoints are + // CEL-rejected). An empty block means server-auth TLS verified against + // the system trust roots. // +optional TLS *EtcdMirrorTLS `json:"tls,omitempty"` - // Auth configures etcd username/password (RBAC) auth for THIS side, - // ambient-or-secretRef per the objectstore_creds.go pattern. + // Auth configures etcd username/password (RBAC) auth for THIS side. // +optional Auth *EtcdMirrorAuth `json:"auth,omitempty"` } @@ -180,32 +275,44 @@ type EtcdMirrorServiceRef struct { Port string `json:"port,omitempty"` } -// EtcdMirrorTLS configures the mirror agent's client TLS identity for one side -// of a mirror. Plain secretRef, not a reuse of EtcdClusterTLS/TLSSurface -- -// the mirror agent is always a client to clusters it does not own, so the -// issuer-selection machinery in TLSSurface/ProviderCertManagerConfig doesn't apply. +// EtcdMirrorTLS configures the mirror agent's client TLS for one side of a +// mirror. Plain secretRef, not a reuse of EtcdClusterTLS/TLSSurface — the +// agent is always a client to clusters it does not own, so issuer-selection +// machinery doesn't apply. +// +// ROTATION CONTRACT: the agent re-reads TLS material from the mounted Secret +// on every handshake (transport.TLSInfo file paths, not a one-shot +// tls.Config); certificate rotation requires no pod restart. // // +kubebuilder:validation:XValidation:rule="!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk",message="insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk to also be true" -// +kubebuilder:validation:XValidation:rule="has(self.secretRef.name) && size(self.secretRef.name) > 0",message="secretRef.name is required" +// +kubebuilder:validation:XValidation:rule="!has(self.secretRef) || (has(self.secretRef.name) && size(self.secretRef.name) > 0)",message="secretRef.name must be non-empty when secretRef is set" type EtcdMirrorTLS struct { // SecretRef names a Secret (in the EtcdMirror's namespace) holding this // side's TLS material in the standard kubernetes.io/tls-compatible shape: - // - ca.crt: PEM CA bundle used to verify the peer's server certificate. - // Required for TLS unless InsecureSkipVerify is true. - // - tls.crt / tls.key: PEM client certificate + key, for mTLS. Optional -- - // omit both for server-auth-only TLS (verify the peer, authenticate to - // it via Auth instead, or not at all). - // This is the same key layout a cert-manager Certificate's spec.secretName - // Secret uses, so a cert-manager Certificate (or `kubectl create secret - // tls`, or any other issuance mechanism) can populate this Secret with zero - // coupling from this CRD to cert-manager. - SecretRef corev1.LocalObjectReference `json:"secretRef"` + // - ca.crt: PEM CA bundle used to verify the peer's server certificate + // (unless CABundleRef overrides it, or InsecureSkipVerify is true). + // - tls.crt / tls.key: PEM client certificate + key, for mTLS. + // Optional — omit both for server-auth-only TLS. + // Nil means no client identity and verification against the system trust + // roots (the etcdctl default). Note a source running with + // --client-cert-auth (the RKE1 default) rejects certless clients at the + // handshake regardless of etcd RBAC auth; server-auth-only + Auth is not + // viable against such a source. + // +optional + SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"` + + // CABundleRef optionally sources the trust anchors from a separate + // Secret or ConfigMap key, decoupling trust from the identity Secret + // (Gateway API caCertificateRefs precedent). Takes precedence over + // SecretRef's ca.crt. + // +optional + CABundleRef *EtcdMirrorCABundleRef `json:"caBundleRef,omitempty"` // InsecureSkipVerify disables server certificate verification. Strongly // discouraged, especially for a source reached over the public internet. // Requires InsecureSkipVerifyAcknowledgeRisk to also be set true - // (CEL-enforced on this type). The controller additionally emits a - // standing Warning event whenever this is true. + // (CEL-enforced). The controller additionally emits a standing Warning + // event whenever this is true. // +optional // +kubebuilder:default=false InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` @@ -218,69 +325,152 @@ type EtcdMirrorTLS struct { // +kubebuilder:default=false InsecureSkipVerifyAcknowledgeRisk bool `json:"insecureSkipVerifyAcknowledgeRisk,omitempty"` - // ServerName overrides the TLS ServerName (SNI) used for verification, for - // cases where the dialed endpoint's address doesn't match a SAN on the - // certificate (e.g. dialing an NLB IP directly -- see EndpointList's - // IP-literal guidance). + // ServerName overrides the TLS ServerName (SNI) used for verification, + // for cases where the dialed address doesn't match a SAN on the + // certificate (e.g. dialing an NLB IP directly). Applies to EVERY + // endpoint in the list, so mixing endpoints with different certificates + // behind one ServerName will fail verification on the mismatched ones. // +optional ServerName string `json:"serverName,omitempty"` } +// EtcdMirrorCABundleRef points at one key of a Secret or ConfigMap holding a +// PEM CA bundle. +type EtcdMirrorCABundleRef struct { + // Kind is Secret or ConfigMap. Defaults to ConfigMap. + // +optional + // +kubebuilder:validation:Enum=Secret;ConfigMap + // +kubebuilder:default=ConfigMap + Kind string `json:"kind,omitempty"` + + // Name of the Secret or ConfigMap, in the EtcdMirror's namespace. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Key within the object. Defaults to "ca.crt". + // +optional + Key string `json:"key,omitempty"` +} + // EtcdMirrorAuth configures etcd RBAC username/password auth for one side. // // +kubebuilder:validation:XValidation:rule="has(self.secretRef.name) && size(self.secretRef.name) > 0",message="secretRef.name is required" type EtcdMirrorAuth struct { - // SecretRef names a Secret holding "username" and "password" keys. If this - // whole Auth block is nil, the agent does not call etcd's Authenticate() - // at all. There is no ambient fallback for etcd username/password auth - // (unlike objectstore's IRSA/Workload Identity). + // SecretRef names a Secret holding "username" and "password" keys. If + // this whole Auth block is nil, the agent does not call etcd's + // Authenticate() at all; the pinned v3 client transparently re-auths on + // token expiry. PRECEDENCE: when both a client certificate and Auth are + // supplied, etcd uses the token identity, not the certificate CN — the + // Auth user must hold the range-scoped role. SecretRef corev1.LocalObjectReference `json:"secretRef"` } -// EtcdMirrorSyncSpec tunes the mirror's runtime sync behavior. Defaults are -// chosen to match etcdctl make-mirror's own defaults where one exists (e.g. -// MaxTxnOps=128). +// EtcdMirrorSyncSpec tunes the mirror's runtime sync behavior. +// +// KEY REWRITE — one formula, no other composition: +// +// key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) +// +// (anchored strip-and-reprefix; never a substring replace). +// +// BATCHING INVARIANT: target Txns flush ONLY at source-revision boundaries — +// a source revision's events are never split across Txns, whole revisions +// are coalesced up to the MaxTxnOps/TxnFlushBytes watermarks, and one op +// slot in MaxTxnOps is always reserved for the checkpoint write that rides +// in the same Txn. A single source revision larger than MaxTxnOps is applied +// as one oversized Txn (provision the target's --max-txn-ops accordingly) +// with the checkpoint held until it lands. +// +// RETENTION PREREQUISITE: the source's compaction retention window must +// exceed the worst-case initial scan + throttled drain time, approximately +// sourceKeyCount / min(effective scan rate, MaxOpsPerSecond). If it does +// not, genesis (and every forced resync) loses the race with compaction and +// the mirror livelocks (surfaced via the resync-loop detector). type EtcdMirrorSyncSpec struct { - // DestPrefix rewrites Source.Prefix to a different prefix on the target, - // via an anchored strip-and-reprefix (strings.TrimPrefix(key, - // source.Prefix), then destPrefix + rest) -- never make-mirror's naive - // first-occurrence strings.Replace. Mutually exclusive with NoDestPrefix. - // Empty (and NoDestPrefix false) means the source prefix is reused - // verbatim on the target, landing keys under Target.Prefix + the stripped - // key remainder. + // DestPrefix is the middle term of the rewrite formula above. Default "" + // means the source prefix is stripped and key remainders land directly + // under target.prefix. Immutable after creation. // +optional DestPrefix string `json:"destPrefix,omitempty"` - // NoDestPrefix strips Source.Prefix entirely rather than remapping it, - // mirroring make-mirror's --no-dest-prefix. Mutually exclusive with a - // non-empty DestPrefix (rejected by EtcdMirrorSpec's XValidation rule). + // ExcludePrefixes lists source key prefixes (full source-side keys, e.g. + // "/registry/events/") skipped entirely: not scanned, not watched, not + // counted, not pruned. Use to drop high-churn low-value ranges and cut + // WAN cost, or to skip lease-backed ranges that don't survive mirroring. + // Nested/duplicate entries are normalized by the agent (an entry covered + // by another is dropped). Immutable after creation (it defines the + // mirrored range): removing an exclusion would require a backfill scan + // the checkpoint-resume path never runs, and adding one would strand + // already-mirrored keys as permanent orphans — change it via + // delete-and-recreate with an appropriate initialSync.mode instead. + // +optional + // +kubebuilder:validation:MaxItems=64 + ExcludePrefixes []string `json:"excludePrefixes,omitempty"` + + // MaxTxnOps bounds how many operations the agent batches into a single + // target Txn, including the reserved checkpoint-write slot. Must not + // exceed the target's --max-txn-ops (etcd default 128). Defaults to 128. + // +optional + // +kubebuilder:validation:Minimum=2 + MaxTxnOps int32 `json:"maxTxnOps,omitempty"` + + // TxnFlushBytes is the byte watermark at which a batch is flushed (at the + // next source-revision boundary). Keep well under etcd's request size + // limits: a Txn over ~1.5MiB is rejected by the server and one over 2MiB + // by the client send cap — both classified permanent errors, not + // throttling. Defaults to 1Mi. // +optional - NoDestPrefix bool `json:"noDestPrefix,omitempty"` + TxnFlushBytes *resource.Quantity `json:"txnFlushBytes,omitempty"` - // MaxTxnOps bounds how many put/delete operations the agent batches into a - // single destination Txn, applied uniformly to BOTH the initial SyncBase - // phase and the watch-driven SyncUpdates phase. Defaults to 128 when unset. + // PageKeyLimit bounds keys per source scan page during InitialSync and + // reconciliation. The scan is pull-based, one page in flight — no + // read-ahead — so this and PageBytes bound agent memory. Defaults to 512. // +optional // +kubebuilder:validation:Minimum=1 - MaxTxnOps int32 `json:"maxTxnOps,omitempty"` + PageKeyLimit int32 `json:"pageKeyLimit,omitempty"` + + // PageBytes bounds bytes per source scan page. Defaults to 1Mi. + // +optional + PageBytes *resource.Quantity `json:"pageBytes,omitempty"` + + // WatchBufferBytes bounds the memory used to buffer watch events + // observed from R0 while the genesis scan runs (the reflector replay + // buffer). On overflow the agent cancels the source watch and restarts + // the scan from a fresh R0 (see the InitialSyncCompactionRaced event) — + // a bounded retry instead of unbounded growth when source churn outruns + // scan+apply throughput. Defaults to 16Mi (must stay in lockstep with + // pkg/mirroragent's DefaultWatchBufferBytes). + // +optional + WatchBufferBytes *resource.Quantity `json:"watchBufferBytes,omitempty"` - // MaxOpsPerSecond rate-limits the agent's destination write rate (a simple - // token bucket over puts+deletes/sec), applied to BOTH InitialSync's Txn - // stream and SyncUpdates. Zero (default) means unlimited. + // MaxOpsPerSecond rate-limits the agent's target write rate (a token + // bucket over puts+deletes/sec), applied to both the genesis scan and + // watch-driven applies. Zero (default) means unlimited. Mind the + // retention prerequisite above when throttling. // +optional // +kubebuilder:validation:Minimum=0 MaxOpsPerSecond int32 `json:"maxOpsPerSecond,omitempty"` - // ReconnectBackoff bounds the retry/backoff loop wrapping every Syncer call - // and every destination Txn call. Defaults to exponential backoff from 1s - // to 30s, uncapped in attempt count, when unset. + // RequestTimeout is the per-RPC context deadline applied to every unary + // call on both sides (watches excluded — they are long-lived by design + // and covered by progress-notification liveness instead). Without it a + // blackholed call through an NLB never errors and backoff never engages. + // Defaults to 30s. // +optional - ReconnectBackoff *EtcdMirrorBackoffSpec `json:"reconnectBackoff,omitempty"` + RequestTimeout *metav1.Duration `json:"requestTimeout,omitempty"` - // DialTimeout bounds how long the agent waits to establish the initial - // client connection to each side. Defaults to 10s when unset. + // DialTimeout bounds establishing the initial client connection to each + // side. Defaults to 10s. // +optional DialTimeout *metav1.Duration `json:"dialTimeout,omitempty"` + + // ReconnectBackoff bounds the retry/backoff loop wrapping connection-class + // errors. Throttling-class errors (target rate rejection) use a more + // conservative curve derived from the same bounds; quota exhaustion + // (TargetQuotaExhausted) and permanent errors are never retried through + // this loop. Defaults to exponential backoff from 1s to 30s. + // +optional + ReconnectBackoff *EtcdMirrorBackoffSpec `json:"reconnectBackoff,omitempty"` } type EtcdMirrorBackoffSpec struct { @@ -290,144 +480,256 @@ type EtcdMirrorBackoffSpec struct { MaxDelay *metav1.Duration `json:"maxDelay,omitempty"` } -// EtcdMirrorCheckpointSpec configures the agent's local PVC-backed checkpoint. +// EtcdMirrorCheckpointSpec configures the reserved checkpoint/fence key the +// agent maintains IN THE TARGET etcd. The checkpoint (the source-revision +// watermark plus {linkUID, epoch, role}) is written in the SAME Txn as every +// applied batch and fenced with a mod-revision compare on EVERY write path +// (applies, reconciliation repairs, prune deletes), so two agents can never +// interleave writes and a straggler apply after cutover fails loudly. The +// key is excluded by exact match from scans, counts, prune passes, and the +// RequireEmpty check; the target RBAC grant must cover it; CR deletion +// removes it via a delete-one-key finalizer. type EtcdMirrorCheckpointSpec struct { - // StorageSpec requests persistent storage for the checkpoint file, reusing - // EtcdClusterSpec's StorageSpec shape. When nil the controller mounts an - // emptyDir instead. StorageSpec should be set for any production mirror; a - // small size (e.g. 64Mi) is sufficient. - // +optional - StorageSpec *StorageSpec `json:"storageSpec,omitempty"` - - // SyncInterval controls how often the agent flushes its in-memory - // last-applied revision to the checkpoint file (atomic write: temp file + - // fsync + rename). Defaults to 5s when unset. - // +optional - SyncInterval *metav1.Duration `json:"syncInterval,omitempty"` + // Key overrides the reserved checkpoint key. Defaults to the effective + // destination prefix + "\x00etcdmirror-checkpoint" — the \x00 byte after + // the prefix cannot collide with any real key under it. MUST live under + // the effective destination prefix (target.prefix + sync.destPrefix, + // CEL-enforced): the range-scoped target credential covers it, and the + // exact-match exclusion from scans/counts/prune only works inside the + // mirrored range. Immutable after creation. + // +optional + Key string `json:"key,omitempty"` } -// EtcdMirrorReconciliationSpec configures an OPTIONAL periodic full -// reconciliation pass layered on top of the continuous watch-based mirror. +// EtcdMirrorReconciliationSpec configures the periodic full reconciliation +// pass. The same engine also runs unconditionally (regardless of Enabled or +// DeleteOrphans) as the post-forced-resync mark-and-sweep, the +// OverwriteAndPrune genesis pass, and the Drain verification pass. type EtcdMirrorReconciliationSpec struct { - // Enabled toggles the periodic full reconciliation pass. Defaults to - // false: it is a diff of the full prefix contents on both sides - // (O(keyspace size)), so it is opt-in. + // Enabled toggles the PERIODIC pass. Defaults to false: it is a full + // diff of the prefix contents on both sides (O(keyspace)), so it is + // opt-in. // +optional Enabled bool `json:"enabled,omitempty"` - // Interval between reconciliation passes. Defaults to 1h when Enabled and - // unset. + // Interval between periodic passes. Defaults to 1h when Enabled. // +optional Interval *metav1.Duration `json:"interval,omitempty"` - // DeleteOrphans, when true, allows reconciliation to DELETE target keys - // under the destination prefix that have no corresponding source key. - // Defaults to false. + // DeleteOrphans, when true, allows the PERIODIC pass to delete target + // keys under the destination prefix that have no corresponding source + // key. Defaults to false. (Forced-resync sweeps and OverwriteAndPrune + // always delete orphans; this knob only governs the periodic pass.) // +optional DeleteOrphans bool `json:"deleteOrphans,omitempty"` } // EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. -// Unlike BackupPhase/RestorePhase, most phases here are NOT terminal -- a -// healthy mirror spends its life cycling between Syncing and (briefly, on -// transient errors) Degraded; there is no "Completed" state. +// Unlike BackupPhase/RestorePhase, most phases here are NOT terminal — a +// healthy mirror spends its life in Syncing; there is no "Completed" state. type EtcdMirrorPhase string const ( // EtcdMirrorPhasePending means the EtcdMirror has been accepted but the // agent workload has not been created yet. EtcdMirrorPhasePending EtcdMirrorPhase = "Pending" - // EtcdMirrorPhaseConnecting means the agent pod is running and establishing - // client connections to both Source and Target. + // EtcdMirrorPhaseConnecting means the agent pod is running, establishing + // client connections to both sides and probing versions/cluster IDs. EtcdMirrorPhaseConnecting EtcdMirrorPhase = "Connecting" - // EtcdMirrorPhaseInitialSync means the agent is running SyncBase: the - // paginated, revision-pinned full range scan establishing the base - // revision. Entered on a genesis start (no valid checkpoint), a - // cluster-identity-invalidated checkpoint, or an agent-initiated forced - // resync after compaction (see EtcdMirrorConditionCompacted). + // EtcdMirrorPhaseInitialSync means the agent is running the genesis scan: + // an UNPINNED chunked scan with the watch already open from the revision + // observed before the scan started, buffered events replayed over the + // scanned base (reflector pattern). Because pages read at the current + // revision, mid-scan compaction cannot fail the scan. Also entered during + // a forced resync (then with condition Compacted=True/Reason=ForcedResync). EtcdMirrorPhaseInitialSync EtcdMirrorPhase = "InitialSync" - // EtcdMirrorPhaseSyncing is the steady-state: SyncBase has completed (or - // was skipped via checkpoint resume) and SyncUpdates is watching and - // applying live changes. A healthy mirror spends effectively all its time - // here. + // EtcdMirrorPhaseSyncing is the steady state: watching and applying live + // changes, watermark advancing via progress notifications. EtcdMirrorPhaseSyncing EtcdMirrorPhase = "Syncing" - // EtcdMirrorPhaseDegraded means the agent hit a recoverable condition - // (reconnect backoff to either side, target throttling backoff, or an - // in-progress compaction-forced resync) and is retrying/self-healing. - // Non-terminal: expected to return to Syncing/InitialSync automatically - // without operator action. + // EtcdMirrorPhaseDegraded means the agent is in a retry/backoff loop + // (connection or throttling class) and is expected to self-heal. Forced + // resyncs are NOT Degraded; they report as InitialSync + Compacted=True. EtcdMirrorPhaseDegraded EtcdMirrorPhase = "Degraded" - // EtcdMirrorPhasePaused means spec.paused is true; the agent StatefulSet is - // scaled to zero. The checkpoint is retained so resuming picks up from the - // last-applied revision (subject to the same cluster-identity check on - // resume as any other restart). + // EtcdMirrorPhasePaused means spec.paused is true; the agent Deployment + // is scaled to zero. The checkpoint is retained in the target. EtcdMirrorPhasePaused EtcdMirrorPhase = "Paused" - // EtcdMirrorPhaseFailed means the mirror hit a terminal, non-recoverable - // error (ExpectEmptyPrefix violated at genesis, malformed cert material, - // unresolvable spec misconfiguration) requiring operator intervention. + // EtcdMirrorPhaseFailed means a terminal, non-recoverable error requiring + // operator intervention: EmptyTargetViolation at genesis, source below + // the 3.4 version floor (UnsupportedVersion), a permanent-class write + // error (oversized revision vs target limits), malformed cert material, + // or unresolvable spec misconfiguration. EtcdMirrorPhaseFailed EtcdMirrorPhase = "Failed" ) // Condition types reported on EtcdMirror status. +// +// PAGING ALGEBRA (for alert authors): page on Available=False sustained for +// your tolerance window UNLESS Compacted=True AND progress fields are +// advancing (a forced resync healing itself); TargetQuotaExhausted and +// ResyncLoopDetected page immediately — neither self-heals. const ( - // EtcdMirrorConditionAvailable is True only when the agent pod is running, - // in the Syncing phase, AND the loop-liveness check shows forward progress - // within the configured staleness threshold. A wedged main loop that keeps - // answering /statusz with a stale-but-structurally-valid response is - // explicitly NOT allowed to read as Available=True; see - // ReplicationLagExceeded below for the companion signal. + // EtcdMirrorConditionAvailable is True only when the agent pod is + // running, in the Syncing phase, AND the checkpoint watermark is + // advancing (via applies or watch progress notifications) within the + // staleness threshold. Watermark-derived, not apply-derived: an idle + // prefix on a live watch stays Available; a wedged loop that stops + // confirming progress does not. EtcdMirrorConditionAvailable = "Available" - // EtcdMirrorConditionSourceReachable is True when the agent's last attempt - // to reach Source succeeded. Split from TargetReachable because in the - // primary use case (RKE1 source over the public internet) source + // EtcdMirrorConditionSourceReachable is True when the agent's last + // attempt to reach Source succeeded. Split from TargetReachable because + // in the primary use case (source over the public internet) source // reachability is the most likely persistent failure mode. EtcdMirrorConditionSourceReachable = "SourceReachable" // EtcdMirrorConditionTargetReachable is the target-side analogue. EtcdMirrorConditionTargetReachable = "TargetReachable" // EtcdMirrorConditionTargetThrottled is True while the agent is backing - // off from throttling-class errors on Target (etcd ErrTooManyRequests / - // gRPC ResourceExhausted / quota-exhausted), handled distinctly from a - // plain connection drop (a more conservative backoff curve). Kept separate - // from TargetReachable=False because "target is up but rejecting my write - // rate" is a different, actionable signal than "target is unreachable." + // off from throttling-class errors on Target (rate rejection / + // ErrTooManyRequests). Distinct from TargetReachable=False ("up but + // rejecting my write rate" is actionable differently) and from + // TargetQuotaExhausted (backoff cannot heal a full quota). EtcdMirrorConditionTargetThrottled = "TargetThrottled" - // EtcdMirrorConditionInitialSyncComplete is True once SyncBase has - // completed at least once against the currently-checkpointed source - // cluster identity. Durable -- does not flip back to False on a - // compaction-forced resync (Compacted covers that). Reset to False if the - // checkpoint is invalidated by a cluster-identity mismatch. + // EtcdMirrorConditionTargetQuotaExhausted is True when a target write + // failed with etcd's NOSPACE (rpctypes.ErrNoSpace). Permanent until an + // operator compacts/defrags/disarms the target; the agent stops writing + // rather than burning backoff against a full quota. Detected from the + // typed write-path error, never AlarmList (which needs root). + EtcdMirrorConditionTargetQuotaExhausted = "TargetQuotaExhausted" + // EtcdMirrorConditionInitialSyncComplete is True once the genesis scan + // has completed against the currently-checkpointed cluster identities. + // Durable across forced resyncs (Compacted covers those); reset to False + // when the checkpoint is invalidated by a mismatch of EITHER bound + // cluster ID (source or target), which also re-arms the RequireEmpty + // check. EtcdMirrorConditionInitialSyncComplete = "InitialSyncComplete" - // EtcdMirrorConditionCompacted is True (Reason "ForcedResync") while the - // agent is auto-healing a forced fresh SyncBase after detecting - // source-side compaction raced the watch (a WatchResponse with - // Canceled=true and CompactRevision != 0), rather than crash-looping the - // way etcdctl make-mirror does. Reverts to False once the forced resync - // completes and steady-state watching resumes. + // EtcdMirrorConditionCompacted is True (Reason ForcedResync) while the + // agent heals from source compaction outrunning the watch (restart or + // pause longer than retention; a WatchResponse with CompactRevision != + // 0). Mid-scan compaction is NOT in this class — the unpinned scan is + // immune by construction. Every forced resync ends with a mandatory + // mark-and-sweep prune. Reverts to False when steady-state resumes. EtcdMirrorConditionCompacted = "Compacted" - // EtcdMirrorConditionReplicationLagExceeded is True when SourceRevision - - // LastAppliedRevision (or, more precisely, the agent's own "time since - // last successful destination apply" loop-liveness measure) has exceeded a - // threshold for a sustained duration D. A deadlocked target-write retry - // loop can keep the process alive and /statusz technically responding, - // but it cannot keep making progress, and this condition is derived from - // progress, not liveness. Threshold and duration D are agent-internal - // constants in v1 (not a spec knob). + // EtcdMirrorConditionResyncLoopDetected is True when N consecutive forced + // resyncs completed without reaching steady state — the livelock + // signature of source retention < scan+drain time. Does not self-heal: + // raise retention, raise MaxOpsPerSecond, or shrink the prefix. + EtcdMirrorConditionResyncLoopDetected = "ResyncLoopDetected" + // EtcdMirrorConditionReplicationLagExceeded is True when the checkpoint + // watermark has stayed more than a threshold behind the source's + // current revision for a sustained duration. Both terms come from the + // same watch/progress machinery (never from comparing the two live + // status fields, which snapshot at different instants). Threshold and + // duration are agent-internal constants in v1. EtcdMirrorConditionReplicationLagExceeded = "ReplicationLagExceeded" // EtcdMirrorConditionDriftDetected is True when the last reconciliation - // pass (if spec.reconciliation.enabled) found a nonzero number of - // orphaned/missing keys. Carries counts in Message. Present only when - // reconciliation is enabled; sticky until the next pass reports clean. + // pass found orphaned/missing keys. Carries counts in Message. Sticky + // until the next pass reports clean. EtcdMirrorConditionDriftDetected = "DriftDetected" - // EtcdMirrorConditionEmptyTargetViolation is True (and terminal, Phase -> - // Failed) when ExpectEmptyPrefix was set and the destination prefix was - // found non-empty before the first-ever InitialSync began. + // EtcdMirrorConditionEmptyTargetViolation is True (terminal, Phase -> + // Failed) when initialSync.mode is RequireEmpty and the destination + // prefix was non-empty at genesis. The condition Message embeds the + // exact `etcdctl del` command for the offending range. The reserved + // checkpoint key is excluded by exact match. EtcdMirrorConditionEmptyTargetViolation = "EmptyTargetViolation" + // EtcdMirrorConditionCutoverReady is True when spec.mode is Drain, the + // watermark has reached status.cutover.drainTargetRevision, and the + // verification pass succeeded. From then the fence key's role is + // Primary and any straggler mirror apply fails its compare. Gate + // promotion on `kubectl wait --for=condition=CutoverReady`. + EtcdMirrorConditionCutoverReady = "CutoverReady" + // EtcdMirrorConditionInvariantsHeld is the composed verification verdict: + // True when ReplicationLagExceeded is False, the per-side key counts + // (status.sourceKeyCount/targetKeyCount, reserved key excluded) are + // equal, DriftDetected is False, and the pass that produced the counts + // is fresh. Freshness: with the periodic reconciliation pass enabled, + // within 2x spec.reconciliation.interval; with it disabled the counts + // only refresh on mandatory passes (forced-resync sweeps, + // OverwriteAndPrune genesis, drain verification), so the condition is + // Unknown/stale-reasoned once the last such pass ages out — enable + // reconciliation to make this a continuous signal. It means + // "verification invariants hold", never "safe to cut over" — cutover is + // gated on CutoverReady, which additionally requires spec.mode Drain and + // a reached drainTargetRevision. + EtcdMirrorConditionInvariantsHeld = "InvariantsHeld" + // EtcdMirrorConditionLearnerEndpoint is True when the maintenance + // Status() probe reports IsLearner=true for a configured endpoint. + // Non-blocking (learners self-heal and the balancer routes around them), + // but a learner pick can serve stale reads mid-catch-up. + EtcdMirrorConditionLearnerEndpoint = "LearnerEndpoint" + // EtcdMirrorConditionPrefixConflict is controller-set: another + // EtcdMirror targets an overlapping effective destination range on the + // same target cluster. Declared now so the name is API contract before + // any setter exists. Independent of the controller check, the agent + // itself refuses (permanently, Phase=Failed) to prune a reserved fence + // key owned by a different link, so an undetected overlap stops loudly + // instead of destroying the sibling mirror's fence and data. + EtcdMirrorConditionPrefixConflict = "PrefixConflict" + // EtcdMirrorConditionDirectionConflict is controller-set: this + // mirror's bound source/target cluster IDs are the inverse of another + // EtcdMirror's — two CRs forming a two-way loop, caught by cluster-ID + // binding even when a respelled endpoint string would fool a spec + // comparison. + EtcdMirrorConditionDirectionConflict = "DirectionConflict" +) + +// Condition reasons and Event reasons that are part of the API contract. +const ( + // EtcdMirrorReasonForcedResync is the Compacted condition's reason while + // a forced resync is in flight. + EtcdMirrorReasonForcedResync = "ForcedResync" + // EtcdMirrorReasonUnsupportedVersion is the Failed-phase reason when the + // source is below the 3.4 floor. + EtcdMirrorReasonUnsupportedVersion = "UnsupportedVersion" + + // EtcdMirrorReasonCompacted / EtcdMirrorReasonClusterIDMismatch name WHY + // a forced resync was required (mirroring pkg/mirroragent's ResyncReason + // values). Surfaces: the Compacted condition's MESSAGE and the + // forced-resync events' messages (and, later, the forced-resync metric's + // reason label). The Compacted condition's Reason is always ForcedResync + // while a resync is in flight — alert on that, not on these — and + // forcedResyncCount is a plain counter with no per-reason breakdown. The + // ClusterIDMismatch message additionally names the mismatched side. + EtcdMirrorReasonCompacted = "Compacted" + EtcdMirrorReasonClusterIDMismatch = "ClusterIDMismatch" + // EtcdMirrorReasonCheckpointInvalid is a Failed-phase reason: the stored + // checkpoint was corrupt or of an unknown wire version. PERMANENT — the + // agent fails closed and never auto-resyncs; the operator must inspect + // and delete the reserved key to recover. + EtcdMirrorReasonCheckpointInvalid = "CheckpointInvalid" + + // EtcdMirrorEventForcedResyncStarted / Completed bracket every forced + // resync (Warning/Normal respectively). + EtcdMirrorEventForcedResyncStarted = "ForcedResyncStarted" + EtcdMirrorEventForcedResyncCompleted = "ForcedResyncCompleted" + // EtcdMirrorEventCheckpointInvalidated is emitted when the checkpoint is + // discarded because a bound cluster ID (source or target) no longer + // matches, forcing genesis and re-arming the RequireEmpty check. + EtcdMirrorEventCheckpointInvalidated = "CheckpointInvalidated" + // EtcdMirrorEventInitialSyncCompactionRaced marks an InitialSync attempt + // aborted and restarted from a fresh R0. Two causes, named in the event + // message: WatchBufferOverflow (the replay buffer exceeded + // sync.watchBufferBytes before the base scan completed — a memory-bound + // retry, NOT a compaction race) and WatchCompactedMidScan (a watch + // reconnect landed below the source compact revision — the rare genuine + // race). One event name so operators can alert on scan restarts; the + // cause string prevents conflating "buffer too small for churn" with + // "compaction won a race the design eliminates". Repeated occurrences + // count toward ResyncLoopDetected. + EtcdMirrorEventInitialSyncCompactionRaced = "InitialSyncCompactionRaced" ) // EtcdMirrorStatus defines the observed state of an EtcdMirror. Progress // fields are synced periodically (not per-op), so they are a coarse, -// point-in-time mirror of the agent's authoritative local checkpoint, useful -// for kubectl/dashboards, not an audit log. +// point-in-time mirror of the agent's authoritative checkpoint in the target +// etcd, useful for kubectl/dashboards, not an audit log. +// +// CUTOVER GATE CONTRACT: never compute "caught up" by comparing +// SourceRevision to LastAppliedRevision — they snapshot at different +// instants, and SourceRevision advances on out-of-prefix writes (revisions +// are cluster-global). The manual gate is: quiesce source writers, read the +// source's current revision R yourself (`etcdctl endpoint status`), then +// poll until LastAppliedRevision >= R. The in-CR gate is spec.mode=Drain + +// the CutoverReady condition. Relax target RBAC only after the mirror is +// paused or deleted. type EtcdMirrorStatus struct { // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` @@ -435,67 +737,108 @@ type EtcdMirrorStatus struct { // +optional Phase EtcdMirrorPhase `json:"phase,omitempty"` - // LastAppliedRevision is the source etcd revision through which the target - // is known to be caught up, as of the last periodic status sync. The - // agent's local checkpoint file is the authoritative, hot-path copy; this - // is a periodically-synced mirror of it for observability. + // LastAppliedRevision is the checkpoint watermark: the source revision + // through which the target is caught up, advanced by applies AND by + // watch progress notifications on idle prefixes. The fenced checkpoint + // key in the target etcd is the authoritative copy; this mirrors it. // +optional LastAppliedRevision int64 `json:"lastAppliedRevision,omitempty"` // SourceRevision is the source cluster's revision as of the last status - // sync, for computing an approximate replication lag (SourceRevision - - // LastAppliedRevision) at a glance. + // sync. Cluster-global: it advances on writes outside the mirrored + // prefix, so SourceRevision - LastAppliedRevision OVERSTATES lag for + // prefix-scoped mirrors. See the cutover gate contract above. // +optional SourceRevision int64 `json:"sourceRevision,omitempty"` - // SourceClusterID is the source etcd cluster's cluster ID, as observed on - // the response header of the agent's most recent successful call against - // Source. Surfaced for operator debugging of checkpoint-invalidation - // events -- a changed SourceClusterID across reconciles is the visible - // symptom of "source endpoint now points at a different cluster than the - // checkpoint was taken against." + // SourceClusterID and TargetClusterID are the cluster IDs both bound + // into the checkpoint. Either changing across reconciles is the visible + // symptom of an endpoint now pointing at a different cluster than the + // checkpoint was taken against (CheckpointInvalidated event, forced + // genesis, RequireEmpty re-armed). // +optional SourceClusterID string `json:"sourceClusterID,omitempty"` + // +optional + TargetClusterID string `json:"targetClusterID,omitempty"` + + // SourceVersion and TargetVersion are the etcd server versions from the + // maintenance Status() probe at connect. + // +optional + SourceVersion string `json:"sourceVersion,omitempty"` + // +optional + TargetVersion string `json:"targetVersion,omitempty"` - // InitialSyncKeyCount, InitialSyncStartTime, InitialSyncCompletionTime - // track the base-sync phase for observability. + // InitialSyncKeyCount is the number of keys applied so far by the + // current/last genesis scan. Live-updating during InitialSync (each + // status sync), so InitialSyncKeyCount/InitialSyncTotalKeyCount is a + // progress fraction. // +optional InitialSyncKeyCount int64 `json:"initialSyncKeyCount,omitempty"` + // InitialSyncTotalKeyCount is the denominator: the source-side key count + // under the prefix observed at scan start (first page's RangeResponse + // count). + // +optional + InitialSyncTotalKeyCount int64 `json:"initialSyncTotalKeyCount,omitempty"` // +optional InitialSyncStartTime *metav1.Time `json:"initialSyncStartTime,omitempty"` // +optional InitialSyncCompletionTime *metav1.Time `json:"initialSyncCompletionTime,omitempty"` - // ForcedResyncCount counts how many times the agent has auto-healed from a - // source-compaction-raced-the-watch error, OR a cluster-identity-mismatch - // checkpoint invalidation, by re-running SyncBase. Monotonically - // increasing, never reset. + // LeaseBackedKeyCount is the number of mirrored keys whose source copy is + // lease-backed (kv.Lease != 0). Mirrored copies are NOT lease-backed — + // leases are stripped (see Fidelity Caveats in docs/etcdmirror.md) — so a + // nonzero count means the cutover runbook's purge/re-lease step applies. + // +optional + LeaseBackedKeyCount int64 `json:"leaseBackedKeyCount,omitempty"` + + // ForcedResyncCount counts forced resyncs (compaction outran the watch, + // checkpoint invalidated by a cluster-ID mismatch, or checkpoint + // corrupt/unknown-version). Monotonic, never reset. // +optional ForcedResyncCount int32 `json:"forcedResyncCount,omitempty"` // LastReconciliationTime and LastReconciliationDrift record the most - // recent periodic reconciliation pass, when spec.reconciliation.enabled. + // recent reconciliation pass (periodic or mandatory). // +optional LastReconciliationTime *metav1.Time `json:"lastReconciliationTime,omitempty"` // +optional LastReconciliationDrift *EtcdMirrorDriftInfo `json:"lastReconciliationDrift,omitempty"` + // SourceKeyCount and TargetKeyCount are the per-side key counts from the + // most recent diff/verification pass (reserved checkpoint key and + // excluded prefixes not counted; drain-verification source reads pinned + // at the drained revision with a compacted-fallback re-read). Populated + // by every pass that runs regardless of spec.reconciliation.enabled — + // the mandatory mark-and-sweep after any forced resync, the + // OverwriteAndPrune genesis pass, and the drain verification — plus the + // periodic pass when it is enabled. NOT refreshed on every status sync: + // a healthy RequireEmpty mirror that never forces a resync only gets + // counts from an enabled periodic pass. This is the equality signal + // InvariantsHeld reads; status.cutover's copies remain the frozen + // drain-time snapshot. + // +optional + SourceKeyCount int64 `json:"sourceKeyCount,omitempty"` + // +optional + TargetKeyCount int64 `json:"targetKeyCount,omitempty"` + // LastStatusSyncTime is when status was last refreshed from the agent, so - // staleness (e.g. a wedged agent that stopped responding but hasn't - // crashed) is directly observable. + // staleness of everything above is directly observable. // +optional LastStatusSyncTime *metav1.Time `json:"lastStatusSyncTime,omitempty"` - // LastProgressTime is when the agent last recorded ANY successful - // destination apply (a completed Txn during InitialSync, SyncUpdates, or - // reconciliation). Distinct from LastStatusSyncTime: /statusz can keep - // responding on-time from a wedged loop that has stopped applying writes, - // so this field is what ReplicationLagExceeded is actually derived from. + // LastProgressTime is when the watermark last advanced (apply or watch + // progress notification). Distinct from LastStatusSyncTime: status can + // keep syncing from a wedged loop; this field is what Available and + // ReplicationLagExceeded are derived from. // +optional LastProgressTime *metav1.Time `json:"lastProgressTime,omitempty"` + // Cutover is populated while spec.mode is Drain. + // +optional + Cutover *EtcdMirrorCutoverStatus `json:"cutover,omitempty"` + // AgentPod is the name of the current agent pod (the sole pod of the - // size-1 StatefulSet), for convenient kubectl logs/exec. + // size-1 Deployment), for convenient kubectl logs/exec. // +optional AgentPod string `json:"agentPod,omitempty"` @@ -507,29 +850,72 @@ type EtcdMirrorStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` } +// EtcdMirrorCutoverStatus tracks a Drain-mode cutover. +type EtcdMirrorCutoverStatus struct { + // DrainTargetRevision is the source revision observed when Drain was + // requested — the revision the watermark must reach. + // +optional + DrainTargetRevision int64 `json:"drainTargetRevision,omitempty"` + // DrainedRevision is the watermark at which the drain completed. + // +optional + DrainedRevision int64 `json:"drainedRevision,omitempty"` + // VerifiedTime is when the post-drain verification pass succeeded. + // +optional + VerifiedTime *metav1.Time `json:"verifiedTime,omitempty"` + // SourceKeyCount and TargetKeyCount are the per-side key counts from the + // verification pass (source read pinned at the drained revision; + // reserved checkpoint key excluded). + // +optional + SourceKeyCount int64 `json:"sourceKeyCount,omitempty"` + // +optional + TargetKeyCount int64 `json:"targetKeyCount,omitempty"` + // LeasedKeyCount is LeaseBackedKeyCount frozen at drain completion, for + // the runbook's purge/re-lease step. + // +optional + LeasedKeyCount int64 `json:"leasedKeyCount,omitempty"` +} + type EtcdMirrorDriftInfo struct { + // MissingKeys were present on the source but absent on the target. MissingKeys int64 `json:"missingKeys,omitempty"` - OrphanKeys int64 `json:"orphanKeys,omitempty"` - Repaired bool `json:"repaired,omitempty"` + // DivergentKeys were present on both sides with different values — + // distinct from MissingKeys so "a resync dropped keys" is never + // conflated with "a blind window went stale". + DivergentKeys int64 `json:"divergentKeys,omitempty"` + // OrphanKeys were present on the target with no source counterpart. + OrphanKeys int64 `json:"orphanKeys,omitempty"` + // Repaired is true when the pass wrote fixes rather than only reporting. + Repaired bool `json:"repaired,omitempty"` } // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Source",type=string,JSONPath=`.spec.source.prefix` -// +kubebuilder:printcolumn:name="Target",type=string,JSONPath=`.spec.target.prefix` +// +kubebuilder:printcolumn:name="Available",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].status` // +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` // +kubebuilder:printcolumn:name="Revision",type=integer,JSONPath=`.status.lastAppliedRevision` +// +kubebuilder:printcolumn:name="Source-Rev",type=integer,JSONPath=`.status.sourceRevision` +// +kubebuilder:printcolumn:name="Last-Progress",type=date,JSONPath=`.status.lastProgressTime` +// +kubebuilder:printcolumn:name="Source",type=string,priority=1,JSONPath=`.spec.source.prefix` +// +kubebuilder:printcolumn:name="Target",type=string,priority=1,JSONPath=`.spec.target.prefix` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` -// EtcdMirror is the Schema for the etcdmirrors API. It describes a continuous, -// one-way key-range sync from a source etcd cluster to a target etcd cluster, -// run as a single supervised pod (a size-1 StatefulSet) in this cluster. +// EtcdMirror is the Schema for the etcdmirrors API. It describes a +// continuous, one-way key-range sync from a source etcd cluster to a target +// etcd cluster, run as a single supervised stateless pod (a size-1 +// Deployment; progress lives in a fenced checkpoint key in the target etcd, +// not on a volume). +// +// It is a byte-copy of keys and values, not a replica: revisions, versions, +// and create/mod ordering are target-assigned, and leases are stripped. See +// Fidelity Caveats in docs/etcdmirror.md before depending on anything but +// key/value content. // -// EtcdMirror is deliberately one-way only: clientv3/mirror.Syncer has no -// bidirectional primitive, and etcd itself has no concept of "this write -// originated from a mirror, don't re-mirror it back." Two opposite-direction -// EtcdMirrors pointed at each other create an unbounded write-ping-pong with no -// conflict resolution; see Non-Goals. +// Two-way sync: never — etcd revisions are cluster-local and there is no +// per-key provenance channel, so bidirectional sync is structurally +// inexpressible. Reversal for cutover/failback: yes — delete the CR and +// create a new one with swapped endpoints, initialSync.mode +// OverwriteAndPrune, only after the forward mirror reported CutoverReady. +// See docs/etcdmirror.md. type EtcdMirror struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 7cb29323..0f6a17af 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -269,18 +269,23 @@ func (in *EtcdMirrorBackoffSpec) DeepCopy() *EtcdMirrorBackoffSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EtcdMirrorCheckpointSpec) DeepCopyInto(out *EtcdMirrorCheckpointSpec) { +func (in *EtcdMirrorCABundleRef) DeepCopyInto(out *EtcdMirrorCABundleRef) { *out = *in - if in.StorageSpec != nil { - in, out := &in.StorageSpec, &out.StorageSpec - *out = new(StorageSpec) - (*in).DeepCopyInto(*out) - } - if in.SyncInterval != nil { - in, out := &in.SyncInterval, &out.SyncInterval - *out = new(metav1.Duration) - **out = **in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorCABundleRef. +func (in *EtcdMirrorCABundleRef) DeepCopy() *EtcdMirrorCABundleRef { + if in == nil { + return nil } + out := new(EtcdMirrorCABundleRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorCheckpointSpec) DeepCopyInto(out *EtcdMirrorCheckpointSpec) { + *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorCheckpointSpec. @@ -293,6 +298,25 @@ func (in *EtcdMirrorCheckpointSpec) DeepCopy() *EtcdMirrorCheckpointSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorCutoverStatus) DeepCopyInto(out *EtcdMirrorCutoverStatus) { + *out = *in + if in.VerifiedTime != nil { + in, out := &in.VerifiedTime, &out.VerifiedTime + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorCutoverStatus. +func (in *EtcdMirrorCutoverStatus) DeepCopy() *EtcdMirrorCutoverStatus { + if in == nil { + return nil + } + out := new(EtcdMirrorCutoverStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EtcdMirrorDriftInfo) DeepCopyInto(out *EtcdMirrorDriftInfo) { *out = *in @@ -324,7 +348,7 @@ func (in *EtcdMirrorEndpoint) DeepCopyInto(out *EtcdMirrorEndpoint) { if in.TLS != nil { in, out := &in.TLS, &out.TLS *out = new(EtcdMirrorTLS) - **out = **in + (*in).DeepCopyInto(*out) } if in.Auth != nil { in, out := &in.Auth, &out.Auth @@ -343,6 +367,21 @@ func (in *EtcdMirrorEndpoint) DeepCopy() *EtcdMirrorEndpoint { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorInitialSyncSpec) DeepCopyInto(out *EtcdMirrorInitialSyncSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorInitialSyncSpec. +func (in *EtcdMirrorInitialSyncSpec) DeepCopy() *EtcdMirrorInitialSyncSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorInitialSyncSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EtcdMirrorList) DeepCopyInto(out *EtcdMirrorList) { *out = *in @@ -415,11 +454,16 @@ func (in *EtcdMirrorSpec) DeepCopyInto(out *EtcdMirrorSpec) { *out = *in in.Source.DeepCopyInto(&out.Source) in.Target.DeepCopyInto(&out.Target) + if in.InitialSync != nil { + in, out := &in.InitialSync, &out.InitialSync + *out = new(EtcdMirrorInitialSyncSpec) + **out = **in + } in.Sync.DeepCopyInto(&out.Sync) if in.Checkpoint != nil { in, out := &in.Checkpoint, &out.Checkpoint *out = new(EtcdMirrorCheckpointSpec) - (*in).DeepCopyInto(*out) + **out = **in } if in.Reconciliation != nil { in, out := &in.Reconciliation, &out.Reconciliation @@ -431,6 +475,11 @@ func (in *EtcdMirrorSpec) DeepCopyInto(out *EtcdMirrorSpec) { *out = new(PodTemplate) (*in).DeepCopyInto(*out) } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorSpec. @@ -471,6 +520,11 @@ func (in *EtcdMirrorStatus) DeepCopyInto(out *EtcdMirrorStatus) { in, out := &in.LastProgressTime, &out.LastProgressTime *out = (*in).DeepCopy() } + if in.Cutover != nil { + in, out := &in.Cutover, &out.Cutover + *out = new(EtcdMirrorCutoverStatus) + (*in).DeepCopyInto(*out) + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) @@ -493,16 +547,41 @@ func (in *EtcdMirrorStatus) DeepCopy() *EtcdMirrorStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EtcdMirrorSyncSpec) DeepCopyInto(out *EtcdMirrorSyncSpec) { *out = *in - if in.ReconnectBackoff != nil { - in, out := &in.ReconnectBackoff, &out.ReconnectBackoff - *out = new(EtcdMirrorBackoffSpec) - (*in).DeepCopyInto(*out) + if in.ExcludePrefixes != nil { + in, out := &in.ExcludePrefixes, &out.ExcludePrefixes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TxnFlushBytes != nil { + in, out := &in.TxnFlushBytes, &out.TxnFlushBytes + x := (*in).DeepCopy() + *out = &x + } + if in.PageBytes != nil { + in, out := &in.PageBytes, &out.PageBytes + x := (*in).DeepCopy() + *out = &x + } + if in.WatchBufferBytes != nil { + in, out := &in.WatchBufferBytes, &out.WatchBufferBytes + x := (*in).DeepCopy() + *out = &x + } + if in.RequestTimeout != nil { + in, out := &in.RequestTimeout, &out.RequestTimeout + *out = new(metav1.Duration) + **out = **in } if in.DialTimeout != nil { in, out := &in.DialTimeout, &out.DialTimeout *out = new(metav1.Duration) **out = **in } + if in.ReconnectBackoff != nil { + in, out := &in.ReconnectBackoff, &out.ReconnectBackoff + *out = new(EtcdMirrorBackoffSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorSyncSpec. @@ -518,7 +597,16 @@ func (in *EtcdMirrorSyncSpec) DeepCopy() *EtcdMirrorSyncSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EtcdMirrorTLS) DeepCopyInto(out *EtcdMirrorTLS) { *out = *in - out.SecretRef = in.SecretRef + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.CABundleRef != nil { + in, out := &in.CABundleRef, &out.CABundleRef + *out = new(EtcdMirrorCABundleRef) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorTLS. diff --git a/config/crd/bases/operator.etcd.io_etcdmirrors.yaml b/config/crd/bases/operator.etcd.io_etcdmirrors.yaml index 0cf10bed..8e497c1e 100644 --- a/config/crd/bases/operator.etcd.io_etcdmirrors.yaml +++ b/config/crd/bases/operator.etcd.io_etcdmirrors.yaml @@ -15,11 +15,8 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - jsonPath: .spec.source.prefix - name: Source - type: string - - jsonPath: .spec.target.prefix - name: Target + - jsonPath: .status.conditions[?(@.type=="Available")].status + name: Available type: string - jsonPath: .status.phase name: Phase @@ -27,6 +24,20 @@ spec: - jsonPath: .status.lastAppliedRevision name: Revision type: integer + - jsonPath: .status.sourceRevision + name: Source-Rev + type: integer + - jsonPath: .status.lastProgressTime + name: Last-Progress + type: date + - jsonPath: .spec.source.prefix + name: Source + priority: 1 + type: string + - jsonPath: .spec.target.prefix + name: Target + priority: 1 + type: string - jsonPath: .metadata.creationTimestamp name: Age type: date @@ -34,15 +45,23 @@ spec: schema: openAPIV3Schema: description: |- - EtcdMirror is the Schema for the etcdmirrors API. It describes a continuous, - one-way key-range sync from a source etcd cluster to a target etcd cluster, - run as a single supervised pod (a size-1 StatefulSet) in this cluster. + EtcdMirror is the Schema for the etcdmirrors API. It describes a + continuous, one-way key-range sync from a source etcd cluster to a target + etcd cluster, run as a single supervised stateless pod (a size-1 + Deployment; progress lives in a fenced checkpoint key in the target etcd, + not on a volume). - EtcdMirror is deliberately one-way only: clientv3/mirror.Syncer has no - bidirectional primitive, and etcd itself has no concept of "this write - originated from a mirror, don't re-mirror it back." Two opposite-direction - EtcdMirrors pointed at each other create an unbounded write-ping-pong with no - conflict resolution; see Non-Goals. + It is a byte-copy of keys and values, not a replica: revisions, versions, + and create/mod ordering are target-assigned, and leases are stripped. See + Fidelity Caveats in docs/etcdmirror.md before depending on anything but + key/value content. + + Two-way sync: never — etcd revisions are cluster-local and there is no + per-key provenance channel, so bidirectional sync is structurally + inexpressible. Reversal for cutover/failback: yes — delete the CR and + create a new one with swapped endpoints, initialSync.mode + OverwriteAndPrune, only after the forward mirror reported CutoverReady. + See docs/etcdmirror.md. properties: apiVersion: description: |- @@ -62,85 +81,95 @@ spec: metadata: type: object spec: - description: EtcdMirrorSpec defines the desired state of an EtcdMirror. + description: |- + EtcdMirrorSpec defines the desired state of an EtcdMirror. + + Range-defining and rewrite fields are immutable (CEL transition rules + below): source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes + and checkpoint.key. Changing what range is mirrored, or where it lands, + mid-life silently diverges: a restarted agent resumes from its checkpoint + without a scan, so removing an exclusion never backfills pre-existing keys + and adding one strands already-mirrored keys as permanent orphans — all + with every condition green. Endpoints stay mutable — rotating an NLB DNS + name or adding a member to the same cluster is routine; pointing at a + different cluster is caught at runtime by the checkpoint's dual-cluster-ID + binding, not by spec validation. + + The transition rules compare VALUES, presence-normalized: for these + fields the empty value and an absent field are semantically identical + (prefix "" = whole keyspace, destPrefix "" = strip the source prefix), and + Go typed clients drop explicit "" through omitempty — presence-based rules + would reject every typed-client update of a CR created with an explicit + empty string. properties: checkpoint: - description: Checkpoint configures the agent's local durable progress - checkpoint. + description: Checkpoint configures the reserved checkpoint/fence key + on the target. properties: - storageSpec: + key: description: |- - StorageSpec requests persistent storage for the checkpoint file, reusing - EtcdClusterSpec's StorageSpec shape. When nil the controller mounts an - emptyDir instead. StorageSpec should be set for any production mirror; a - small size (e.g. 64Mi) is sufficient. - properties: - accessModes: - type: string - pvcName: - type: string - storageClassName: - type: string - volumeSizeLimit: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - volumeSizeRequest: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - volumeSizeRequest - type: object - syncInterval: + Key overrides the reserved checkpoint key. Defaults to the effective + destination prefix + "\x00etcdmirror-checkpoint" — the \x00 byte after + the prefix cannot collide with any real key under it. MUST live under + the effective destination prefix (target.prefix + sync.destPrefix, + CEL-enforced): the range-scoped target credential covers it, and the + exact-match exclusion from scans/counts/prune only works inside the + mirrored range. Immutable after creation. + type: string + type: object + initialSync: + description: |- + InitialSync governs genesis behavior: how pre-existing destination keys + are treated (Mode) and optionally where replication starts + (StartRevision). + properties: + mode: + default: RequireEmpty description: |- - SyncInterval controls how often the agent flushes its in-memory - last-applied revision to the checkpoint file (atomic write: temp file + - fsync + rename). Defaults to 5s when unset. + Mode governs pre-existing destination keys at genesis. Defaults to + RequireEmpty (refuse a non-empty destination prefix). + enum: + - RequireEmpty + - Overwrite + - OverwriteAndPrune type: string + startRevision: + description: |- + StartRevision, when > 0, skips the genesis scan entirely and starts + watching from StartRevision+1. For fidelity-preserving seeds: restore + the target from a source snapshot (`etcdutl snapshot restore + --bump-revision --mark-compacted`), then mirror only the delta. + Requires Mode Overwrite or OverwriteAndPrune (CEL-enforced). + format: int64 + minimum: 0 + type: integer type: object - expectEmptyPrefix: + x-kubernetes-validations: + - message: initialSync.startRevision requires initialSync.mode Overwrite + or OverwriteAndPrune (a seeded target is not empty) + rule: '!(has(self.startRevision) && self.startRevision > 0 && (!has(self.mode) + || self.mode == ''RequireEmpty''))' + mode: + default: Sync description: |- - ExpectEmptyPrefix, when true, makes the controller verify the target's - effective destination prefix has zero keys before the agent's FIRST-EVER - SyncBase begins (i.e. no valid checkpoint exists yet for this source - cluster identity), refusing to proceed (Phase -> Failed, condition - EmptyTargetViolation) if it is already non-empty. Analogous to - EtcdRestore's assertClusterEmpty guard. - - SCOPE, STATED PLAINLY: this is single-shot, genesis-only protection. It - is checked exactly once, at the moment a fresh (or cluster-identity- - invalidated, see Checkpoint Lifecycle) InitialSync begins. It does NOT - re-arm on a compaction-triggered forced resync later in the CR's life - (that resync reuses the already-established target prefix ownership), - and it does NOT protect against a foreign writer landing keys under the - same target prefix at any point AFTER the first successful sync -- there - is no key-tagging/provenance marker in v1 that would let the agent - distinguish "my own prior output" from "someone else's write" on a - later re-check (see Non-Goals). Operators who need ongoing enforcement - that nothing else writes to the target prefix must arrange that via - etcd RBAC (grant only this mirror's target credential write access to - the prefix) rather than relying on ExpectEmptyPrefix, which is a - bring-up guard, not a standing invariant. - Defaults to false; operators standing up a NEW mirror onto a fresh - prefix should set this true. - type: boolean + Mode selects continuous replication (Sync, the default) or a cutover + drain (Drain). See EtcdMirrorModeDrain for the cutover contract. + enum: + - Sync + - Drain + type: string paused: description: |- - Paused, when true, tells the controller to scale the agent StatefulSet to - zero replicas without deleting the CR or its checkpoint PVC. For planned - maintenance windows on either cluster without losing sync position. The - agent's own retry/backoff loop handles transient interruptions on its - own; Paused is for deliberate, operator-initiated stops. + Paused, when true, scales the agent Deployment to zero without deleting + the CR or its checkpoint. The checkpoint lives in the target etcd, so + resume picks up from the last fenced watermark. NOTE: pausing longer + than the source's compaction retention guarantees a full forced resync + on resume — there is no free lunch past the retention window. type: boolean podTemplate: description: |- - PodTemplate carries scheduling/affinity/labels/annotations for the agent - pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. + PodTemplate carries scheduling/affinity/labels/annotations for the + agent pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. properties: metadata: description: Metadata is the metadata to add to the pod. @@ -1128,43 +1157,116 @@ spec: reconciliation: description: |- Reconciliation optionally enables a periodic full diff-and-repair pass - layered on top of the continuous watch-based mirror. + layered on top of the continuous watch-based mirror. Independent of + this setting, one reconciliation-with-delete pass always runs after any + forced resync (mark-and-sweep), and as the OverwriteAndPrune genesis + pass and the Drain verification pass. properties: deleteOrphans: description: |- - DeleteOrphans, when true, allows reconciliation to DELETE target keys - under the destination prefix that have no corresponding source key. - Defaults to false. + DeleteOrphans, when true, allows the PERIODIC pass to delete target + keys under the destination prefix that have no corresponding source + key. Defaults to false. (Forced-resync sweeps and OverwriteAndPrune + always delete orphans; this knob only governs the periodic pass.) type: boolean enabled: description: |- - Enabled toggles the periodic full reconciliation pass. Defaults to - false: it is a diff of the full prefix contents on both sides - (O(keyspace size)), so it is opt-in. + Enabled toggles the PERIODIC pass. Defaults to false: it is a full + diff of the prefix contents on both sides (O(keyspace)), so it is + opt-in. type: boolean interval: - description: |- - Interval between reconciliation passes. Defaults to 1h when Enabled and - unset. + description: Interval between periodic passes. Defaults to 1h + when Enabled. type: string type: object + resources: + description: |- + Resources are the agent container's compute resources. The agent's + memory model is bounded by Sync.PageBytes (single in-flight scan page, + no unbounded read-ahead); size limits accordingly. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object source: description: |- Source is the etcd cluster keys are read from. EtcdMirror never writes back to Source; the agent's source-side client is only ever used for Get/Watch, never Put/Delete/Txn. + + VERSION FLOOR: source etcd must be >= 3.4 (probed via maintenance + Status() at connect; below the floor the mirror goes Failed with reason + UnsupportedVersion). >= 3.4.25 / 3.5.8 is the recommended floor: below + it, watch progress notifications are unreliable and the agent cannot + trust the watermark machinery that drives lag, the checkpoint, and the + Drain gate. properties: auth: - description: |- - Auth configures etcd username/password (RBAC) auth for THIS side, - ambient-or-secretRef per the objectstore_creds.go pattern. + description: Auth configures etcd username/password (RBAC) auth + for THIS side. properties: secretRef: description: |- - SecretRef names a Secret holding "username" and "password" keys. If this - whole Auth block is nil, the agent does not call etcd's Authenticate() - at all. There is no ambient fallback for etcd username/password auth - (unlike objectstore's IRSA/Workload Identity). + SecretRef names a Secret holding "username" and "password" keys. If + this whole Auth block is nil, the agent does not call etcd's + Authenticate() at all; the pinned v3 client transparently re-auths on + token expiry. PRECEDENCE: when both a client certificate and Auth are + supplied, etcd uses the token identity, not the certificate CN — the + Auth user must hold the range-scoped role. properties: name: default: "" @@ -1189,24 +1291,19 @@ spec: EndpointList is a raw set of etcd client-URL host:port (or scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". This is the ONLY supported mechanism for a cluster external to this - Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB) -- - there is deliberately no "tunnel" or "port-forward" mode. A - `kubectl port-forward` is a client-attached, ephemeral process tied to a - human's terminal; it does not survive this mirror pod restarting, - rescheduling, or the operator itself restarting, which defeats the - entire point of a supervised, restart-tolerant workload. If you need a - persistent tunnel, terminate it yourself (VPN, Interconnect, NLB) - upstream of this CR and hand EtcdMirror the resulting stable - endpoint(s); network reachability itself is out of scope for this CRD. + Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB); + there is deliberately no tunnel/port-forward mode — terminate any + tunnel upstream and hand EtcdMirror the resulting stable endpoint(s). + + Prefer listing per-member endpoints over a single load-balancer VIP: a + TCP-health-checked VIP cannot see etcd quorum, and the client's own + balancer handles per-member failover. - IP-LITERAL ENDPOINTS: if any entry here is a bare IP literal (plausible - for an NLB endpoint with no DNS name), Go's TLS stack requires an IP SAN - (not a DNS SAN) on the peer certificate for verification to succeed. - Either set the corresponding EtcdMirrorTLS.ServerName to a hostname that - IS present as a DNS SAN on the certificate, or ensure the certificate - carries an IP SAN matching the literal. Operators who hit a verification - failure here should fix the SAN/ServerName mismatch, not reach for - InsecureSkipVerify to work around it. + IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN) + to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a + hostname present as a DNS SAN on the certificate, or ensure the + certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch; + do not reach for InsecureSkipVerify. items: type: string type: array @@ -1214,16 +1311,14 @@ spec: description: |- Prefix is the etcd key prefix on THIS side. On Source, only keys under this prefix are synced; empty means the whole keyspace. On Target, this - is the prefix under which the agent writes mirrored keys after any - Sync.DestPrefix remap is applied. + is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's + rewrite formula). Immutable after creation. type: string serviceRef: description: |- - ServiceRef points at a Kubernetes Service in this cluster whose DNS name - resolves the etcd client endpoint(s). The rarer same-cluster or - co-located case (e.g. mirroring between two EtcdClusters both running in - this GKE cluster). Namespace defaults to the EtcdMirror's own namespace - when empty. + ServiceRef points at a Kubernetes Service in this cluster whose DNS + name resolves the etcd client endpoint(s), for the same-cluster case. + Namespace defaults to the EtcdMirror's own namespace when empty. properties: name: description: Name is the Service name. @@ -1244,17 +1339,45 @@ spec: type: object tls: description: |- - TLS configures the agent's client TLS identity/trust for THIS side. Nil - means the agent dials this side in cleartext. + TLS configures the agent's client TLS for THIS side. Nil means the + agent dials this side in cleartext (and https:// endpoints are + CEL-rejected). An empty block means server-auth TLS verified against + the system trust roots. properties: + caBundleRef: + description: |- + CABundleRef optionally sources the trust anchors from a separate + Secret or ConfigMap key, decoupling trust from the identity Secret + (Gateway API caCertificateRefs precedent). Takes precedence over + SecretRef's ca.crt. + properties: + key: + description: Key within the object. Defaults to "ca.crt". + type: string + kind: + default: ConfigMap + description: Kind is Secret or ConfigMap. Defaults to + ConfigMap. + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the Secret or ConfigMap, in the EtcdMirror's + namespace. + minLength: 1 + type: string + required: + - name + type: object insecureSkipVerify: default: false description: |- InsecureSkipVerify disables server certificate verification. Strongly discouraged, especially for a source reached over the public internet. Requires InsecureSkipVerifyAcknowledgeRisk to also be set true - (CEL-enforced on this type). The controller additionally emits a - standing Warning event whenever this is true. + (CEL-enforced). The controller additionally emits a standing Warning + event whenever this is true. type: boolean insecureSkipVerifyAcknowledgeRisk: default: false @@ -1268,15 +1391,15 @@ spec: description: |- SecretRef names a Secret (in the EtcdMirror's namespace) holding this side's TLS material in the standard kubernetes.io/tls-compatible shape: - - ca.crt: PEM CA bundle used to verify the peer's server certificate. - Required for TLS unless InsecureSkipVerify is true. - - tls.crt / tls.key: PEM client certificate + key, for mTLS. Optional -- - omit both for server-auth-only TLS (verify the peer, authenticate to - it via Auth instead, or not at all). - This is the same key layout a cert-manager Certificate's spec.secretName - Secret uses, so a cert-manager Certificate (or `kubectl create secret - tls`, or any other issuance mechanism) can populate this Secret with zero - coupling from this CRD to cert-manager. + - ca.crt: PEM CA bundle used to verify the peer's server certificate + (unless CABundleRef overrides it, or InsecureSkipVerify is true). + - tls.crt / tls.key: PEM client certificate + key, for mTLS. + Optional — omit both for server-auth-only TLS. + Nil means no client identity and verification against the system trust + roots (the etcdctl default). Note a source running with + --client-cert-auth (the RKE1 default) rejects certless clients at the + handshake regardless of etcd RBAC auth; server-auth-only + Auth is not + viable against such a source. properties: name: default: "" @@ -1291,111 +1414,176 @@ spec: x-kubernetes-map-type: atomic serverName: description: |- - ServerName overrides the TLS ServerName (SNI) used for verification, for - cases where the dialed endpoint's address doesn't match a SAN on the - certificate (e.g. dialing an NLB IP directly -- see EndpointList's - IP-literal guidance). + ServerName overrides the TLS ServerName (SNI) used for verification, + for cases where the dialed address doesn't match a SAN on the + certificate (e.g. dialing an NLB IP directly). Applies to EVERY + endpoint in the list, so mixing endpoints with different certificates + behind one ServerName will fail verification on the mismatched ones. type: string - required: - - secretRef type: object x-kubernetes-validations: - message: insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk to also be true rule: '!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk' - - message: secretRef.name is required - rule: has(self.secretRef.name) && size(self.secretRef.name) - > 0 + - message: secretRef.name must be non-empty when secretRef is + set + rule: '!has(self.secretRef) || (has(self.secretRef.name) && + size(self.secretRef.name) > 0)' type: object x-kubernetes-validations: - message: exactly one of endpointList or serviceRef must be set rule: (has(self.endpointList) && size(self.endpointList) > 0) != has(self.serviceRef) + - message: 'http:// endpoints conflict with a tls block: use https:// + endpoints or remove tls' + rule: '!(has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''http://'')))' + - message: https:// endpoints require a tls block (an empty tls block + selects server-auth TLS against system trust roots) + rule: '!(!has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''https://'')))' sync: description: |- - Sync tunes runtime sync behavior (batching, rate limiting, prefix - rewrite, backoff). + Sync tunes runtime sync behavior (batching, paging, rate limiting, + prefix rewrite, timeouts, backoff). properties: destPrefix: description: |- - DestPrefix rewrites Source.Prefix to a different prefix on the target, - via an anchored strip-and-reprefix (strings.TrimPrefix(key, - source.Prefix), then destPrefix + rest) -- never make-mirror's naive - first-occurrence strings.Replace. Mutually exclusive with NoDestPrefix. - Empty (and NoDestPrefix false) means the source prefix is reused - verbatim on the target, landing keys under Target.Prefix + the stripped - key remainder. + DestPrefix is the middle term of the rewrite formula above. Default "" + means the source prefix is stripped and key remainders land directly + under target.prefix. Immutable after creation. type: string dialTimeout: description: |- - DialTimeout bounds how long the agent waits to establish the initial - client connection to each side. Defaults to 10s when unset. + DialTimeout bounds establishing the initial client connection to each + side. Defaults to 10s. type: string + excludePrefixes: + description: |- + ExcludePrefixes lists source key prefixes (full source-side keys, e.g. + "/registry/events/") skipped entirely: not scanned, not watched, not + counted, not pruned. Use to drop high-churn low-value ranges and cut + WAN cost, or to skip lease-backed ranges that don't survive mirroring. + Nested/duplicate entries are normalized by the agent (an entry covered + by another is dropped). Immutable after creation (it defines the + mirrored range): removing an exclusion would require a backfill scan + the checkpoint-resume path never runs, and adding one would strand + already-mirrored keys as permanent orphans — change it via + delete-and-recreate with an appropriate initialSync.mode instead. + items: + type: string + maxItems: 64 + type: array maxOpsPerSecond: description: |- - MaxOpsPerSecond rate-limits the agent's destination write rate (a simple - token bucket over puts+deletes/sec), applied to BOTH InitialSync's Txn - stream and SyncUpdates. Zero (default) means unlimited. + MaxOpsPerSecond rate-limits the agent's target write rate (a token + bucket over puts+deletes/sec), applied to both the genesis scan and + watch-driven applies. Zero (default) means unlimited. Mind the + retention prerequisite above when throttling. format: int32 minimum: 0 type: integer maxTxnOps: description: |- - MaxTxnOps bounds how many put/delete operations the agent batches into a - single destination Txn, applied uniformly to BOTH the initial SyncBase - phase and the watch-driven SyncUpdates phase. Defaults to 128 when unset. + MaxTxnOps bounds how many operations the agent batches into a single + target Txn, including the reserved checkpoint-write slot. Must not + exceed the target's --max-txn-ops (etcd default 128). Defaults to 128. format: int32 - minimum: 1 + minimum: 2 type: integer - noDestPrefix: + pageBytes: + anyOf: + - type: integer + - type: string + description: PageBytes bounds bytes per source scan page. Defaults + to 1Mi. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + pageKeyLimit: description: |- - NoDestPrefix strips Source.Prefix entirely rather than remapping it, - mirroring make-mirror's --no-dest-prefix. Mutually exclusive with a - non-empty DestPrefix (rejected by EtcdMirrorSpec's XValidation rule). - type: boolean + PageKeyLimit bounds keys per source scan page during InitialSync and + reconciliation. The scan is pull-based, one page in flight — no + read-ahead — so this and PageBytes bound agent memory. Defaults to 512. + format: int32 + minimum: 1 + type: integer reconnectBackoff: description: |- - ReconnectBackoff bounds the retry/backoff loop wrapping every Syncer call - and every destination Txn call. Defaults to exponential backoff from 1s - to 30s, uncapped in attempt count, when unset. + ReconnectBackoff bounds the retry/backoff loop wrapping connection-class + errors. Throttling-class errors (target rate rejection) use a more + conservative curve derived from the same bounds; quota exhaustion + (TargetQuotaExhausted) and permanent errors are never retried through + this loop. Defaults to exponential backoff from 1s to 30s. properties: initialDelay: type: string maxDelay: type: string type: object + requestTimeout: + description: |- + RequestTimeout is the per-RPC context deadline applied to every unary + call on both sides (watches excluded — they are long-lived by design + and covered by progress-notification liveness instead). Without it a + blackholed call through an NLB never errors and backoff never engages. + Defaults to 30s. + type: string + txnFlushBytes: + anyOf: + - type: integer + - type: string + description: |- + TxnFlushBytes is the byte watermark at which a batch is flushed (at the + next source-revision boundary). Keep well under etcd's request size + limits: a Txn over ~1.5MiB is rejected by the server and one over 2MiB + by the client send cap — both classified permanent errors, not + throttling. Defaults to 1Mi. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + watchBufferBytes: + anyOf: + - type: integer + - type: string + description: |- + WatchBufferBytes bounds the memory used to buffer watch events + observed from R0 while the genesis scan runs (the reflector replay + buffer). On overflow the agent cancels the source watch and restarts + the scan from a fresh R0 (see the InitialSyncCompactionRaced event) — + a bounded retry instead of unbounded growth when source churn outruns + scan+apply throughput. Defaults to 16Mi (must stay in lockstep with + pkg/mirroragent's DefaultWatchBufferBytes). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true type: object target: description: |- Target is the etcd cluster keys are written into. - SECURITY PREREQUISITE: Target.Auth's credential (and/or the target - client certificate's associated etcd RBAC role) MUST be an etcd RBAC - user restricted via a range-scoped grant-permission to Target.Prefix (or - Sync.DestPrefix's resulting range) -- never a cluster-admin-equivalent - credential. The agent's client-side rewriteKey/prefix logic is defense - against bugs in the agent's own code; it is NOT a security boundary. A - compromised agent process, a buggy rewriteKey, or an empty/typo'd prefix - with an over-privileged credential can write or delete anywhere in the - target cluster's entire keyspace. This is a deployment prerequisite the - operator (human) must configure in etcd itself (via `etcdctl role - grant-permission --prefix=true readwrite `) before - pointing an EtcdMirror at it; the controller cannot create or enforce - etcd-native RBAC roles itself, but it does check for and surface an - unrestricted-looking target credential where etcd's auth API makes that - detectable (see Safety Guards). + SECURITY PREREQUISITE: Target's credential (etcd RBAC user and/or the + client certificate's role) MUST be range-scoped to the effective + destination prefix — never a cluster-admin-equivalent credential. The + agent's client-side rewrite logic is defense against bugs in its own + code, NOT a security boundary. The grant must also cover the reserved + checkpoint key (see Checkpoint). Configure via `etcdctl role + grant-permission --prefix=true readwrite ` before + pointing an EtcdMirror at the cluster. + + The target must run with auto-compaction enabled: forced-resync churn + and prune passes march an uncompacted target toward its storage quota + (2GiB by default), which surfaces as TargetQuotaExhausted. properties: auth: - description: |- - Auth configures etcd username/password (RBAC) auth for THIS side, - ambient-or-secretRef per the objectstore_creds.go pattern. + description: Auth configures etcd username/password (RBAC) auth + for THIS side. properties: secretRef: description: |- - SecretRef names a Secret holding "username" and "password" keys. If this - whole Auth block is nil, the agent does not call etcd's Authenticate() - at all. There is no ambient fallback for etcd username/password auth - (unlike objectstore's IRSA/Workload Identity). + SecretRef names a Secret holding "username" and "password" keys. If + this whole Auth block is nil, the agent does not call etcd's + Authenticate() at all; the pinned v3 client transparently re-auths on + token expiry. PRECEDENCE: when both a client certificate and Auth are + supplied, etcd uses the token identity, not the certificate CN — the + Auth user must hold the range-scoped role. properties: name: default: "" @@ -1420,24 +1608,19 @@ spec: EndpointList is a raw set of etcd client-URL host:port (or scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". This is the ONLY supported mechanism for a cluster external to this - Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB) -- - there is deliberately no "tunnel" or "port-forward" mode. A - `kubectl port-forward` is a client-attached, ephemeral process tied to a - human's terminal; it does not survive this mirror pod restarting, - rescheduling, or the operator itself restarting, which defeats the - entire point of a supervised, restart-tolerant workload. If you need a - persistent tunnel, terminate it yourself (VPN, Interconnect, NLB) - upstream of this CR and hand EtcdMirror the resulting stable - endpoint(s); network reachability itself is out of scope for this CRD. + Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB); + there is deliberately no tunnel/port-forward mode — terminate any + tunnel upstream and hand EtcdMirror the resulting stable endpoint(s). - IP-LITERAL ENDPOINTS: if any entry here is a bare IP literal (plausible - for an NLB endpoint with no DNS name), Go's TLS stack requires an IP SAN - (not a DNS SAN) on the peer certificate for verification to succeed. - Either set the corresponding EtcdMirrorTLS.ServerName to a hostname that - IS present as a DNS SAN on the certificate, or ensure the certificate - carries an IP SAN matching the literal. Operators who hit a verification - failure here should fix the SAN/ServerName mismatch, not reach for - InsecureSkipVerify to work around it. + Prefer listing per-member endpoints over a single load-balancer VIP: a + TCP-health-checked VIP cannot see etcd quorum, and the client's own + balancer handles per-member failover. + + IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN) + to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a + hostname present as a DNS SAN on the certificate, or ensure the + certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch; + do not reach for InsecureSkipVerify. items: type: string type: array @@ -1445,16 +1628,14 @@ spec: description: |- Prefix is the etcd key prefix on THIS side. On Source, only keys under this prefix are synced; empty means the whole keyspace. On Target, this - is the prefix under which the agent writes mirrored keys after any - Sync.DestPrefix remap is applied. + is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's + rewrite formula). Immutable after creation. type: string serviceRef: description: |- - ServiceRef points at a Kubernetes Service in this cluster whose DNS name - resolves the etcd client endpoint(s). The rarer same-cluster or - co-located case (e.g. mirroring between two EtcdClusters both running in - this GKE cluster). Namespace defaults to the EtcdMirror's own namespace - when empty. + ServiceRef points at a Kubernetes Service in this cluster whose DNS + name resolves the etcd client endpoint(s), for the same-cluster case. + Namespace defaults to the EtcdMirror's own namespace when empty. properties: name: description: Name is the Service name. @@ -1475,17 +1656,45 @@ spec: type: object tls: description: |- - TLS configures the agent's client TLS identity/trust for THIS side. Nil - means the agent dials this side in cleartext. + TLS configures the agent's client TLS for THIS side. Nil means the + agent dials this side in cleartext (and https:// endpoints are + CEL-rejected). An empty block means server-auth TLS verified against + the system trust roots. properties: + caBundleRef: + description: |- + CABundleRef optionally sources the trust anchors from a separate + Secret or ConfigMap key, decoupling trust from the identity Secret + (Gateway API caCertificateRefs precedent). Takes precedence over + SecretRef's ca.crt. + properties: + key: + description: Key within the object. Defaults to "ca.crt". + type: string + kind: + default: ConfigMap + description: Kind is Secret or ConfigMap. Defaults to + ConfigMap. + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the Secret or ConfigMap, in the EtcdMirror's + namespace. + minLength: 1 + type: string + required: + - name + type: object insecureSkipVerify: default: false description: |- InsecureSkipVerify disables server certificate verification. Strongly discouraged, especially for a source reached over the public internet. Requires InsecureSkipVerifyAcknowledgeRisk to also be set true - (CEL-enforced on this type). The controller additionally emits a - standing Warning event whenever this is true. + (CEL-enforced). The controller additionally emits a standing Warning + event whenever this is true. type: boolean insecureSkipVerifyAcknowledgeRisk: default: false @@ -1499,15 +1708,15 @@ spec: description: |- SecretRef names a Secret (in the EtcdMirror's namespace) holding this side's TLS material in the standard kubernetes.io/tls-compatible shape: - - ca.crt: PEM CA bundle used to verify the peer's server certificate. - Required for TLS unless InsecureSkipVerify is true. - - tls.crt / tls.key: PEM client certificate + key, for mTLS. Optional -- - omit both for server-auth-only TLS (verify the peer, authenticate to - it via Auth instead, or not at all). - This is the same key layout a cert-manager Certificate's spec.secretName - Secret uses, so a cert-manager Certificate (or `kubectl create secret - tls`, or any other issuance mechanism) can populate this Secret with zero - coupling from this CRD to cert-manager. + - ca.crt: PEM CA bundle used to verify the peer's server certificate + (unless CABundleRef overrides it, or InsecureSkipVerify is true). + - tls.crt / tls.key: PEM client certificate + key, for mTLS. + Optional — omit both for server-auth-only TLS. + Nil means no client identity and verification against the system trust + roots (the etcdctl default). Note a source running with + --client-cert-auth (the RKE1 default) rejects certless clients at the + handshake regardless of etcd RBAC auth; server-auth-only + Auth is not + viable against such a source. properties: name: default: "" @@ -1522,45 +1731,83 @@ spec: x-kubernetes-map-type: atomic serverName: description: |- - ServerName overrides the TLS ServerName (SNI) used for verification, for - cases where the dialed endpoint's address doesn't match a SAN on the - certificate (e.g. dialing an NLB IP directly -- see EndpointList's - IP-literal guidance). + ServerName overrides the TLS ServerName (SNI) used for verification, + for cases where the dialed address doesn't match a SAN on the + certificate (e.g. dialing an NLB IP directly). Applies to EVERY + endpoint in the list, so mixing endpoints with different certificates + behind one ServerName will fail verification on the mismatched ones. type: string - required: - - secretRef type: object x-kubernetes-validations: - message: insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk to also be true rule: '!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk' - - message: secretRef.name is required - rule: has(self.secretRef.name) && size(self.secretRef.name) - > 0 + - message: secretRef.name must be non-empty when secretRef is + set + rule: '!has(self.secretRef) || (has(self.secretRef.name) && + size(self.secretRef.name) > 0)' type: object x-kubernetes-validations: - message: exactly one of endpointList or serviceRef must be set rule: (has(self.endpointList) && size(self.endpointList) > 0) != has(self.serviceRef) + - message: 'http:// endpoints conflict with a tls block: use https:// + endpoints or remove tls' + rule: '!(has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''http://'')))' + - message: https:// endpoints require a tls block (an empty tls block + selects server-auth TLS against system trust roots) + rule: '!(!has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''https://'')))' required: - source - target type: object x-kubernetes-validations: - - message: sync.destPrefix and sync.noDestPrefix are mutually exclusive - rule: '!(has(self.sync) && has(self.sync.destPrefix) && size(self.sync.destPrefix) - > 0 && has(self.sync.noDestPrefix) && self.sync.noDestPrefix)' + - message: source.prefix is immutable + rule: '(has(self.source.prefix) ? self.source.prefix : "") == (has(oldSelf.source.prefix) + ? oldSelf.source.prefix : "")' + - message: target.prefix is immutable + rule: '(has(self.target.prefix) ? self.target.prefix : "") == (has(oldSelf.target.prefix) + ? oldSelf.target.prefix : "")' + - message: sync.destPrefix is immutable + rule: '(has(self.sync) && has(self.sync.destPrefix) ? self.sync.destPrefix + : "") == (has(oldSelf.sync) && has(oldSelf.sync.destPrefix) ? oldSelf.sync.destPrefix + : "")' + - message: sync.excludePrefixes is immutable + rule: '(has(self.sync) && has(self.sync.excludePrefixes) ? self.sync.excludePrefixes + : []) == (has(oldSelf.sync) && has(oldSelf.sync.excludePrefixes) ? + oldSelf.sync.excludePrefixes : [])' + - message: checkpoint.key is immutable + rule: '(has(self.checkpoint) && has(self.checkpoint.key) ? self.checkpoint.key + : "") == (has(oldSelf.checkpoint) && has(oldSelf.checkpoint.key) ? + oldSelf.checkpoint.key : "")' + - message: checkpoint.key must live under the effective destination prefix + (target.prefix + sync.destPrefix) + rule: '!has(self.checkpoint) || !has(self.checkpoint.key) || self.checkpoint.key + == "" || self.checkpoint.key.startsWith((has(self.target.prefix) ? + self.target.prefix : "") + (has(self.sync) && has(self.sync.destPrefix) + ? self.sync.destPrefix : ""))' status: description: |- EtcdMirrorStatus defines the observed state of an EtcdMirror. Progress fields are synced periodically (not per-op), so they are a coarse, - point-in-time mirror of the agent's authoritative local checkpoint, useful - for kubectl/dashboards, not an audit log. + point-in-time mirror of the agent's authoritative checkpoint in the target + etcd, useful for kubectl/dashboards, not an audit log. + + CUTOVER GATE CONTRACT: never compute "caught up" by comparing + SourceRevision to LastAppliedRevision — they snapshot at different + instants, and SourceRevision advances on out-of-prefix writes (revisions + are cluster-global). The manual gate is: quiesce source writers, read the + source's current revision R yourself (`etcdctl endpoint status`), then + poll until LastAppliedRevision >= R. The in-CR gate is spec.mode=Drain + + the CutoverReady condition. Relax target RBAC only after the mirror is + paused or deleted. properties: agentPod: description: |- AgentPod is the name of the current agent pod (the sole pod of the - size-1 StatefulSet), for convenient kubectl logs/exec. + size-1 Deployment), for convenient kubectl logs/exec. type: string conditions: items: @@ -1621,12 +1868,47 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + cutover: + description: Cutover is populated while spec.mode is Drain. + properties: + drainTargetRevision: + description: |- + DrainTargetRevision is the source revision observed when Drain was + requested — the revision the watermark must reach. + format: int64 + type: integer + drainedRevision: + description: DrainedRevision is the watermark at which the drain + completed. + format: int64 + type: integer + leasedKeyCount: + description: |- + LeasedKeyCount is LeaseBackedKeyCount frozen at drain completion, for + the runbook's purge/re-lease step. + format: int64 + type: integer + sourceKeyCount: + description: |- + SourceKeyCount and TargetKeyCount are the per-side key counts from the + verification pass (source read pinned at the drained revision; + reserved checkpoint key excluded). + format: int64 + type: integer + targetKeyCount: + format: int64 + type: integer + verifiedTime: + description: VerifiedTime is when the post-drain verification + pass succeeded. + format: date-time + type: string + type: object forcedResyncCount: description: |- - ForcedResyncCount counts how many times the agent has auto-healed from a - source-compaction-raced-the-watch error, OR a cluster-identity-mismatch - checkpoint invalidation, by re-running SyncBase. Monotonically - increasing, never reset. + ForcedResyncCount counts forced resyncs (compaction outran the watch, + checkpoint invalidated by a cluster-ID mismatch, or checkpoint + corrupt/unknown-version). Monotonic, never reset. format: int32 type: integer initialSyncCompletionTime: @@ -1634,80 +1916,135 @@ spec: type: string initialSyncKeyCount: description: |- - InitialSyncKeyCount, InitialSyncStartTime, InitialSyncCompletionTime - track the base-sync phase for observability. + InitialSyncKeyCount is the number of keys applied so far by the + current/last genesis scan. Live-updating during InitialSync (each + status sync), so InitialSyncKeyCount/InitialSyncTotalKeyCount is a + progress fraction. format: int64 type: integer initialSyncStartTime: format: date-time type: string + initialSyncTotalKeyCount: + description: |- + InitialSyncTotalKeyCount is the denominator: the source-side key count + under the prefix observed at scan start (first page's RangeResponse + count). + format: int64 + type: integer lastAppliedRevision: description: |- - LastAppliedRevision is the source etcd revision through which the target - is known to be caught up, as of the last periodic status sync. The - agent's local checkpoint file is the authoritative, hot-path copy; this - is a periodically-synced mirror of it for observability. + LastAppliedRevision is the checkpoint watermark: the source revision + through which the target is caught up, advanced by applies AND by + watch progress notifications on idle prefixes. The fenced checkpoint + key in the target etcd is the authoritative copy; this mirrors it. format: int64 type: integer lastProgressTime: description: |- - LastProgressTime is when the agent last recorded ANY successful - destination apply (a completed Txn during InitialSync, SyncUpdates, or - reconciliation). Distinct from LastStatusSyncTime: /statusz can keep - responding on-time from a wedged loop that has stopped applying writes, - so this field is what ReplicationLagExceeded is actually derived from. + LastProgressTime is when the watermark last advanced (apply or watch + progress notification). Distinct from LastStatusSyncTime: status can + keep syncing from a wedged loop; this field is what Available and + ReplicationLagExceeded are derived from. format: date-time type: string lastReconciliationDrift: properties: + divergentKeys: + description: |- + DivergentKeys were present on both sides with different values — + distinct from MissingKeys so "a resync dropped keys" is never + conflated with "a blind window went stale". + format: int64 + type: integer missingKeys: + description: MissingKeys were present on the source but absent + on the target. format: int64 type: integer orphanKeys: + description: OrphanKeys were present on the target with no source + counterpart. format: int64 type: integer repaired: + description: Repaired is true when the pass wrote fixes rather + than only reporting. type: boolean type: object lastReconciliationTime: description: |- LastReconciliationTime and LastReconciliationDrift record the most - recent periodic reconciliation pass, when spec.reconciliation.enabled. + recent reconciliation pass (periodic or mandatory). format: date-time type: string lastStatusSyncTime: description: |- LastStatusSyncTime is when status was last refreshed from the agent, so - staleness (e.g. a wedged agent that stopped responding but hasn't - crashed) is directly observable. + staleness of everything above is directly observable. format: date-time type: string + leaseBackedKeyCount: + description: |- + LeaseBackedKeyCount is the number of mirrored keys whose source copy is + lease-backed (kv.Lease != 0). Mirrored copies are NOT lease-backed — + leases are stripped (see Fidelity Caveats in docs/etcdmirror.md) — so a + nonzero count means the cutover runbook's purge/re-lease step applies. + format: int64 + type: integer observedGeneration: format: int64 type: integer phase: description: |- EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. - Unlike BackupPhase/RestorePhase, most phases here are NOT terminal -- a - healthy mirror spends its life cycling between Syncing and (briefly, on - transient errors) Degraded; there is no "Completed" state. + Unlike BackupPhase/RestorePhase, most phases here are NOT terminal — a + healthy mirror spends its life in Syncing; there is no "Completed" state. type: string sourceClusterID: description: |- - SourceClusterID is the source etcd cluster's cluster ID, as observed on - the response header of the agent's most recent successful call against - Source. Surfaced for operator debugging of checkpoint-invalidation - events -- a changed SourceClusterID across reconciles is the visible - symptom of "source endpoint now points at a different cluster than the - checkpoint was taken against." + SourceClusterID and TargetClusterID are the cluster IDs both bound + into the checkpoint. Either changing across reconciles is the visible + symptom of an endpoint now pointing at a different cluster than the + checkpoint was taken against (CheckpointInvalidated event, forced + genesis, RequireEmpty re-armed). type: string + sourceKeyCount: + description: |- + SourceKeyCount and TargetKeyCount are the per-side key counts from the + most recent diff/verification pass (reserved checkpoint key and + excluded prefixes not counted; drain-verification source reads pinned + at the drained revision with a compacted-fallback re-read). Populated + by every pass that runs regardless of spec.reconciliation.enabled — + the mandatory mark-and-sweep after any forced resync, the + OverwriteAndPrune genesis pass, and the drain verification — plus the + periodic pass when it is enabled. NOT refreshed on every status sync: + a healthy RequireEmpty mirror that never forces a resync only gets + counts from an enabled periodic pass. This is the equality signal + InvariantsHeld reads; status.cutover's copies remain the frozen + drain-time snapshot. + format: int64 + type: integer sourceRevision: description: |- SourceRevision is the source cluster's revision as of the last status - sync, for computing an approximate replication lag (SourceRevision - - LastAppliedRevision) at a glance. + sync. Cluster-global: it advances on writes outside the mirrored + prefix, so SourceRevision - LastAppliedRevision OVERSTATES lag for + prefix-scoped mirrors. See the cutover gate contract above. + format: int64 + type: integer + sourceVersion: + description: |- + SourceVersion and TargetVersion are the etcd server versions from the + maintenance Status() probe at connect. + type: string + targetClusterID: + type: string + targetKeyCount: format: int64 type: integer + targetVersion: + type: string type: object type: object served: true diff --git a/config/samples/operator_v1alpha1_etcdmirror.yaml b/config/samples/operator_v1alpha1_etcdmirror.yaml index 94b33159..3a3558ec 100644 --- a/config/samples/operator_v1alpha1_etcdmirror.yaml +++ b/config/samples/operator_v1alpha1_etcdmirror.yaml @@ -6,8 +6,13 @@ metadata: app.kubernetes.io/managed-by: kustomize name: etcdmirror-sample spec: + # Sync = continuous replication; flip to Drain at cutover time and wait for + # the CutoverReady condition. + mode: Sync + # External RKE1/AWS source reached over a public NLB, over TLS with a - # CA-verified server certificate. + # CA-verified server certificate. https:// endpoints require a tls block + # (an empty tls block would mean system trust roots). source: endpointList: - "https://etcd-rke1.example.com:2379" @@ -24,25 +29,58 @@ spec: serviceRef: name: etcdcluster-sample-client port: client - prefix: "/mirrored/registry/" + prefix: "/mirrored/" tls: secretRef: name: etcdmirror-sample-target-tls - expectEmptyPrefix: true + initialSync: + # RequireEmpty (default): refuse a non-empty destination prefix at + # genesis. Overwrite / OverwriteAndPrune for re-pointing at a + # previously-populated prefix (e.g. failback). + mode: RequireEmpty sync: - maxTxnOps: 128 - maxOpsPerSecond: 500 + # Key rewrite — ONE formula: + # key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) + # + # source key destPrefix resulting target key + # /registry/pods/x "" /mirrored/pods/x + # /registry/pods/x "registry/" /mirrored/registry/pods/x + # /registry/pods/x "backup/" /mirrored/backup/pods/x + # + # Default "" strips the source prefix. Immutable after creation. Leave + # unset for the default — an explicit destPrefix: "" is equivalent but + # pointless. + # destPrefix: "registry/" + # High-churn, lease-backed, or low-value ranges to skip entirely. + # Immutable after creation (it defines the mirrored range): change via + # delete-and-recreate with an appropriate initialSync.mode. + excludePrefixes: + - "/registry/events/" + maxTxnOps: 128 # includes one reserved slot for the checkpoint write + txnFlushBytes: 1Mi + pageKeyLimit: 512 + pageBytes: 1Mi + # Replay-buffer bound during the genesis scan; raise for high-churn + # sources (overflow restarts the scan from a fresh revision, surfaced + # as the InitialSyncCompactionRaced event). + # watchBufferBytes: 16Mi + maxOpsPerSecond: 500 # keep source retention > scan+drain time at this rate + requestTimeout: 30s dialTimeout: 10s reconnectBackoff: initialDelay: 1s maxDelay: 30s - checkpoint: - storageSpec: - volumeSizeRequest: 64Mi - syncInterval: 5s + # The checkpoint lives in the TARGET etcd, fenced and written with each + # batch. Default key: + "\x00etcdmirror-checkpoint". + # An override MUST stay under the effective destination prefix (the + # range-scoped target credential covers it; CEL-enforced) — pick a + # different suffix under it if the default collides with your own + # reserved-key scheme. + # checkpoint: + # key: "/mirrored/\x00my-own-checkpoint-name" reconciliation: enabled: true diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index 6a32be4e..b27eafb2 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -11,6 +11,8 @@ Package v1alpha1 contains API Schema definitions for the operator v1alpha1 API g ### Resource Types - [EtcdCluster](#etcdcluster) - [EtcdClusterList](#etcdclusterlist) +- [EtcdMirror](#etcdmirror) +- [EtcdMirrorList](#etcdmirrorlist) @@ -115,6 +117,444 @@ _Appears in:_ +#### EtcdMirror + + + +EtcdMirror is the Schema for the etcdmirrors API. It describes a +continuous, one-way key-range sync from a source etcd cluster to a target +etcd cluster, run as a single supervised stateless pod (a size-1 +Deployment; progress lives in a fenced checkpoint key in the target etcd, +not on a volume). + +It is a byte-copy of keys and values, not a replica: revisions, versions, +and create/mod ordering are target-assigned, and leases are stripped. See +Fidelity Caveats in docs/etcdmirror.md before depending on anything but +key/value content. + +Two-way sync: never — etcd revisions are cluster-local and there is no +per-key provenance channel, so bidirectional sync is structurally +inexpressible. Reversal for cutover/failback: yes — delete the CR and +create a new one with swapped endpoints, initialSync.mode +OverwriteAndPrune, only after the forward mirror reported CutoverReady. +See docs/etcdmirror.md. + + + +_Appears in:_ +- [EtcdMirrorList](#etcdmirrorlist) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `operator.etcd.io/v1alpha1` | | | +| `kind` _string_ | `EtcdMirror` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[EtcdMirrorSpec](#etcdmirrorspec)_ | | | | + + +#### EtcdMirrorAuth + + + +EtcdMirrorAuth configures etcd RBAC username/password auth for one side. + + + +_Appears in:_ +- [EtcdMirrorEndpoint](#etcdmirrorendpoint) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `secretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#localobjectreference-v1-core)_ | SecretRef names a Secret holding "username" and "password" keys. If
this whole Auth block is nil, the agent does not call etcd's
Authenticate() at all; the pinned v3 client transparently re-auths on
token expiry. PRECEDENCE: when both a client certificate and Auth are
supplied, etcd uses the token identity, not the certificate CN — the
Auth user must hold the range-scoped role. | | | + + +#### EtcdMirrorBackoffSpec + + + + + + + +_Appears in:_ +- [EtcdMirrorSyncSpec](#etcdmirrorsyncspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `initialDelay` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | | | Optional: \{\}
| +| `maxDelay` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | | | Optional: \{\}
| + + +#### EtcdMirrorCABundleRef + + + +EtcdMirrorCABundleRef points at one key of a Secret or ConfigMap holding a +PEM CA bundle. + + + +_Appears in:_ +- [EtcdMirrorTLS](#etcdmirrortls) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `kind` _string_ | Kind is Secret or ConfigMap. Defaults to ConfigMap. | ConfigMap | Enum: [Secret ConfigMap]
Optional: \{\}
| +| `name` _string_ | Name of the Secret or ConfigMap, in the EtcdMirror's namespace. | | MinLength: 1
| +| `key` _string_ | Key within the object. Defaults to "ca.crt". | | Optional: \{\}
| + + +#### EtcdMirrorCheckpointSpec + + + +EtcdMirrorCheckpointSpec configures the reserved checkpoint/fence key the +agent maintains IN THE TARGET etcd. The checkpoint (the source-revision +watermark plus {linkUID, epoch, role}) is written in the SAME Txn as every +applied batch and fenced with a mod-revision compare on EVERY write path +(applies, reconciliation repairs, prune deletes), so two agents can never +interleave writes and a straggler apply after cutover fails loudly. The +key is excluded by exact match from scans, counts, prune passes, and the +RequireEmpty check; the target RBAC grant must cover it; CR deletion +removes it via a delete-one-key finalizer. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `key` _string_ | Key overrides the reserved checkpoint key. Defaults to the effective
destination prefix + "\x00etcdmirror-checkpoint" — the \x00 byte after
the prefix cannot collide with any real key under it. MUST live under
the effective destination prefix (target.prefix + sync.destPrefix,
CEL-enforced): the range-scoped target credential covers it, and the
exact-match exclusion from scans/counts/prune only works inside the
mirrored range. Immutable after creation. | | Optional: \{\}
| + + +#### EtcdMirrorCutoverStatus + + + +EtcdMirrorCutoverStatus tracks a Drain-mode cutover. + + + +_Appears in:_ +- [EtcdMirrorStatus](#etcdmirrorstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `drainTargetRevision` _integer_ | DrainTargetRevision is the source revision observed when Drain was
requested — the revision the watermark must reach. | | Optional: \{\}
| +| `drainedRevision` _integer_ | DrainedRevision is the watermark at which the drain completed. | | Optional: \{\}
| +| `verifiedTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#time-v1-meta)_ | VerifiedTime is when the post-drain verification pass succeeded. | | Optional: \{\}
| +| `sourceKeyCount` _integer_ | SourceKeyCount and TargetKeyCount are the per-side key counts from the
verification pass (source read pinned at the drained revision;
reserved checkpoint key excluded). | | Optional: \{\}
| +| `targetKeyCount` _integer_ | | | Optional: \{\}
| +| `leasedKeyCount` _integer_ | LeasedKeyCount is LeaseBackedKeyCount frozen at drain completion, for
the runbook's purge/re-lease step. | | Optional: \{\}
| + + +#### EtcdMirrorDriftInfo + + + + + + + +_Appears in:_ +- [EtcdMirrorStatus](#etcdmirrorstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `missingKeys` _integer_ | MissingKeys were present on the source but absent on the target. | | | +| `divergentKeys` _integer_ | DivergentKeys were present on both sides with different values —
distinct from MissingKeys so "a resync dropped keys" is never
conflated with "a blind window went stale". | | | +| `orphanKeys` _integer_ | OrphanKeys were present on the target with no source counterpart. | | | +| `repaired` _boolean_ | Repaired is true when the pass wrote fixes rather than only reporting. | | | + + +#### EtcdMirrorEndpoint + + + +EtcdMirrorEndpoint describes how to reach, authenticate to, and scope one +side (source or target) of a mirror. Both sides need an identical shape +(address resolution + prefix + TLS + auth), so one type serves both roles, +the same way BackupDestination is reused verbatim between EtcdBackup and +EtcdRestore rather than forked into near-duplicate per-role types. + +Exactly one of EndpointList or ServiceRef must be set. An empty +endpointList ([]) is treated as unset, per Kubernetes list conventions — +so `endpointList: []` alongside a serviceRef is accepted. + +Endpoint scheme and the TLS block must agree (CEL-enforced both ways): +http:// endpoints with a tls block would silently drop TLS at dial time; +https:// endpoints without one would dial with undeclared system-roots +TLS. The agent derives the dial scheme from the presence of the tls block, +so the declared contract is true by construction. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `endpointList` _string array_ | EndpointList is a raw set of etcd client-URL host:port (or
scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379".
This is the ONLY supported mechanism for a cluster external to this
Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB);
there is deliberately no tunnel/port-forward mode — terminate any
tunnel upstream and hand EtcdMirror the resulting stable endpoint(s).
Prefer listing per-member endpoints over a single load-balancer VIP: a
TCP-health-checked VIP cannot see etcd quorum, and the client's own
balancer handles per-member failover.
IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN)
to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a
hostname present as a DNS SAN on the certificate, or ensure the
certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch;
do not reach for InsecureSkipVerify. | | Optional: \{\}
| +| `serviceRef` _[EtcdMirrorServiceRef](#etcdmirrorserviceref)_ | ServiceRef points at a Kubernetes Service in this cluster whose DNS
name resolves the etcd client endpoint(s), for the same-cluster case.
Namespace defaults to the EtcdMirror's own namespace when empty. | | Optional: \{\}
| +| `prefix` _string_ | Prefix is the etcd key prefix on THIS side. On Source, only keys under
this prefix are synced; empty means the whole keyspace. On Target, this
is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's
rewrite formula). Immutable after creation. | | Optional: \{\}
| +| `tls` _[EtcdMirrorTLS](#etcdmirrortls)_ | TLS configures the agent's client TLS for THIS side. Nil means the
agent dials this side in cleartext (and https:// endpoints are
CEL-rejected). An empty block means server-auth TLS verified against
the system trust roots. | | Optional: \{\}
| +| `auth` _[EtcdMirrorAuth](#etcdmirrorauth)_ | Auth configures etcd username/password (RBAC) auth for THIS side. | | Optional: \{\}
| + + +#### EtcdMirrorInitialSyncMode + +_Underlying type:_ _string_ + +EtcdMirrorInitialSyncMode governs how the agent treats pre-existing keys +under the effective destination prefix at genesis (first-ever sync, or a +checkpoint invalidated by a cluster-identity mismatch). + + + +_Appears in:_ +- [EtcdMirrorInitialSyncSpec](#etcdmirrorinitialsyncspec) + +| Field | Description | +| --- | --- | +| `RequireEmpty` | EtcdMirrorInitialSyncRequireEmpty refuses to start if the destination
prefix already holds any key (Phase -> Failed, condition
EmptyTargetViolation). The reserved checkpoint key is excluded by exact
match.
RE-ARM CONTRACT: a source OR target cluster-ID mismatch invalidates
the checkpoint, forces genesis, and RE-ARMS this check. An ordinary
forced resync (Compacted) does NOT re-check RequireEmpty: the decoded,
ownership-validated fence proves the destination data is this link's
own.
| +| `Overwrite` | EtcdMirrorInitialSyncOverwrite scans and writes over whatever is there.
Keys present on the target but absent on the source are left alone.
| +| `OverwriteAndPrune` | EtcdMirrorInitialSyncOverwriteAndPrune is Overwrite plus one mandatory
orphan-prune pass after the scan: target keys under the destination
prefix with no source counterpart are deleted. This makes reversal onto
a previously-populated prefix (failback) a first-class correct
operation instead of silently resurrecting deleted keys.
| + + +#### EtcdMirrorInitialSyncSpec + + + +EtcdMirrorInitialSyncSpec governs the genesis scan. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `mode` _[EtcdMirrorInitialSyncMode](#etcdmirrorinitialsyncmode)_ | Mode governs pre-existing destination keys at genesis. Defaults to
RequireEmpty (refuse a non-empty destination prefix). | RequireEmpty | Enum: [RequireEmpty Overwrite OverwriteAndPrune]
Optional: \{\}
| +| `startRevision` _integer_ | StartRevision, when > 0, skips the genesis scan entirely and starts
watching from StartRevision+1. For fidelity-preserving seeds: restore
the target from a source snapshot (`etcdutl snapshot restore
--bump-revision --mark-compacted`), then mirror only the delta.
Requires Mode Overwrite or OverwriteAndPrune (CEL-enforced). | | Minimum: 0
Optional: \{\}
| + + +#### EtcdMirrorList + + + +EtcdMirrorList contains a list of EtcdMirror. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `operator.etcd.io/v1alpha1` | | | +| `kind` _string_ | `EtcdMirrorList` | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[EtcdMirror](#etcdmirror) array_ | | | | + + +#### EtcdMirrorMode + +_Underlying type:_ _string_ + +EtcdMirrorMode selects the mirror's operating mode. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | +| --- | --- | +| `Sync` | EtcdMirrorModeSync is normal continuous replication.
| +| `Drain` | EtcdMirrorModeDrain prepares for cutover: the agent records the source
revision observed when Drain is requested (status.cutover.drainTargetRevision),
keeps replicating until the checkpoint watermark reaches it, runs a
verification pass (per-side key counts, lease-backed key count), then
sets the CutoverReady condition and flips the fence key's role to
Primary so any straggler apply fails its mod-revision compare loudly.
Runbook: quiesce source writers -> set mode=Drain ->
`kubectl wait --for=condition=CutoverReady etcdmirror/` ->
purge/re-lease lease-backed keys -> repoint clients -> delete the CR.
| + + +#### EtcdMirrorPhase + +_Underlying type:_ _string_ + +EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. +Unlike BackupPhase/RestorePhase, most phases here are NOT terminal — a +healthy mirror spends its life in Syncing; there is no "Completed" state. + + + +_Appears in:_ +- [EtcdMirrorStatus](#etcdmirrorstatus) + +| Field | Description | +| --- | --- | +| `Pending` | EtcdMirrorPhasePending means the EtcdMirror has been accepted but the
agent workload has not been created yet.
| +| `Connecting` | EtcdMirrorPhaseConnecting means the agent pod is running, establishing
client connections to both sides and probing versions/cluster IDs.
| +| `InitialSync` | EtcdMirrorPhaseInitialSync means the agent is running the genesis scan:
an UNPINNED chunked scan with the watch already open from the revision
observed before the scan started, buffered events replayed over the
scanned base (reflector pattern). Because pages read at the current
revision, mid-scan compaction cannot fail the scan. Also entered during
a forced resync (then with condition Compacted=True/Reason=ForcedResync).
| +| `Syncing` | EtcdMirrorPhaseSyncing is the steady state: watching and applying live
changes, watermark advancing via progress notifications.
| +| `Degraded` | EtcdMirrorPhaseDegraded means the agent is in a retry/backoff loop
(connection or throttling class) and is expected to self-heal. Forced
resyncs are NOT Degraded; they report as InitialSync + Compacted=True.
| +| `Paused` | EtcdMirrorPhasePaused means spec.paused is true; the agent Deployment
is scaled to zero. The checkpoint is retained in the target.
| +| `Failed` | EtcdMirrorPhaseFailed means a terminal, non-recoverable error requiring
operator intervention: EmptyTargetViolation at genesis, source below
the 3.4 version floor (UnsupportedVersion), a permanent-class write
error (oversized revision vs target limits), malformed cert material,
or unresolvable spec misconfiguration.
| + + +#### EtcdMirrorReconciliationSpec + + + +EtcdMirrorReconciliationSpec configures the periodic full reconciliation +pass. The same engine also runs unconditionally (regardless of Enabled or +DeleteOrphans) as the post-forced-resync mark-and-sweep, the +OverwriteAndPrune genesis pass, and the Drain verification pass. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `enabled` _boolean_ | Enabled toggles the PERIODIC pass. Defaults to false: it is a full
diff of the prefix contents on both sides (O(keyspace)), so it is
opt-in. | | Optional: \{\}
| +| `interval` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | Interval between periodic passes. Defaults to 1h when Enabled. | | Optional: \{\}
| +| `deleteOrphans` _boolean_ | DeleteOrphans, when true, allows the PERIODIC pass to delete target
keys under the destination prefix that have no corresponding source
key. Defaults to false. (Forced-resync sweeps and OverwriteAndPrune
always delete orphans; this knob only governs the periodic pass.) | | Optional: \{\}
| + + +#### EtcdMirrorServiceRef + + + +EtcdMirrorServiceRef points at a Service and the client port on it to dial. + + + +_Appears in:_ +- [EtcdMirrorEndpoint](#etcdmirrorendpoint) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name is the Service name. | | MinLength: 1
| +| `namespace` _string_ | Namespace defaults to the EtcdMirror's own namespace when empty. | | Optional: \{\}
| +| `port` _string_ | Port is the Service port name or number exposing etcd's client API.
Defaults to "client" when empty, matching PodMonitorSpec.Port's
convention elsewhere in this API group. | | Optional: \{\}
| + + +#### EtcdMirrorSpec + + + +EtcdMirrorSpec defines the desired state of an EtcdMirror. + +Range-defining and rewrite fields are immutable (CEL transition rules +below): source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes +and checkpoint.key. Changing what range is mirrored, or where it lands, +mid-life silently diverges: a restarted agent resumes from its checkpoint +without a scan, so removing an exclusion never backfills pre-existing keys +and adding one strands already-mirrored keys as permanent orphans — all +with every condition green. Endpoints stay mutable — rotating an NLB DNS +name or adding a member to the same cluster is routine; pointing at a +different cluster is caught at runtime by the checkpoint's dual-cluster-ID +binding, not by spec validation. + +The transition rules compare VALUES, presence-normalized: for these +fields the empty value and an absent field are semantically identical +(prefix "" = whole keyspace, destPrefix "" = strip the source prefix), and +Go typed clients drop explicit "" through omitempty — presence-based rules +would reject every typed-client update of a CR created with an explicit +empty string. + + + +_Appears in:_ +- [EtcdMirror](#etcdmirror) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `mode` _[EtcdMirrorMode](#etcdmirrormode)_ | Mode selects continuous replication (Sync, the default) or a cutover
drain (Drain). See EtcdMirrorModeDrain for the cutover contract. | Sync | Enum: [Sync Drain]
Optional: \{\}
| +| `source` _[EtcdMirrorEndpoint](#etcdmirrorendpoint)_ | Source is the etcd cluster keys are read from. EtcdMirror never writes
back to Source; the agent's source-side client is only ever used for
Get/Watch, never Put/Delete/Txn.
VERSION FLOOR: source etcd must be >= 3.4 (probed via maintenance
Status() at connect; below the floor the mirror goes Failed with reason
UnsupportedVersion). >= 3.4.25 / 3.5.8 is the recommended floor: below
it, watch progress notifications are unreliable and the agent cannot
trust the watermark machinery that drives lag, the checkpoint, and the
Drain gate. | | | +| `target` _[EtcdMirrorEndpoint](#etcdmirrorendpoint)_ | Target is the etcd cluster keys are written into.
SECURITY PREREQUISITE: Target's credential (etcd RBAC user and/or the
client certificate's role) MUST be range-scoped to the effective
destination prefix — never a cluster-admin-equivalent credential. The
agent's client-side rewrite logic is defense against bugs in its own
code, NOT a security boundary. The grant must also cover the reserved
checkpoint key (see Checkpoint). Configure via `etcdctl role
grant-permission --prefix=true readwrite ` before
pointing an EtcdMirror at the cluster.
The target must run with auto-compaction enabled: forced-resync churn
and prune passes march an uncompacted target toward its storage quota
(2GiB by default), which surfaces as TargetQuotaExhausted. | | | +| `initialSync` _[EtcdMirrorInitialSyncSpec](#etcdmirrorinitialsyncspec)_ | InitialSync governs genesis behavior: how pre-existing destination keys
are treated (Mode) and optionally where replication starts
(StartRevision). | | Optional: \{\}
| +| `sync` _[EtcdMirrorSyncSpec](#etcdmirrorsyncspec)_ | Sync tunes runtime sync behavior (batching, paging, rate limiting,
prefix rewrite, timeouts, backoff). | | Optional: \{\}
| +| `checkpoint` _[EtcdMirrorCheckpointSpec](#etcdmirrorcheckpointspec)_ | Checkpoint configures the reserved checkpoint/fence key on the target. | | Optional: \{\}
| +| `reconciliation` _[EtcdMirrorReconciliationSpec](#etcdmirrorreconciliationspec)_ | Reconciliation optionally enables a periodic full diff-and-repair pass
layered on top of the continuous watch-based mirror. Independent of
this setting, one reconciliation-with-delete pass always runs after any
forced resync (mark-and-sweep), and as the OverwriteAndPrune genesis
pass and the Drain verification pass. | | Optional: \{\}
| +| `podTemplate` _[PodTemplate](#podtemplate)_ | PodTemplate carries scheduling/affinity/labels/annotations for the
agent pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. | | Optional: \{\}
| +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#resourcerequirements-v1-core)_ | Resources are the agent container's compute resources. The agent's
memory model is bounded by Sync.PageBytes (single in-flight scan page,
no unbounded read-ahead); size limits accordingly. | | Optional: \{\}
| +| `paused` _boolean_ | Paused, when true, scales the agent Deployment to zero without deleting
the CR or its checkpoint. The checkpoint lives in the target etcd, so
resume picks up from the last fenced watermark. NOTE: pausing longer
than the source's compaction retention guarantees a full forced resync
on resume — there is no free lunch past the retention window. | | Optional: \{\}
| + + + + +#### EtcdMirrorSyncSpec + + + +EtcdMirrorSyncSpec tunes the mirror's runtime sync behavior. + +KEY REWRITE — one formula, no other composition: + + key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) + +(anchored strip-and-reprefix; never a substring replace). + +BATCHING INVARIANT: target Txns flush ONLY at source-revision boundaries — +a source revision's events are never split across Txns, whole revisions +are coalesced up to the MaxTxnOps/TxnFlushBytes watermarks, and one op +slot in MaxTxnOps is always reserved for the checkpoint write that rides +in the same Txn. A single source revision larger than MaxTxnOps is applied +as one oversized Txn (provision the target's --max-txn-ops accordingly) +with the checkpoint held until it lands. + +RETENTION PREREQUISITE: the source's compaction retention window must +exceed the worst-case initial scan + throttled drain time, approximately +sourceKeyCount / min(effective scan rate, MaxOpsPerSecond). If it does +not, genesis (and every forced resync) loses the race with compaction and +the mirror livelocks (surfaced via the resync-loop detector). + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `destPrefix` _string_ | DestPrefix is the middle term of the rewrite formula above. Default ""
means the source prefix is stripped and key remainders land directly
under target.prefix. Immutable after creation. | | Optional: \{\}
| +| `excludePrefixes` _string array_ | ExcludePrefixes lists source key prefixes (full source-side keys, e.g.
"/registry/events/") skipped entirely: not scanned, not watched, not
counted, not pruned. Use to drop high-churn low-value ranges and cut
WAN cost, or to skip lease-backed ranges that don't survive mirroring.
Nested/duplicate entries are normalized by the agent (an entry covered
by another is dropped). Immutable after creation (it defines the
mirrored range): removing an exclusion would require a backfill scan
the checkpoint-resume path never runs, and adding one would strand
already-mirrored keys as permanent orphans — change it via
delete-and-recreate with an appropriate initialSync.mode instead. | | MaxItems: 64
Optional: \{\}
| +| `maxTxnOps` _integer_ | MaxTxnOps bounds how many operations the agent batches into a single
target Txn, including the reserved checkpoint-write slot. Must not
exceed the target's --max-txn-ops (etcd default 128). Defaults to 128. | | Minimum: 2
Optional: \{\}
| +| `txnFlushBytes` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#quantity-resource-api)_ | TxnFlushBytes is the byte watermark at which a batch is flushed (at the
next source-revision boundary). Keep well under etcd's request size
limits: a Txn over ~1.5MiB is rejected by the server and one over 2MiB
by the client send cap — both classified permanent errors, not
throttling. Defaults to 1Mi. | | Optional: \{\}
| +| `pageKeyLimit` _integer_ | PageKeyLimit bounds keys per source scan page during InitialSync and
reconciliation. The scan is pull-based, one page in flight — no
read-ahead — so this and PageBytes bound agent memory. Defaults to 512. | | Minimum: 1
Optional: \{\}
| +| `pageBytes` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#quantity-resource-api)_ | PageBytes bounds bytes per source scan page. Defaults to 1Mi. | | Optional: \{\}
| +| `watchBufferBytes` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#quantity-resource-api)_ | WatchBufferBytes bounds the memory used to buffer watch events
observed from R0 while the genesis scan runs (the reflector replay
buffer). On overflow the agent cancels the source watch and restarts
the scan from a fresh R0 (see the InitialSyncCompactionRaced event) —
a bounded retry instead of unbounded growth when source churn outruns
scan+apply throughput. Defaults to 16Mi (must stay in lockstep with
pkg/mirroragent's DefaultWatchBufferBytes). | | Optional: \{\}
| +| `maxOpsPerSecond` _integer_ | MaxOpsPerSecond rate-limits the agent's target write rate (a token
bucket over puts+deletes/sec), applied to both the genesis scan and
watch-driven applies. Zero (default) means unlimited. Mind the
retention prerequisite above when throttling. | | Minimum: 0
Optional: \{\}
| +| `requestTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | RequestTimeout is the per-RPC context deadline applied to every unary
call on both sides (watches excluded — they are long-lived by design
and covered by progress-notification liveness instead). Without it a
blackholed call through an NLB never errors and backoff never engages.
Defaults to 30s. | | Optional: \{\}
| +| `dialTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | DialTimeout bounds establishing the initial client connection to each
side. Defaults to 10s. | | Optional: \{\}
| +| `reconnectBackoff` _[EtcdMirrorBackoffSpec](#etcdmirrorbackoffspec)_ | ReconnectBackoff bounds the retry/backoff loop wrapping connection-class
errors. Throttling-class errors (target rate rejection) use a more
conservative curve derived from the same bounds; quota exhaustion
(TargetQuotaExhausted) and permanent errors are never retried through
this loop. Defaults to exponential backoff from 1s to 30s. | | Optional: \{\}
| + + +#### EtcdMirrorTLS + + + +EtcdMirrorTLS configures the mirror agent's client TLS for one side of a +mirror. Plain secretRef, not a reuse of EtcdClusterTLS/TLSSurface — the +agent is always a client to clusters it does not own, so issuer-selection +machinery doesn't apply. + +ROTATION CONTRACT: the agent re-reads TLS material from the mounted Secret +on every handshake (transport.TLSInfo file paths, not a one-shot +tls.Config); certificate rotation requires no pod restart. + + + +_Appears in:_ +- [EtcdMirrorEndpoint](#etcdmirrorendpoint) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `secretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#localobjectreference-v1-core)_ | SecretRef names a Secret (in the EtcdMirror's namespace) holding this
side's TLS material in the standard kubernetes.io/tls-compatible shape:
- ca.crt: PEM CA bundle used to verify the peer's server certificate
(unless CABundleRef overrides it, or InsecureSkipVerify is true).
- tls.crt / tls.key: PEM client certificate + key, for mTLS.
Optional — omit both for server-auth-only TLS.
Nil means no client identity and verification against the system trust
roots (the etcdctl default). Note a source running with
--client-cert-auth (the RKE1 default) rejects certless clients at the
handshake regardless of etcd RBAC auth; server-auth-only + Auth is not
viable against such a source. | | Optional: \{\}
| +| `caBundleRef` _[EtcdMirrorCABundleRef](#etcdmirrorcabundleref)_ | CABundleRef optionally sources the trust anchors from a separate
Secret or ConfigMap key, decoupling trust from the identity Secret
(Gateway API caCertificateRefs precedent). Takes precedence over
SecretRef's ca.crt. | | Optional: \{\}
| +| `insecureSkipVerify` _boolean_ | InsecureSkipVerify disables server certificate verification. Strongly
discouraged, especially for a source reached over the public internet.
Requires InsecureSkipVerifyAcknowledgeRisk to also be set true
(CEL-enforced). The controller additionally emits a standing Warning
event whenever this is true. | false | Optional: \{\}
| +| `insecureSkipVerifyAcknowledgeRisk` _boolean_ | InsecureSkipVerifyAcknowledgeRisk must independently be set true
whenever InsecureSkipVerify is true (CEL-enforced companion field). Its
only purpose is to require a deliberate, separate, reviewable line in
the manifest diff before disabling TLS verification. | false | Optional: \{\}
| +| `serverName` _string_ | ServerName overrides the TLS ServerName (SNI) used for verification,
for cases where the dialed address doesn't match a SAN on the
certificate (e.g. dialing an NLB IP directly). Applies to EVERY
endpoint in the list, so mixing endpoints with different certificates
behind one ServerName will fail verification on the mismatched ones. | | Optional: \{\}
| + + #### MemberStatus @@ -153,6 +593,24 @@ _Appears in:_ | `labels` _object (keys:string, values:string)_ | | | | +#### PodSpec + + + + + + + +_Appears in:_ +- [PodTemplate](#podtemplate) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#affinity-v1-core)_ | | | | +| `nodeSelector` _object (keys:string, values:string)_ | | | | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#toleration-v1-core) array_ | | | | + + #### PodTemplate @@ -163,10 +621,12 @@ _Appears in:_ _Appears in:_ - [EtcdClusterSpec](#etcdclusterspec) +- [EtcdMirrorSpec](#etcdmirrorspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `metadata` _[PodMetadata](#podmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PodSpec](#podspec)_ | | | | #### ProviderAutoConfig diff --git a/docs/etcdmirror.md b/docs/etcdmirror.md new file mode 100644 index 00000000..2c754511 --- /dev/null +++ b/docs/etcdmirror.md @@ -0,0 +1,141 @@ +# EtcdMirror + +`EtcdMirror` continuously copies a key range from a source etcd cluster into a +target etcd cluster, one way, as a single supervised stateless pod. Progress is +checkpointed in a reserved, fenced key **in the target etcd** — written in the +same transaction as every applied batch — so the agent has no volume and can be +rescheduled freely. + +It is a **byte-copy of keys and values, not a replica**. If you need a replica, +add members to the cluster; if you need a point-in-time copy with revision +fidelity, use `etcdutl snapshot restore`. + +## What is and is not preserved + +| Property | Preserved? | Notes | +| --- | --- | --- | +| Key names | yes | rewritten by one formula: `key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix)` | +| Values | yes | byte-identical | +| Per-revision atomicity | yes | batches flush only at source-revision boundaries; a source revision is never split across target Txns | +| Revisions / mod_revision / create_revision | **no** | target-assigned. Stored fence tokens, persisted watch bookmarks, and CreateRevision-ordered elections do not survive mirroring | +| Version counters | **no** | target-assigned | +| Leases / TTLs | **no** | leases are stripped; a lease-backed source key becomes a permanent target key. Masked while the mirror runs (source expiry replicates as a delete); at cutover every in-flight leased key is immortal. `status.leaseBackedKeyCount` reports exposure; the cutover runbook includes a purge/re-lease step. Consider `sync.excludePrefixes` for lease-heavy ranges | +| Cross-key Txn atomicity | best-effort | whole revisions are coalesced; a revision larger than `maxTxnOps` is applied as one oversized Txn (provision the target's `--max-txn-ops`) | + +A fidelity-preserving alternative for migrations: seed the target with +`etcdutl snapshot restore --bump-revision --mark-compacted`, then mirror only +the delta using `initialSync.startRevision`. + +## Sync engine behavior + +- **Genesis (InitialSync):** unpinned chunked scan (one byte-bounded page in + flight) with the watch already open from the revision observed before the + scan; buffered events replay over the scanned base. Mid-scan compaction on + the source therefore cannot fail the scan. +- **Steady state:** watch with progress notifications; the checkpoint watermark + advances even when the mirrored prefix is idle. +- **Forced resync** (watch outrun by compaction — e.g. the mirror was down or + paused longer than the source's compaction retention): reported as + `Phase=InitialSync` with condition `Compacted=True/ForcedResync`, bracketed by + `ForcedResyncStarted`/`ForcedResyncCompleted` events. Every forced resync ends + with a mandatory mark-and-sweep prune, so deletes that happened during the + blind window do not resurrect. +- **Retention prerequisite:** source compaction retention must exceed the + worst-case scan + throttled drain time (roughly + `keyCount / min(scanRate, maxOpsPerSecond)`), or genesis/forced resyncs + livelock — surfaced as `ResyncLoopDetected`, which does not self-heal. +- **Checkpoint fencing:** the reserved key carries `{linkUID, epoch, role}` and + every write path (applies, reconciliation repairs, prune deletes) is fenced + with a mod-revision compare, so two agents can never interleave and a + straggler apply after cutover fails loudly. On a failed compare the engine + re-reads the fence — a Txn that committed while its response was lost (WAN + timeout) is recognized as this agent's own write and adopted, never + misreported as a fence violation. +- **Destination overlap guard:** a prune pass that finds ANOTHER link's + reserved fence key inside this link's destination prefix stops with a + permanent prefix-conflict error instead of deleting the sibling mirror's + fence and data. + +## Monitoring / paging algebra + +Page on `Available=False` sustained, **unless** `Compacted=True` and progress +fields are advancing (a forced resync healing itself). `TargetQuotaExhausted` +and `ResyncLoopDetected` page immediately — neither self-heals. + +Never compute lag as `status.sourceRevision - status.lastAppliedRevision`: +revisions are cluster-global, so out-of-prefix source writes inflate the +difference, and the two fields snapshot at different instants. + +The `InitialSyncCompactionRaced` event marks a genesis-scan attempt aborted +and restarted from a fresh revision. One event name, two causes named in the +message — do not conflate them: + +- `WatchBufferOverflow`: the replay buffer exceeded `sync.watchBufferBytes` + before the base scan completed. A memory-bound retry, **not** a compaction + race — raise `watchBufferBytes` or the scan rate for high-churn sources. +- `WatchCompactedMidScan`: a watch reconnect landed below the source compact + revision — the rare genuine race. + +Repeated occurrences of either count toward `ResyncLoopDetected`. + +`InvariantsHeld=True` means the verification invariants hold (lag within +threshold, per-side key counts equal, no drift, pass fresh) — it never means +"safe to cut over"; that is `CutoverReady`, which additionally requires +`spec.mode: Drain` and a reached drain target revision. + +## Operations: error taxonomy and runbook + +| Error | gRPC code | Class | Retry policy | Condition / Reason | +| --- | --- | --- | --- | --- | +| `ErrCompacted` on watch reopen | OutOfRange | Resync | forced resync (scan + mandatory prune); never generic retry | `Compacted=True/ForcedResync`; `forcedResyncCount`++ (`Compacted`) | +| `ErrNoSpace` | ResourceExhausted | Quota | park on slow flat timer; never hot-loop; recovers without genesis once operator compacts/defrags/disarms | `TargetQuotaExhausted=True` (pages immediately) | +| client send cap ("trying to send message larger than max") | ResourceExhausted | Permanent | never retried identically; redacted key surfaced | Failed if unavoidable | +| `ErrTooManyRequests` | ResourceExhausted | Throttle | conservative distinct curve | `TargetThrottled=True` | +| `ErrRequestTooLarge` | InvalidArgument | Permanent | one shrink attempt at revision granularity, else Failed | redacted key surfaced | +| `ErrTooManyOps` | InvalidArgument | Permanent | one shrink attempt at revision granularity; else raise target `--max-txn-ops` or lower `spec.sync.maxTxnOps` | Failed | +| Unavailable / `ErrNoLeader` | Unavailable | Transient | ReconnectBackoff curve | Source/TargetReachable=False (reason NoLeader when applicable) | +| DeadlineExceeded (requestTimeout) | DeadlineExceeded | Transient | ReconnectBackoff — the blackholed-NLB recovery path | reachability, reason RequestTimeout | +| auth token expiry | — | non-issue | clientv3 refreshes transparently | none | +| source version < 3.4 | — | Permanent | never | Failed/`UnsupportedVersion` | +| corrupt/unknown-version checkpoint | — | Permanent (fail closed) | never; operator deletes reserved key | Failed/`CheckpointInvalid` | +| linkUID / cluster-ID mismatch | — | expected transition | genesis + RequireEmpty re-arm | `CheckpointInvalidated` event | +| fence Compare loss (normal apply) | — | optimistic-concurrency loss | re-read, recompute, retry (jitter, not reconnect curve) | internal; persistent genesis-claim loss → FenceError (permanent) | +| N consecutive resyncs, no steady period | — | livelock (meta) | does not self-heal | `ResyncLoopDetected=True` (pages immediately) | + +## Cutover and reversal + +Two-way sync is out of scope permanently: etcd revisions are cluster-local and +there is no per-key provenance channel, so bidirectional sync is structurally +inexpressible without an application-visible format change. + +Cutover (forward): quiesce source writers, set `spec.mode: Drain`, then +`kubectl wait --for=condition=CutoverReady etcdmirror/`. The +`status.cutover` block records the drained revision, verification counts, and +the lease-backed key count for the purge/re-lease step. Once CutoverReady, the +fence key's role is Primary and any straggler mirror write fails its compare. + +Reversal (failback) is delete-and-recreate: delete the CR, create a new one +with swapped endpoints and `initialSync.mode: OverwriteAndPrune` — the +mandatory prune pass removes keys deleted on the new primary since cutover. +Only reverse after the forward mirror reached CutoverReady. + +## Prerequisites (summary) + +- Source etcd >= 3.4 (hard floor, probed at connect); >= 3.4.25 / 3.5.8 + recommended — below that, watch progress notifications are unreliable. +- Target credential range-scoped via etcd RBAC to the effective destination + prefix, **including the reserved checkpoint key** + (default `\x00etcdmirror-checkpoint`; an + override must stay under the effective destination prefix — CEL-enforced). +- Range-defining fields (`source.prefix`, `target.prefix`, `sync.destPrefix`, + `sync.excludePrefixes`, `checkpoint.key`) are immutable: the resume path + never re-scans, so an edited range silently diverges. Change them via + delete-and-recreate with an appropriate `initialSync.mode`. +- Target runs with auto-compaction enabled (resync churn otherwise marches the + default 2GiB quota toward NOSPACE / `TargetQuotaExhausted`). +- Source compaction retention satisfies the formula above for your key count + and rate limit. +- A source running `--client-cert-auth` (RKE1 default) rejects certless + clients at the handshake: supply a client certificate; username/password + auth alone is not viable there. When both are supplied, the token identity + wins and must hold the range-scoped role. diff --git a/internal/controller/etcdmirror_cel_test.go b/internal/controller/etcdmirror_cel_test.go index 70fcf476..56467c5c 100644 --- a/internal/controller/etcdmirror_cel_test.go +++ b/internal/controller/etcdmirror_cel_test.go @@ -23,17 +23,20 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/controller-runtime/pkg/client" ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" ) // validSourceEndpoint and validTargetEndpoint are baseline endpoints that -// satisfy EtcdMirrorEndpoint's own oneOf rule, so each test case only needs -// to perturb the field(s) it's actually exercising. +// satisfy EtcdMirrorEndpoint's own CEL rules (oneOf, scheme-vs-TLS), so each +// test case only needs to perturb the field(s) it's actually exercising. The +// source endpoint is deliberately scheme-less so TLS blocks can be added or +// removed freely without tripping the scheme rules. func validSourceEndpoint() ecv1alpha1.EtcdMirrorEndpoint { return ecv1alpha1.EtcdMirrorEndpoint{ - EndpointList: []string{"https://etcd-source.example.com:2379"}, + EndpointList: []string{"etcd-source.example.com:2379"}, Prefix: "/registry/", } } @@ -45,68 +48,33 @@ func validTargetEndpoint() ecv1alpha1.EtcdMirrorEndpoint { } } -// TestEtcdMirrorSyncPrefixCELValidation drives the sync.destPrefix / -// sync.noDestPrefix mutual-exclusion XValidation rule on EtcdMirrorSpec -// against the real envtest apiserver. -func TestEtcdMirrorSyncPrefixCELValidation(t *testing.T) { - if k8sClient == nil { - t.Skip("envtest apiserver not available") - } - - tests := []struct { - name string - sync ecv1alpha1.EtcdMirrorSyncSpec - wantApply bool - }{ - { - name: "neither destPrefix nor noDestPrefix set accepted", - sync: ecv1alpha1.EtcdMirrorSyncSpec{}, - wantApply: true, - }, - { - name: "destPrefix alone accepted", - sync: ecv1alpha1.EtcdMirrorSyncSpec{DestPrefix: "/other/"}, - wantApply: true, - }, - { - name: "noDestPrefix alone accepted", - sync: ecv1alpha1.EtcdMirrorSyncSpec{NoDestPrefix: true}, - wantApply: true, - }, - { - name: "destPrefix and noDestPrefix together rejected", - sync: ecv1alpha1.EtcdMirrorSyncSpec{DestPrefix: "/other/", NoDestPrefix: true}, - wantApply: false, +func newMirror(prefix string, spec ecv1alpha1.EtcdMirrorSpec) *ecv1alpha1.EtcdMirror { + return &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: prefix, + Namespace: "default", }, + Spec: spec, } +} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - em := &ecv1alpha1.EtcdMirror{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "cel-mirror-syncprefix-", - Namespace: "default", - }, - Spec: ecv1alpha1.EtcdMirrorSpec{ - Source: validSourceEndpoint(), - Target: validTargetEndpoint(), - Sync: tt.sync, - }, - } - err := k8sClient.Create(t.Context(), em) - if tt.wantApply { - require.NoError(t, err, "apiserver should accept a valid sync spec") - _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) - } else { - assert.Error(t, err, "apiserver should reject destPrefix+noDestPrefix via CEL") - } - }) +// createAndCheck applies the mirror and asserts admission matched wantApply, +// deleting it again on success. +func createAndCheck(t *testing.T, em *ecv1alpha1.EtcdMirror, wantApply bool, msg string) { + t.Helper() + err := k8sClient.Create(t.Context(), em) + if wantApply { + require.NoError(t, err, msg) + _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) + } else { + assert.Error(t, err, msg) } } // TestEtcdMirrorEndpointOneOfCELValidation drives the endpointList/serviceRef // exactly-one-of XValidation rule on EtcdMirrorEndpoint, exercised on both -// Source and Target. +// Source and Target. An empty endpointList is deliberately treated as unset +// (k8s list conventions), so [] alongside a serviceRef must be ACCEPTED. func TestEtcdMirrorEndpointOneOfCELValidation(t *testing.T) { if k8sClient == nil { t.Skip("envtest apiserver not available") @@ -114,10 +82,14 @@ func TestEtcdMirrorEndpointOneOfCELValidation(t *testing.T) { neither := ecv1alpha1.EtcdMirrorEndpoint{Prefix: "/x/"} both := ecv1alpha1.EtcdMirrorEndpoint{ - EndpointList: []string{"https://etcd.example.com:2379"}, + EndpointList: []string{"etcd.example.com:2379"}, + ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-client"}, + } + emptyListOnly := ecv1alpha1.EtcdMirrorEndpoint{EndpointList: []string{}} + emptyListWithServiceRef := ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{}, ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-client"}, } - emptyList := ecv1alpha1.EtcdMirrorEndpoint{EndpointList: []string{}} tests := []struct { name string @@ -145,10 +117,18 @@ func TestEtcdMirrorEndpointOneOfCELValidation(t *testing.T) { }, { name: "source with empty endpointList and no serviceRef rejected", - source: emptyList, + source: emptyListOnly, target: validTargetEndpoint(), wantApply: false, }, + { + // Pinned deliberately: empty list == unset, so this is the + // serviceRef-only case, not the both-set case. + name: "source with empty endpointList plus serviceRef accepted", + source: emptyListWithServiceRef, + target: validTargetEndpoint(), + wantApply: true, + }, { name: "target with neither endpointList nor serviceRef rejected", source: validSourceEndpoint(), @@ -165,25 +145,108 @@ func TestEtcdMirrorEndpointOneOfCELValidation(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - em := &ecv1alpha1.EtcdMirror{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "cel-mirror-endpoint-", - Namespace: "default", - }, - Spec: ecv1alpha1.EtcdMirrorSpec{ - Source: tt.source, - Target: tt.target, - }, - } - err := k8sClient.Create(t.Context(), em) - if tt.wantApply { - require.NoError(t, err, "apiserver should accept exactly-one-of endpointList/serviceRef") - _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) - } else { - assert.Error(t, err, "apiserver should reject endpointList/serviceRef oneOf violation via CEL") + em := newMirror("cel-mirror-endpoint-", ecv1alpha1.EtcdMirrorSpec{ + Source: tt.source, + Target: tt.target, + }) + createAndCheck(t, em, tt.wantApply, "endpointList/serviceRef oneOf") + }) + } +} + +// TestEtcdMirrorEndpointSchemeTLSCELValidation drives the two scheme-vs-TLS +// XValidation rules on EtcdMirrorEndpoint: http:// forbids a tls block, +// https:// requires one (an empty tls block means system trust roots). +func TestEtcdMirrorEndpointSchemeTLSCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tlsBlock := &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: &corev1.LocalObjectReference{Name: "etcd-mirror-tls"}, + } + + tests := []struct { + name string + endpoints []string + tls *ecv1alpha1.EtcdMirrorTLS + wantApply bool + }{ + { + name: "https endpoint with tls block accepted", + endpoints: []string{"https://etcd.example.com:2379"}, + tls: tlsBlock, + wantApply: true, + }, + { + name: "https endpoint with empty tls block (system roots) accepted", + endpoints: []string{"https://etcd.example.com:2379"}, + tls: &ecv1alpha1.EtcdMirrorTLS{}, + wantApply: true, + }, + { + name: "https endpoint without tls block rejected", + endpoints: []string{"https://etcd.example.com:2379"}, + tls: nil, + wantApply: false, + }, + { + name: "http endpoint without tls block accepted", + endpoints: []string{"http://etcd.example.com:2379"}, + tls: nil, + wantApply: true, + }, + { + name: "http endpoint with tls block rejected", + endpoints: []string{"http://etcd.example.com:2379"}, + tls: tlsBlock, + wantApply: false, + }, + { + name: "mixed http and https endpoints with tls block rejected", + endpoints: []string{"https://a.example.com:2379", "http://b.example.com:2379"}, + tls: tlsBlock, + wantApply: false, + }, + { + name: "scheme-less endpoint with tls block accepted", + endpoints: []string{"etcd.example.com:2379"}, + tls: tlsBlock, + wantApply: true, + }, + { + name: "scheme-less endpoint without tls block accepted", + endpoints: []string{"etcd.example.com:2379"}, + tls: nil, + wantApply: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: tt.endpoints, + Prefix: "/registry/", + TLS: tt.tls, } + em := newMirror("cel-mirror-scheme-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "scheme-vs-TLS rule on source") }) } + + t.Run("scheme rules also apply to target", func(t *testing.T) { + em := newMirror("cel-mirror-scheme-", ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"https://etcd-target.example.com:2379"}, + Prefix: "/mirrored/", + }, + }) + createAndCheck(t, em, false, "https target without tls must be rejected") + }) } // TestEtcdMirrorTLSInsecureSkipVerifyCELValidation drives the @@ -194,7 +257,7 @@ func TestEtcdMirrorTLSInsecureSkipVerifyCELValidation(t *testing.T) { t.Skip("envtest apiserver not available") } - secretRef := corev1.LocalObjectReference{Name: "etcd-mirror-tls"} + secretRef := &corev1.LocalObjectReference{Name: "etcd-mirror-tls"} tests := []struct { name string @@ -220,6 +283,14 @@ func TestEtcdMirrorTLSInsecureSkipVerifyCELValidation(t *testing.T) { }, wantApply: true, }, + { + name: "insecureSkipVerify with acknowledgement and no secretRef accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{ + InsecureSkipVerify: true, + InsecureSkipVerifyAcknowledgeRisk: true, + }, + wantApply: true, + }, { name: "insecureSkipVerify without acknowledgement rejected", tls: &ecv1alpha1.EtcdMirrorTLS{ @@ -242,31 +313,19 @@ func TestEtcdMirrorTLSInsecureSkipVerifyCELValidation(t *testing.T) { t.Run(tt.name, func(t *testing.T) { source := validSourceEndpoint() source.TLS = tt.tls - em := &ecv1alpha1.EtcdMirror{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "cel-mirror-tls-", - Namespace: "default", - }, - Spec: ecv1alpha1.EtcdMirrorSpec{ - Source: source, - Target: validTargetEndpoint(), - }, - } - err := k8sClient.Create(t.Context(), em) - if tt.wantApply { - require.NoError(t, err, "apiserver should accept a valid TLS block") - _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) - } else { - assert.Error(t, err, "apiserver should reject insecureSkipVerify without acknowledgement via CEL") - } + em := newMirror("cel-mirror-tls-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "insecureSkipVerify companion rule") }) } } -// TestEtcdMirrorSecretRefNameRequiredCELValidation drives the -// "secretRef.name is required" XValidation rule shared by EtcdMirrorTLS and -// EtcdMirrorAuth, exercised on both the TLS and Auth blocks. -func TestEtcdMirrorSecretRefNameRequiredCELValidation(t *testing.T) { +// TestEtcdMirrorSecretRefCELValidation drives the secretRef rules: on TLS the +// secretRef is optional (nil = system trust roots) but must have a non-empty +// name when present; on Auth it is required with a non-empty name. +func TestEtcdMirrorSecretRefCELValidation(t *testing.T) { if k8sClient == nil { t.Skip("envtest apiserver not available") } @@ -279,12 +338,17 @@ func TestEtcdMirrorSecretRefNameRequiredCELValidation(t *testing.T) { }{ { name: "TLS secretRef with non-empty name accepted", - tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: corev1.LocalObjectReference{Name: "etcd-mirror-tls"}}, + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: &corev1.LocalObjectReference{Name: "etcd-mirror-tls"}}, + wantApply: true, + }, + { + name: "TLS with nil secretRef (system trust roots) accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{}, wantApply: true, }, { name: "TLS secretRef with empty name rejected", - tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: corev1.LocalObjectReference{Name: ""}}, + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: &corev1.LocalObjectReference{Name: ""}}, wantApply: false, }, { @@ -304,23 +368,408 @@ func TestEtcdMirrorSecretRefNameRequiredCELValidation(t *testing.T) { source := validSourceEndpoint() source.TLS = tt.tls source.Auth = tt.auth - em := &ecv1alpha1.EtcdMirror{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "cel-mirror-secretref-", - Namespace: "default", - }, - Spec: ecv1alpha1.EtcdMirrorSpec{ - Source: source, - Target: validTargetEndpoint(), - }, - } - err := k8sClient.Create(t.Context(), em) - if tt.wantApply { - require.NoError(t, err, "apiserver should accept a non-empty secretRef.name") - _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) + em := newMirror("cel-mirror-secretref-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "secretRef name rules") + }) + } +} + +// TestEtcdMirrorCABundleRefValidation drives EtcdMirrorCABundleRef's schema +// validation (kind enum, required name). +func TestEtcdMirrorCABundleRefValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + ref *ecv1alpha1.EtcdMirrorCABundleRef + wantApply bool + }{ + { + name: "configMap caBundleRef accepted", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "ConfigMap", Name: "mirror-trust"}, + wantApply: true, + }, + { + name: "secret caBundleRef with key accepted", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "Secret", Name: "mirror-trust", Key: "bundle.pem"}, + wantApply: true, + }, + { + name: "caBundleRef with empty name rejected", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "ConfigMap", Name: ""}, + wantApply: false, + }, + { + name: "caBundleRef with bogus kind rejected", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "DaemonSet", Name: "mirror-trust"}, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := validSourceEndpoint() + source.TLS = &ecv1alpha1.EtcdMirrorTLS{CABundleRef: tt.ref} + em := newMirror("cel-mirror-cabundle-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "caBundleRef schema rules") + }) + } +} + +// TestEtcdMirrorInitialSyncCELValidation drives the initialSync rules: the +// mode enum, and the startRevision-requires-Overwrite guard (a target seeded +// via snapshot restore cannot also be required empty). +func TestEtcdMirrorInitialSyncCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + initialSync *ecv1alpha1.EtcdMirrorInitialSyncSpec + wantApply bool + }{ + { + name: "nil initialSync accepted", + initialSync: nil, + wantApply: true, + }, + { + name: "explicit RequireEmpty accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: ecv1alpha1.EtcdMirrorInitialSyncRequireEmpty}, + wantApply: true, + }, + { + name: "Overwrite accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwrite}, + wantApply: true, + }, + { + name: "OverwriteAndPrune accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwriteAndPrune}, + wantApply: true, + }, + { + name: "bogus mode rejected by enum", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: "TruncateFirst"}, + wantApply: false, + }, + { + name: "startRevision with Overwrite accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwrite, + StartRevision: 42, + }, + wantApply: true, + }, + { + name: "startRevision with OverwriteAndPrune accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwriteAndPrune, + StartRevision: 42, + }, + wantApply: true, + }, + { + name: "startRevision with explicit RequireEmpty rejected", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncRequireEmpty, + StartRevision: 42, + }, + wantApply: false, + }, + { + name: "startRevision with defaulted mode (RequireEmpty) rejected", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + StartRevision: 42, + }, + wantApply: false, + }, + { + name: "startRevision zero with RequireEmpty accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncRequireEmpty, + StartRevision: 0, + }, + wantApply: true, + }, + { + name: "negative startRevision rejected by minimum", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwrite, + StartRevision: -1, + }, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-initialsync-", ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + InitialSync: tt.initialSync, + }) + createAndCheck(t, em, tt.wantApply, "initialSync rules") + }) + } +} + +// TestEtcdMirrorModeValidation drives the spec.mode enum. +func TestEtcdMirrorModeValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + mode ecv1alpha1.EtcdMirrorMode + wantApply bool + }{ + {name: "mode unset accepted (defaults to Sync)", mode: "", wantApply: true}, + {name: "mode Sync accepted", mode: ecv1alpha1.EtcdMirrorModeSync, wantApply: true}, + {name: "mode Drain accepted", mode: ecv1alpha1.EtcdMirrorModeDrain, wantApply: true}, + {name: "bogus mode rejected", mode: "Bidirectional", wantApply: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-mode-", ecv1alpha1.EtcdMirrorSpec{ + Mode: tt.mode, + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "spec.mode enum") + }) + } +} + +// TestEtcdMirrorImmutabilityCELValidation drives the CEL transition rules: +// source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes and +// checkpoint.key are immutable; endpoints (and ordinary fields) stay mutable +// — same-cluster endpoint rotation is routine and cross-cluster repoints are +// caught at runtime by the checkpoint's cluster-ID binding, not by spec +// validation. +func TestEtcdMirrorImmutabilityCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + baseSpec := func() ecv1alpha1.EtcdMirrorSpec { + return ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Sync: ecv1alpha1.EtcdMirrorSyncSpec{ + DestPrefix: "registry/", + ExcludePrefixes: []string{"/registry/events/"}, + }, + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + // Must live under the effective destination prefix + // (target.prefix "/mirrored/" + destPrefix "registry/"). + Key: "/mirrored/registry/\x00etcdmirror-checkpoint", + }, + } + } + + tests := []struct { + name string + mutate func(em *ecv1alpha1.EtcdMirror) + wantUpdate bool + }{ + { + name: "changing source.prefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Source.Prefix = "/other/" }, + wantUpdate: false, + }, + { + name: "unsetting source.prefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Source.Prefix = "" }, + wantUpdate: false, + }, + { + name: "changing target.prefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Target.Prefix = "/elsewhere/" }, + wantUpdate: false, + }, + { + name: "changing sync.destPrefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.DestPrefix = "moved/" }, + wantUpdate: false, + }, + { + name: "unsetting sync.destPrefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.DestPrefix = "" }, + wantUpdate: false, + }, + { + name: "changing checkpoint.key rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Checkpoint.Key = "/mirrored/registry/\x00other" }, + wantUpdate: false, + }, + { + name: "changing sync.excludePrefixes rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Sync.ExcludePrefixes = []string{"/registry/leases/"} + }, + wantUpdate: false, + }, + { + name: "removing sync.excludePrefixes rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.ExcludePrefixes = nil }, + wantUpdate: false, + }, + { + name: "removing checkpoint block (unsetting key) rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Checkpoint = nil }, + wantUpdate: false, + }, + { + name: "changing source endpoints accepted (endpoints deliberately mutable)", + mutate: func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Source.EndpointList = []string{"etcd-source-b.example.com:2379", "etcd-source-c.example.com:2379"} + }, + wantUpdate: true, + }, + { + name: "changing paused accepted", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Paused = true }, + wantUpdate: true, + }, + { + name: "changing sync.maxOpsPerSecond accepted", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.MaxOpsPerSecond = 250 }, + wantUpdate: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-immutable-", baseSpec()) + require.NoError(t, k8sClient.Create(t.Context(), em), "baseline mirror must be accepted") + defer func() { _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) }() + + tt.mutate(em) + err := k8sClient.Update(t.Context(), em) + if tt.wantUpdate { + assert.NoError(t, err, "update should be accepted") } else { - assert.Error(t, err, "apiserver should reject an empty secretRef.name via CEL") + assert.Error(t, err, "update should be rejected by transition rule") } }) } + + t.Run("setting sync.destPrefix from unset rejected", func(t *testing.T) { + spec := baseSpec() + spec.Sync = ecv1alpha1.EtcdMirrorSyncSpec{} + spec.Checkpoint = nil + em := newMirror("cel-mirror-immutable-", spec) + require.NoError(t, k8sClient.Create(t.Context(), em)) + defer func() { _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) }() + + em.Spec.Sync.DestPrefix = "late/" + assert.Error(t, k8sClient.Update(t.Context(), em), "unset -> set is still a destPrefix mutation") + }) + + // Present-empty and absent are the same VALUE for these fields (and Go + // typed clients drop "" through omitempty), so a CR created from YAML + // with an explicit destPrefix: "" must stay updatable via typed clients + // — presence-based transition rules would 422 every such update on a + // field it never touched. + t.Run("explicit empty destPrefix stays typed-client updatable", func(t *testing.T) { + u := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "operator.etcd.io/v1alpha1", + "kind": "EtcdMirror", + "metadata": map[string]interface{}{ + "generateName": "cel-mirror-emptydest-", + "namespace": "default", + }, + "spec": map[string]interface{}{ + "source": map[string]interface{}{ + "endpointList": []interface{}{"etcd-source.example.com:2379"}, + "prefix": "/registry/", + }, + "target": map[string]interface{}{ + "serviceRef": map[string]interface{}{"name": "etcd-target-client"}, + "prefix": "/mirrored/", + }, + "sync": map[string]interface{}{ + "destPrefix": "", // stored present-but-empty + }, + }, + }} + require.NoError(t, k8sClient.Create(t.Context(), u), "explicit empty destPrefix must be accepted") + defer func() { _ = k8sClient.Delete(t.Context(), u, &client.DeleteOptions{}) }() + + em := &ecv1alpha1.EtcdMirror{} + require.NoError(t, k8sClient.Get(t.Context(), + client.ObjectKey{Namespace: "default", Name: u.GetName()}, em)) + em.Spec.Paused = true // typed round-trip drops destPrefix to absent + assert.NoError(t, k8sClient.Update(t.Context(), em), + "a typed-client update must not be rejected for a field it never touched") + }) +} + +// TestEtcdMirrorCheckpointKeyCELValidation drives the create-time rule that +// checkpoint.key must live under the effective destination prefix: the +// engine rejects anything else permanently at first start, and the key's own +// immutability would otherwise make the Failed CR unrepairable in place. +func TestEtcdMirrorCheckpointKeyCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + spec ecv1alpha1.EtcdMirrorSpec + wantApply bool + }{ + { + name: "key under effective destination prefix accepted", + spec: ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + Key: "/mirrored/\x00my-checkpoint", + }, + }, + wantApply: true, + }, + { + name: "key outside destination prefix rejected", + spec: ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + Key: "/checkpoints/mirror-a", + }, + }, + wantApply: false, + }, + { + name: "key under target.prefix but outside destPrefix rejected", + spec: ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Sync: ecv1alpha1.EtcdMirrorSyncSpec{DestPrefix: "registry/"}, + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + Key: "/mirrored/\x00etcdmirror-checkpoint", + }, + }, + wantApply: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-ckptkey-", tt.spec) + createAndCheck(t, em, tt.wantApply, "checkpoint.key placement") + }) + } } From 82d23523dc67c8d5c7b295ee7613ff29f37fd419 Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Sat, 4 Jul 2026 17:35:59 -0400 Subject: [PATCH 3/5] feat(etcdmirror): add mirroragent replication engine (Design 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Purpose-built replication engine, zero k8s dependency; replaces the clientv3 mirror.Syncer plan. Architecture: unpinned chunked scan + watch-replay from R0 (reflector pattern) — the watch starts at the revision observed before the scan and events are buffered/replayed over the scanned base, eliminating mid-scan compaction as a failure class. Pull-based single-page scan with byte-bounded pages keeps agent memory bounded; the source watch is cancelled on sustained target backoff and resumed from checkpoint. Fence protocol: checkpoint/watermark stored in the target etcd at a reserved key (\x00-after-prefix convention, exact-match excluded from scans/counts/prune/RequireEmpty), written in the same Txn as each applied batch and guarded by a mod_revision compare on every write path (applies, reconciliation repairs, prune deletes). Fence value carries {linkUID, epoch, role}; role flips to Primary at cutover so straggler applies fail their compare loudly. Undecodable checkpoints fail closed; prune-pending is durable across restarts. Batching: flush only at source-revision boundaries (whole revisions coalesced, never split), ~1MiB byte watermark, one MaxTxnOps slot reserved for the checkpoint write. Error taxonomy: ErrCompacted classified distinctly; rpctypes.ErrNoSpace -> TargetQuotaExhausted (permanent until operator acts); oversized-Txn InvalidArgument and >2MiB client send-cap ResourceExhausted both permanent with the offending key surfaced redacted; throttle-distinct backoff; N-consecutive-forced-resync livelock detector. Liveness: watermark-derived progress via WithProgressNotify + WithRequireLeader, client-driven RequestProgress, gRPC keepalives on both clients, per-unary-RPC deadlines (watch excluded). Version probe at connect enforces the >=3.4 source floor and gates progress-notify trust on 3.4.25/3.5.8. Anchored prefix rewrite via the single documented formula. Lease-backed keys detected and counted (kv.Lease != 0). Tests: unit coverage per module plus an embedded-etcd integration suite covering scan/tail convergence, fence overlap and epoch/role rejection, compaction during scan and drain, forced-resync mark-and-sweep, checkpoint resume, prune, and error classification. Co-Authored-By: Claude Fable 5 Signed-off-by: Xavier Lange --- pkg/mirroragent/agent.go | 564 ++++++++ pkg/mirroragent/apply.go | 277 ++++ pkg/mirroragent/backoff.go | 90 ++ pkg/mirroragent/backoff_test.go | 96 ++ pkg/mirroragent/batch.go | 187 +++ pkg/mirroragent/batch_test.go | 192 +++ pkg/mirroragent/client.go | 62 + pkg/mirroragent/config.go | 331 +++++ pkg/mirroragent/config_test.go | 133 ++ pkg/mirroragent/doc.go | 101 ++ pkg/mirroragent/errors.go | 290 +++++ pkg/mirroragent/errors_test.go | 135 ++ pkg/mirroragent/fence.go | 150 +++ pkg/mirroragent/fence_test.go | 110 ++ pkg/mirroragent/helpers_integration_test.go | 305 +++++ pkg/mirroragent/integration_delta_test.go | 1289 +++++++++++++++++++ pkg/mirroragent/integration_test.go | 531 ++++++++ pkg/mirroragent/reconcile.go | 320 +++++ pkg/mirroragent/rewrite.go | 184 +++ pkg/mirroragent/rewrite_test.go | 157 +++ pkg/mirroragent/scan.go | 432 +++++++ pkg/mirroragent/snapshot.go | 177 +++ pkg/mirroragent/tail.go | 289 +++++ pkg/mirroragent/version.go | 65 + pkg/mirroragent/version_test.go | 72 ++ 25 files changed, 6539 insertions(+) create mode 100644 pkg/mirroragent/agent.go create mode 100644 pkg/mirroragent/apply.go create mode 100644 pkg/mirroragent/backoff.go create mode 100644 pkg/mirroragent/backoff_test.go create mode 100644 pkg/mirroragent/batch.go create mode 100644 pkg/mirroragent/batch_test.go create mode 100644 pkg/mirroragent/client.go create mode 100644 pkg/mirroragent/config.go create mode 100644 pkg/mirroragent/config_test.go create mode 100644 pkg/mirroragent/doc.go create mode 100644 pkg/mirroragent/errors.go create mode 100644 pkg/mirroragent/errors_test.go create mode 100644 pkg/mirroragent/fence.go create mode 100644 pkg/mirroragent/fence_test.go create mode 100644 pkg/mirroragent/helpers_integration_test.go create mode 100644 pkg/mirroragent/integration_delta_test.go create mode 100644 pkg/mirroragent/integration_test.go create mode 100644 pkg/mirroragent/reconcile.go create mode 100644 pkg/mirroragent/rewrite.go create mode 100644 pkg/mirroragent/rewrite_test.go create mode 100644 pkg/mirroragent/scan.go create mode 100644 pkg/mirroragent/snapshot.go create mode 100644 pkg/mirroragent/tail.go create mode 100644 pkg/mirroragent/version.go create mode 100644 pkg/mirroragent/version_test.go diff --git a/pkg/mirroragent/agent.go b/pkg/mirroragent/agent.go new file mode 100644 index 00000000..502efe16 --- /dev/null +++ b/pkg/mirroragent/agent.go @@ -0,0 +1,564 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// errDrained signals a completed Drain cutover up the Run stack; Run maps it +// to a nil return. +var errDrained = errors.New("drain completed") + +// errFenceLost means a fence claim raced an older agent generation's write +// between our read and our takeover Txn. fenceViolation adopts the raced +// write's mod revision before returning it, so the caller's retry re-runs +// the takeover against the current fence: loadFence's loop re-reads and +// retries, and applyFenced retries any claim made before this generation's +// first successful commit (the genesis / startFromRevision claims on a +// fresh target). After that first commit (Agent.claimed), older generations +// fail their own compares and can never move the fence again, so a +// post-claim loss escalates to a permanent FenceError. +var errFenceLost = errors.New("fence claim raced an older generation write") + +// Agent is the replication engine. Create with New, drive with Run (once), +// observe with Snapshot from any goroutine. +type Agent struct { + cfg Config + src Client + dst Client + rw *rewriter + bo *backoff + + srcClusterID uint64 + dstClusterID uint64 + // trustProgress gates watermark advancement from watch progress + // notifications (source >= 3.4.25 / 3.5.8). + trustProgress bool + + // fence is the engine's cached copy of the reserved key's value; + // fenceModRev is the mod revision EVERY write path compares against. + // Both are owned by the Run goroutine. + fence FenceValue + fenceModRev int64 + // claimed is true once this generation committed its first fenced Txn: + // from then on a lost fence compare is a permanent violation, never a + // retryable claim race. Owned by the Run goroutine. + claimed bool + // prunePending mirrors FenceValue.PrunePending: set when a forced resync + // starts, stamped into every checkpoint, cleared only after the mandatory + // mark-and-sweep prune completed. Owned by the Run goroutine. + prunePending bool + + // watchCancel cancels the live source watch; applyFenced invokes it via + // cancelSourceWatch on sustained target backoff so clientv3's unbounded + // per-watcher response buffer cannot grow for the duration of a target + // stall. Owned by the Run goroutine (set by genesis/tail, consumed by + // the apply path, which runs on the same goroutine). + watchCancel func() + + drainReq atomic.Bool + + // consecutiveResyncs counts forced resyncs without reaching steady state + // in between (owned by the Run goroutine). restartBo paces Run's + // genesis-restart loop; it is deliberately separate from the shared bo, + // whose curves reset on every successful apply inside a scan attempt. + consecutiveResyncs int + restartBo *backoff + + mu sync.Mutex + snap Snapshot +} + +// startState says how a replication cycle begins: fresh genesis, resumed +// scan, resumed tail, or a forced resync. +type startState struct { + haveCheckpoint bool + scanning bool + scanCursor string + subRevision int64 + watermark int64 + // forced marks a forced resync: full re-scan plus a mandatory + // mark-and-sweep prune pass; RequireEmpty is NOT re-checked (the decoded, + // ownership-validated fence proves the destination data is this link's + // own). Persisted across restarts as FenceValue.PrunePending so a crash + // mid-forced-resync cannot silently drop the owed sweep. + forced bool + // rearmEmpty re-arms the RequireEmpty check (cluster-identity mismatch). + rearmEmpty bool +} + +// New validates cfg (after defaulting) and builds an Agent over the two +// clients. The caller owns client lifecycle and TLS/auth material. +func New(cfg Config, source, target Client) (*Agent, error) { + cfg = cfg.withDefaults() + if err := cfg.Validate(); err != nil { + return nil, err + } + return &Agent{ + cfg: cfg, + src: source, + dst: target, + rw: newRewriter(cfg), + bo: newBackoff(cfg.BackoffInitialDelay, cfg.BackoffMaxDelay), + restartBo: newBackoff(cfg.BackoffInitialDelay, cfg.BackoffMaxDelay), + snap: Snapshot{Phase: PhaseConnecting}, + }, nil +} + +// Snapshot returns a point-in-time copy of the agent's state, safe to retain. +func (a *Agent) Snapshot() Snapshot { + a.mu.Lock() + defer a.mu.Unlock() + s := a.snap + if a.snap.Cutover != nil { + c := *a.snap.Cutover + s.Cutover = &c + } + if a.snap.LastReconcileDrift != nil { + d := *a.snap.LastReconcileDrift + s.LastReconcileDrift = &d + } + return s +} + +// RequestDrain flips a running Sync agent into a drain, as if Mode were +// ModeDrain. +func (a *Agent) RequestDrain() { a.drainReq.Store(true) } + +// Run executes the replication loop until ctx is cancelled (returns +// ctx.Err()), a Drain completes (returns nil), or a permanent failure occurs +// (returns the classified error). +func (a *Agent) Run(ctx context.Context) error { + if err := a.connect(ctx); err != nil { + return a.fail(ctx, err) + } + st, err := a.loadFence(ctx) + if err != nil { + return a.fail(ctx, err) + } + for { + err := a.cycle(ctx, st) + var restart *scanRestartError + switch { + case err == nil: + return nil // drain completed + case ctx.Err() != nil: + return ctx.Err() + case errors.As(err, &restart): + // Bounded genesis retry: restart the scan from a fresh R0. The + // dropped replay buffer may have held deletes, so the restarted + // attempt owes a mark-and-sweep (forced). The dedicated restart + // backoff keeps repeated restarts (churn outrunning the buffer) + // off a hot loop AND escalating: the shared curve resets on every + // successful apply inside a doomed attempt, so it would stay + // pinned at the initial delay forever. + a.noteScanRestart(restart) + if serr := sleepCtx(ctx, a.restartBo.next(ClassTransient)); serr != nil { + return serr + } + st = startState{forced: true} + case Classify(err) == ClassResync: + a.noteResync(err) + st = startState{forced: true} + default: + return a.fail(ctx, err) + } + } +} + +// cycle runs one replication attempt from the given start state. +func (a *Agent) cycle(ctx context.Context, st startState) error { + var err error + switch { + case st.haveCheckpoint && !st.scanning: + err = a.tail(ctx, nil, nil, st.watermark) + case !st.haveCheckpoint && !st.forced && a.cfg.StartRevision > 0: + err = a.startFromRevision(ctx) + default: + err = a.genesis(ctx, st) + } + if errors.Is(err, errDrained) { + return nil + } + return err +} + +// startFromRevision skips the genesis scan (fidelity-preserving snapshot +// seed) and tails from StartRevision+1. +func (a *Agent) startFromRevision(ctx context.Context) error { + if err := a.applyFenced(ctx, nil, a.newFence(a.cfg.StartRevision, false, "", 0), "", 0); err != nil { + return err + } + a.advanceWatermark(a.cfg.StartRevision) + return a.tail(ctx, nil, nil, a.cfg.StartRevision) +} + +// connect probes both sides' version and cluster identity (maintenance +// Status at connect) and enforces the >=3.4 hard floor. +func (a *Agent) connect(ctx context.Context) error { + a.setPhase(PhaseConnecting) + srcInfo, srcID, err := a.probe(ctx, "source", a.src) + if err != nil { + return err + } + dstInfo, dstID, err := a.probe(ctx, "target", a.dst) + if err != nil { + return err + } + a.srcClusterID, a.dstClusterID = srcID, dstID + a.trustProgress = srcInfo.TrustProgressNotify + a.update(func(s *Snapshot) { + s.SourceVersion = srcInfo.Version + s.TargetVersion = dstInfo.Version + s.SourceClusterID = srcID + s.TargetClusterID = dstID + }) + return nil +} + +func (a *Agent) probe(ctx context.Context, side string, cl Client) (versionInfo, uint64, error) { + eps := cl.Endpoints() + if len(eps) == 0 { + return versionInfo{}, 0, &ConfigError{Detail: side + " client has no endpoints"} + } + // Status dials the named endpoint directly, bypassing the balancer, so + // the probe must rotate endpoints itself: one blackholed member must not + // wedge the agent in Connecting while healthy quorum members exist. + attempt := 0 + for { + tctx, cancel := context.WithTimeout(ctx, a.cfg.RequestTimeout) + resp, err := cl.Status(tctx, eps[attempt%len(eps)]) + cancel() + attempt++ + if err == nil { + if attempt > 1 { + a.bo.noteSuccess() + } + vi, verr := classifyVersion(side, resp.Version) + if verr != nil { + return versionInfo{}, 0, verr + } + return vi, resp.Header.ClusterId, nil + } + if ctx.Err() != nil { + return versionInfo{}, 0, ctx.Err() + } + class := Classify(err) + if class == ClassPermanent || class == ClassResync { + return versionInfo{}, 0, fmt.Errorf("probing %s: %w", side, err) + } + a.recordErr(err, class) + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return versionInfo{}, 0, serr + } + } +} + +// loadFence reads the reserved key, validates ownership, and — when a valid +// checkpoint of this link exists — takes the fence over for this agent +// generation so any straggler generation fails its next compare. +func (a *Agent) loadFence(ctx context.Context) (startState, error) { + for { + resp, err := a.getRetry(ctx, a.dst, a.cfg.CheckpointKey) + if err != nil { + return startState{}, err + } + if len(resp.Kvs) == 0 { + // Fresh target: the fence is claimed at genesis start, after the + // RequireEmpty gate, so a violation writes nothing. + a.fenceModRev = 0 + return startState{}, nil + } + kv := resp.Kvs[0] + a.fenceModRev = kv.ModRevision + f, derr := DecodeFenceValue(kv.Value) + if derr != nil { + // Corrupt or unknown-version checkpoint: fail CLOSED. Nothing + // about the stored value is knowable — not the owning link, not + // the epoch, not whether a cutover already flipped the role to + // Primary — so no write (least of all a resync's prune) is + // provably safe. Permanent: the operator must inspect the + // reserved key and delete it to recover. + return startState{}, fmt.Errorf("reserved key %q: %w", a.cfg.CheckpointKey, derr) + } + takeover, st, verr := a.validateFence(f) + if verr != nil { + return startState{}, verr + } + if !takeover { + return st, nil + } + // Take the fence over for this generation before anything else runs. + f.Epoch = a.cfg.Epoch + err = a.commitFenced(ctx, nil, f) + if err == nil { + a.advanceWatermark(f.Watermark) + return startState{ + haveCheckpoint: true, + scanning: f.Scanning, + scanCursor: f.ScanCursor, + subRevision: f.SubRevision, + watermark: f.Watermark, + // A crash mid-forced-resync leaves the owed mark-and-sweep + // recorded in the fence; the resumed scan must still prune. + forced: f.PrunePending, + }, nil + } + if errors.Is(err, errFenceLost) { + continue // an older generation wrote in the read/claim window + } + class := Classify(err) + switch class { + case ClassPermanent, ClassResync: + return startState{}, err + case ClassQuota: + a.recordErr(err, class) + a.update(func(s *Snapshot) { s.QuotaExhausted = true; s.Phase = PhaseDegraded }) + if serr := sleepCtx(ctx, a.cfg.QuotaProbeInterval); serr != nil { + return startState{}, serr + } + default: + a.recordErr(err, class) + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return startState{}, serr + } + } + } +} + +// validateFence checks a decoded checkpoint against this agent's identity. +// takeover is true when the fence is ours to take over; otherwise st is the +// forced-resync start state or err is terminal. +func (a *Agent) validateFence(f FenceValue) (takeover bool, st startState, err error) { + if f.LinkUID != a.cfg.LinkUID { + return false, startState{}, &FenceError{Detail: fmt.Sprintf( + "reserved key %q is owned by link %q, not %q", + a.cfg.CheckpointKey, f.LinkUID, a.cfg.LinkUID)} + } + if f.Role == RolePrimary { + return false, startState{}, &FenceError{ + Detail: "fence role is Primary: cutover completed, mirror writes are forbidden", + } + } + if f.Epoch > a.cfg.Epoch { + return false, startState{}, &FenceError{Detail: fmt.Sprintf( + "newer agent epoch %d owns the link (this agent is epoch %d)", + f.Epoch, a.cfg.Epoch)} + } + if f.SourceClusterID != a.srcClusterID || f.TargetClusterID != a.dstClusterID { + a.noteResync(&ResyncError{Reason: ResyncReasonClusterIDMismatch, Cause: fmt.Errorf( + "checkpoint bound to source=%d target=%d, probed source=%d target=%d", + f.SourceClusterID, f.TargetClusterID, a.srcClusterID, a.dstClusterID)}) + return false, startState{forced: true, rearmEmpty: true}, nil + } + return true, startState{}, nil +} + +// newFence builds the checkpoint document for this agent generation. +func (a *Agent) newFence(watermark int64, scanning bool, cursor string, subrev int64) FenceValue { + return FenceValue{ + LinkUID: a.cfg.LinkUID, + Epoch: a.cfg.Epoch, + Role: RoleMirror, + Watermark: watermark, + Scanning: scanning, + ScanCursor: cursor, + SubRevision: subrev, + PrunePending: a.prunePending, + SourceClusterID: a.srcClusterID, + TargetClusterID: a.dstClusterID, + } +} + +// noteResync records a forced resync and drives the livelock detector. +func (a *Agent) noteResync(err error) { + reason := ResyncReasonCompacted + var re *ResyncError + if errors.As(err, &re) { + reason = re.Reason + } + a.consecutiveResyncs++ + loop := a.consecutiveResyncs >= a.cfg.ResyncLoopThreshold + a.update(func(s *Snapshot) { + s.ForcedResyncCount++ + s.LastResyncReason = reason + s.Compacted = reason == ResyncReasonCompacted + if loop { + s.ResyncLoopDetected = true + } + }) +} + +// noteScanRestart records an aborted genesis attempt (buffer overflow or a +// watch reconnect below the compact revision mid-scan). Restarts do not +// invalidate the checkpoint — ForcedResyncCount is untouched — but they +// count toward the same livelock detector: repeated restarts are the +// signature of churn or retention outrunning scan throughput. +func (a *Agent) noteScanRestart(e *scanRestartError) { + a.consecutiveResyncs++ + loop := a.consecutiveResyncs >= a.cfg.ResyncLoopThreshold + a.update(func(s *Snapshot) { + s.ScanRestartCount++ + s.LastScanRestartCause = e.Cause + if loop { + s.ResyncLoopDetected = true + } + }) +} + +// steadyState is reached on the first successfully applied LIVE watch +// response of a tail: it resets the resync-loop detector and the restart +// backoff. Genesis replay-buffer applies must never reach here — they run +// INSIDE the resync the detector is counting, and a churning source (the +// canonical livelock trigger) guarantees a non-empty replay buffer, so a +// replay-driven reset would keep the detector at zero forever. +func (a *Agent) steadyState() { + a.restartBo.reset() + if a.consecutiveResyncs == 0 { + return + } + a.consecutiveResyncs = 0 + a.update(func(s *Snapshot) { + s.ResyncLoopDetected = false + s.Compacted = false + }) +} + +// cancelSourceWatch tears down the live source watch (if any) so clientv3 +// stops buffering undelivered responses while the target is parked or in +// sustained backoff; the tail re-watches from the checkpoint watermark once +// applies succeed again. Idempotent. +func (a *Agent) cancelSourceWatch() { + if a.watchCancel != nil { + a.watchCancel() + a.watchCancel = nil + } +} + +func (a *Agent) fail(ctx context.Context, err error) error { + if ctx.Err() != nil { + return ctx.Err() + } + class := Classify(err) + a.update(func(s *Snapshot) { + s.LastError = err.Error() + s.LastErrorClass = class + if s.Phase != PhaseDrained { + s.Phase = PhaseFailed + } + }) + return err +} + +func (a *Agent) update(fn func(*Snapshot)) { + a.mu.Lock() + defer a.mu.Unlock() + fn(&a.snap) +} + +func (a *Agent) setPhase(p Phase) { + a.update(func(s *Snapshot) { s.Phase = p }) +} + +func (a *Agent) phase() Phase { + a.mu.Lock() + defer a.mu.Unlock() + return a.snap.Phase +} + +func (a *Agent) watermark() int64 { + a.mu.Lock() + defer a.mu.Unlock() + return a.snap.Watermark +} + +func (a *Agent) advanceWatermark(rev int64) { + a.update(func(s *Snapshot) { + if rev > s.Watermark { + s.Watermark = rev + } + s.LastProgressTime = time.Now() + }) +} + +func (a *Agent) recordErr(err error, class Class) { + a.update(func(s *Snapshot) { + s.LastError = err.Error() + s.LastErrorClass = class + }) +} + +// pace enforces MaxOpsPerSecond with simple pre-write sleeping. +func (a *Agent) pace(ctx context.Context, n int) { + if a.cfg.MaxOpsPerSecond <= 0 || n == 0 { + return + } + d := time.Duration(n) * time.Second / time.Duration(a.cfg.MaxOpsPerSecond) + _ = sleepCtx(ctx, d) +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} + +// getRetry performs a unary Get under the per-RPC deadline with the standard +// read retry policy: transient/throttle back off, resync and permanent +// propagate. A success after retries resets the backoff curves, so one old +// saturated burst does not pin every later isolated retry at the max delay. +func (a *Agent) getRetry( + ctx context.Context, cl Client, key string, opts ...clientv3.OpOption, +) (*clientv3.GetResponse, error) { + retried := false + for { + tctx, cancel := context.WithTimeout(ctx, a.cfg.RequestTimeout) + resp, err := cl.Get(tctx, key, opts...) + cancel() + if err == nil { + if retried { + a.bo.noteSuccess() + } + return resp, nil + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + class := Classify(err) + if class == ClassPermanent || class == ClassResync { + return nil, err + } + a.recordErr(err, class) + retried = true + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return nil, serr + } + } +} diff --git a/pkg/mirroragent/apply.go b/pkg/mirroragent/apply.go new file mode 100644 index 00000000..017c8197 --- /dev/null +++ b/pkg/mirroragent/apply.go @@ -0,0 +1,277 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "errors" + "fmt" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// backoffRoundsBeforeWatchCancel is how many transient/throttle backoff +// rounds a single fenced apply endures before the live source watch is +// cancelled: clientv3 buffers undelivered watch responses without bound, so +// a sustained target stall must stop the source stream (the tail re-watches +// from the checkpoint watermark once applies succeed again). Quota parks +// cancel immediately — they are expected to last minutes to hours. +const backoffRoundsBeforeWatchCancel = 3 + +// applyOps writes one flush set plus checkpoint f in a single fenced Txn, +// through the class-appropriate retry policy. A multi-revision set the +// target rejects for its Txn limits gets ONE shrink attempt at revision +// granularity (the checkpoint advancing with each sub-Txn); a single +// revision that alone trips the limits is irreducible and stays permanent. +func (a *Agent) applyOps(ctx context.Context, fs *flushSet, f FenceValue) error { + a.pace(ctx, len(fs.ops)) + err := a.applyFlushSet(ctx, fs, f) + var tle *TooLargeError + if err == nil || !errors.As(err, &tle) || len(fs.groups) <= 1 { + return err + } + // Shrink: the whole set fit the engine's own watermarks but not the + // target's --max-txn-ops / --max-request-bytes (foreign cluster; its + // flags are not inspectable). Re-commit revision by revision — every + // sub-Txn still cuts at a revision boundary. + for _, g := range fs.groups { + sub := flushSet{ + ops: g.ops, + groups: []revGroup{g}, + watermark: g.rev, + lastSrcKey: g.ops[len(g.ops)-1].srcKey, + } + fsub := f + fsub.Watermark = g.rev + if fsub.Scanning { + fsub.ScanCursor = sub.lastSrcKey + } + if serr := a.applyFlushSet(ctx, &sub, fsub); serr != nil { + return serr + } + } + return nil +} + +// applyFlushSet converts one flush set to ops and commits it fenced. +func (a *Agent) applyFlushSet(ctx context.Context, fs *flushSet, f FenceValue) error { + ops := make([]clientv3.Op, 0, len(fs.ops)) + var nBytes int64 + for _, o := range fs.ops { + nBytes += o.bytes() + if o.isDelete { + ops = append(ops, clientv3.OpDelete(o.key)) + } else { + ops = append(ops, clientv3.OpPut(o.key, o.value)) + } + } + return a.applyFenced(ctx, ops, f, fs.ops[0].key, nBytes) +} + +// applyFenced commits ops plus the checkpoint write under the fence compare, +// retrying per class: transient and throttle back off on their own curves, +// quota parks on the flat probe interval (never backoff — quota only heals +// when an operator acts), resync and permanent errors propagate. A fence +// claim (before this generation's first successful commit) that loses a race +// against an older generation's last write retries the takeover; post-claim, +// a moved fence is a loud permanent failure. +func (a *Agent) applyFenced( + ctx context.Context, ops []clientv3.Op, f FenceValue, firstKey string, nBytes int64, +) error { + prior := a.phase() + rounds := 0 + for { + err := a.commitFenced(ctx, ops, f) + if err == nil { + a.bo.noteSuccess() + a.update(func(s *Snapshot) { + s.Throttled = false + s.QuotaExhausted = false + s.LastError = "" + s.LastErrorClass = "" + if s.Phase == PhaseDegraded && prior != PhaseDegraded { + s.Phase = prior + } + }) + return nil + } + if ctx.Err() != nil { + return ctx.Err() + } + if errors.Is(err, errFenceLost) { + if !a.claimed { + // Pre-claim race: an older generation wrote between our fence + // read and this claim. fenceViolation adopted the raced mod + // revision, so the retry re-runs the takeover against it. + a.recordErr(err, ClassTransient) + if serr := sleepCtx(ctx, a.bo.next(ClassTransient)); serr != nil { + return serr + } + continue + } + // Post-claim, older generations fail their own compares and can + // never move the fence again: surface loudly rather than retrying + // against a stale mod revision forever. + err = &FenceError{Detail: "checkpoint mod revision moved under an active generation"} + } + class := Classify(err) + if class == ClassPermanent && isOversized(err) { + err = &TooLargeError{ + Key: RedactKey(a.cfg.EffectiveDestPrefix(), []byte(firstKey)), + Ops: len(ops) + 1, + Bytes: nBytes, + Cause: err, + } + } + a.recordErr(err, class) + switch class { + case ClassQuota: + a.cancelSourceWatch() + a.update(func(s *Snapshot) { + s.QuotaExhausted = true + s.Phase = PhaseDegraded + }) + if serr := sleepCtx(ctx, a.cfg.QuotaProbeInterval); serr != nil { + return serr + } + case ClassThrottle: + if rounds++; rounds >= backoffRoundsBeforeWatchCancel { + a.cancelSourceWatch() + } + a.update(func(s *Snapshot) { + s.Throttled = true + s.Phase = PhaseDegraded + }) + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return serr + } + case ClassTransient: + if rounds++; rounds >= backoffRoundsBeforeWatchCancel { + a.cancelSourceWatch() + } + a.setPhase(PhaseDegraded) + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return serr + } + default: + return err + } + } +} + +// commitFenced is one fenced Txn attempt: ops plus the checkpoint write in +// the reserved op slot, guarded by the mod-revision compare on the reserved +// key. On success the cached fence and mod revision advance together. A +// failed compare is resolved by fenceViolation, which recognizes this +// generation's own ambiguously-timed-out commit and adopts it as success. +func (a *Agent) commitFenced(ctx context.Context, ops []clientv3.Op, f FenceValue) error { + val, err := f.Encode() + if err != nil { + // Encoding our own fence value can only fail on an engine bug; fail + // closed the same way a corrupt stored checkpoint does. + return &CheckpointInvalidError{Reason: fmt.Sprintf("encoding checkpoint: %v", err)} + } + cmp := clientv3.Compare(clientv3.ModRevision(a.cfg.CheckpointKey), "=", a.fenceModRev) + all := make([]clientv3.Op, 0, len(ops)+1) + all = append(all, ops...) + all = append(all, clientv3.OpPut(a.cfg.CheckpointKey, val)) + tctx, cancel := context.WithTimeout(ctx, a.cfg.RequestTimeout) + resp, err := a.dst.Txn(tctx).If(cmp).Then(all...).Commit() + cancel() + if err != nil { + return err + } + if !resp.Succeeded { + return a.fenceViolation(ctx, f, val) + } + a.fence = f + a.fenceModRev = resp.Header.Revision + a.claimed = true + return nil +} + +// fenceViolation resolves a failed fence compare by re-reading the reserved +// key — never a blind re-Commit (doc.go's retry-ownership contract). Three +// outcomes: +// +// - The stored value is byte-identical to the value this attempt was +// writing: an earlier attempt of this exact Txn committed but its +// response was lost (the classic ambiguous timeout — the Txn's own +// checkpoint Put bumped the fence ModRevision). The data ops landed +// exactly once; adopt the new mod revision and report success (nil). +// - The stored fence is an OLDER generation of this link: a claim raced +// the old generation's last write. Adopt the raced mod revision and +// return errFenceLost so the caller retries the takeover. +// - Anything else (another link, a newer epoch, a Primary role we did not +// write): a genuine, permanent fence violation. +func (a *Agent) fenceViolation(ctx context.Context, f FenceValue, attempted string) error { + resp, err := a.getRetry(ctx, a.dst, a.cfg.CheckpointKey) + if err != nil { + // getRetry already retried transient/throttle reads; what escapes is + // cancellation or a permanent read failure. + return fmt.Errorf("re-reading fence after a failed compare: %w", err) + } + if len(resp.Kvs) == 0 { + return &FenceError{Detail: "the reserved key was deleted under an active generation"} + } + kv := resp.Kvs[0] + if string(kv.Value) == attempted { + a.fence = f + a.fenceModRev = kv.ModRevision + a.claimed = true + return nil + } + stored, derr := DecodeFenceValue(kv.Value) + if derr != nil { + return &FenceError{Detail: "checkpoint mod revision moved and the current fence is undecodable"} + } + switch { + case stored.LinkUID != a.cfg.LinkUID: + return &FenceError{Detail: fmt.Sprintf("fence taken over by link %q", stored.LinkUID)} + case stored.Role == RolePrimary: + return &FenceError{ + Detail: "fence role is Primary: cutover completed, mirror writes are forbidden", + } + case stored.Epoch > a.cfg.Epoch: + return &FenceError{Detail: fmt.Sprintf( + "newer agent epoch %d owns the link (this agent is epoch %d)", stored.Epoch, a.cfg.Epoch)} + case stored.Epoch < a.cfg.Epoch: + a.fenceModRev = kv.ModRevision + return errFenceLost + default: + return &FenceError{Detail: "another agent with the same epoch holds the fence"} + } +} + +// isOversized reports whether err is the server's Txn size/op-count limit or +// the gRPC client send cap — permanent errors that must surface the poison +// batch, never be retried as throttling. +func isOversized(err error) bool { + switch rpctypes.Error(err) { + case rpctypes.ErrTooManyOps, rpctypes.ErrRequestTooLarge: + return true + } + if s, ok := status.FromError(err); ok { + return s.Code() == codes.ResourceExhausted && strings.Contains(s.Message(), "larger than max") + } + return false +} diff --git a/pkg/mirroragent/backoff.go b/pkg/mirroragent/backoff.go new file mode 100644 index 00000000..4f72512a --- /dev/null +++ b/pkg/mirroragent/backoff.go @@ -0,0 +1,90 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import "time" + +// backoff produces class-specific retry delays. Connection-class errors get +// a standard exponential curve within [initial, max]; throttling-class +// errors get a more conservative curve derived from the same bounds (start +// 4x higher, cap 2x higher) — the target asked us to slow down, so we slow +// down harder and longer. Quota and permanent classes never route here. +// +// Retry ownership: clientv3 auto-retries Get only on codes.Unavailable; +// Txn/Put/Delete are write-at-most-once (client-retried only when no +// connection was ever established). The engine owns 100% of write-path +// retry/backoff, driven by [Classify] and these curves. A fenced-Txn retry +// after an ambiguous timeout must re-read the fence first — the Txn's own +// success bumped the fence ModRevision. +type backoff struct { + initial time.Duration + max time.Duration + + transientNext time.Duration + throttleNext time.Duration + lastThrottle time.Time +} + +func newBackoff(initial, maxDelay time.Duration) *backoff { + return &backoff{initial: initial, max: maxDelay} +} + +// next returns the delay before the next attempt for the given class and +// advances that class's curve. Classes without a backoff policy (resync, +// quota, permanent) fall back to the transient curve — callers are expected +// to handle them before consulting backoff. +func (b *backoff) next(c Class) time.Duration { + if c == ClassThrottle { + b.lastThrottle = time.Now() + if b.throttleNext == 0 { + b.throttleNext = minDuration(4*b.initial, 2*b.max) + } + d := b.throttleNext + b.throttleNext = minDuration(2*b.throttleNext, 2*b.max) + return d + } + if b.transientNext == 0 { + b.transientNext = b.initial + } + d := b.transientNext + b.transientNext = minDuration(2*b.transientNext, b.max) + return d +} + +// reset clears both curves unconditionally. +func (b *backoff) reset() { + b.transientNext = 0 + b.throttleNext = 0 +} + +// noteSuccess resets the transient curve immediately but the throttle curve +// only after a full max-delay interval without throttle errors: a target +// still intermittently rejecting the write rate must keep escalating instead +// of restarting from the floor after every successful batch. +func (b *backoff) noteSuccess() { + b.transientNext = 0 + if b.throttleNext != 0 && time.Since(b.lastThrottle) >= b.max { + b.throttleNext = 0 + } +} + +func minDuration(a, b time.Duration) time.Duration { + if a < b { + return a + } + return b +} diff --git a/pkg/mirroragent/backoff_test.go b/pkg/mirroragent/backoff_test.go new file mode 100644 index 00000000..46cf7906 --- /dev/null +++ b/pkg/mirroragent/backoff_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBackoffTransientCurve(t *testing.T) { + b := newBackoff(time.Second, 8*time.Second) + assert.Equal(t, 1*time.Second, b.next(ClassTransient)) + assert.Equal(t, 2*time.Second, b.next(ClassTransient)) + assert.Equal(t, 4*time.Second, b.next(ClassTransient)) + assert.Equal(t, 8*time.Second, b.next(ClassTransient)) + assert.Equal(t, 8*time.Second, b.next(ClassTransient), "the transient curve caps at max") +} + +// TestBackoffThrottleCurveDistinct: the target asked us to slow down, so the +// throttle curve starts higher (4x) and caps higher (2x) than the transient +// curve — and the two advance independently. +func TestBackoffThrottleCurveDistinct(t *testing.T) { + b := newBackoff(time.Second, 8*time.Second) + assert.Equal(t, 4*time.Second, b.next(ClassThrottle)) + assert.Equal(t, 8*time.Second, b.next(ClassThrottle)) + assert.Equal(t, 16*time.Second, b.next(ClassThrottle)) + assert.Equal(t, 16*time.Second, b.next(ClassThrottle), "the throttle curve caps at 2x max") + + assert.Equal(t, 1*time.Second, b.next(ClassTransient), + "throttle advancement must not move the transient curve") +} + +func TestBackoffThrottleStartCappedForTightBounds(t *testing.T) { + b := newBackoff(time.Second, time.Second) + assert.Equal(t, 2*time.Second, b.next(ClassThrottle), + "4x initial is capped at 2x max when the bounds are tight") +} + +func TestBackoffReset(t *testing.T) { + b := newBackoff(time.Second, 8*time.Second) + _ = b.next(ClassTransient) + _ = b.next(ClassTransient) + _ = b.next(ClassThrottle) + b.reset() + assert.Equal(t, 1*time.Second, b.next(ClassTransient), "reset restarts the transient curve") + assert.Equal(t, 4*time.Second, b.next(ClassThrottle), "reset restarts the throttle curve") +} + +// TestBackoffNoteSuccessPreservesThrottleCurve: a success right after a +// throttle delay resets only the transient curve — the throttle curve keeps +// escalating across intermittent successes and resets only after a full +// max-delay interval without throttle errors. +func TestBackoffNoteSuccessPreservesThrottleCurve(t *testing.T) { + b := newBackoff(time.Millisecond, 20*time.Millisecond) + _ = b.next(ClassTransient) + _ = b.next(ClassTransient) + first := b.next(ClassThrottle) + b.noteSuccess() + assert.Equal(t, time.Millisecond, b.next(ClassTransient), + "noteSuccess restarts the transient curve immediately") + assert.Greater(t, b.next(ClassThrottle), first, + "the throttle curve must keep escalating across an immediate success") + + time.Sleep(25 * time.Millisecond) // > max: a genuinely healthy stretch + b.noteSuccess() + assert.Equal(t, 4*time.Millisecond, b.next(ClassThrottle), + "a max-delay-long throttle-free stretch restarts the throttle curve") +} + +// TestBackoffNonRetryClassesUseTransientCurve documents the contract that +// resync/quota/permanent never legitimately reach backoff: callers handle +// them first (quota parks on the flat QuotaProbeInterval instead — pinned by +// the TestTargetQuotaExhausted integration test). If one slips through, it +// falls back to the standard curve rather than spinning. +func TestBackoffNonRetryClassesUseTransientCurve(t *testing.T) { + b := newBackoff(time.Second, 8*time.Second) + assert.Equal(t, 1*time.Second, b.next(ClassQuota)) + assert.Equal(t, 2*time.Second, b.next(ClassPermanent)) + assert.Equal(t, 4*time.Second, b.next(ClassResync)) +} diff --git a/pkg/mirroragent/batch.go b/pkg/mirroragent/batch.go new file mode 100644 index 00000000..d31be421 --- /dev/null +++ b/pkg/mirroragent/batch.go @@ -0,0 +1,187 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +// kvOp is one target-side write. Keys are already rewritten; srcKey is +// retained for scan-cursor bookkeeping only. +type kvOp struct { + key string + value string + isDelete bool + srcKey string +} + +func (o kvOp) bytes() int64 { return int64(len(o.key) + len(o.value)) } + +// revGroup is the complete set of in-scope ops of ONE source revision — the +// atom the batcher never splits. Scan pages use synthetic groups (one key +// per group, rev = the scan base) since a snapshot has no per-key revision +// boundaries to preserve. +type revGroup struct { + rev int64 + ops []kvOp +} + +func (g revGroup) bytes() int64 { + var n int64 + for _, o := range g.ops { + n += o.bytes() + } + return n +} + +// flushSet is one target Txn's worth of ops plus the checkpoint metadata +// that rides in the same Txn's reserved op slot. +type flushSet struct { + ops []kvOp + // groups preserves the revision boundaries inside ops, so a set the + // target rejects for its Txn limits can be re-committed at revision + // granularity (one shrink attempt) instead of failing permanently. + groups []revGroup + // watermark is the last complete source revision in the set (or the + // scan base for scan flushes). + watermark int64 + // lastSrcKey is the last source key in the set, for the scan cursor. + lastSrcKey string + // oversized marks a set that alone exceeds maxOps or maxBytes: a single + // source revision applied as one oversized Txn (the checkpoint is held + // until it lands — it rides in the same Txn). If the target rejects it, + // the set is irreducible: the error is permanent and the offending key + // is surfaced. + oversized bool +} + +// batcher coalesces whole revision groups into flush sets, flushing ONLY at +// source-revision boundaries. maxOps already has the checkpoint's reserved +// op slot subtracted (MaxTxnOps - 1); maxBytes is the TxnFlushBytes +// watermark. +type batcher struct { + maxOps int + maxBytes int64 + + pending []revGroup + pendingKeys map[string]struct{} + pendingOps int + pendingBytes int64 +} + +func newBatcher(maxTxnOps int, txnFlushBytes int64) *batcher { + return &batcher{ + // One op slot is always reserved for the checkpoint write. + maxOps: maxTxnOps - 1, + maxBytes: txnFlushBytes, + } +} + +// add appends one whole revision group and returns any flush sets that +// became due. A group is never split: if appending it would overflow the +// pending set, the pending set is flushed first; a group that alone exceeds +// the limits becomes its own oversized flush set. +func (b *batcher) add(g revGroup) []flushSet { + if len(g.ops) == 0 { + return nil + } + var out []flushSet + gBytes := g.bytes() + + // Flush what's pending if this group doesn't fit on top of it, or if it + // touches a key already pending: etcd rejects duplicate keys within one + // Txn (a catch-up watch response batches up to 1000 revisions, so one + // key modified twice in the window would otherwise put two ops on the + // same key into one flush set — a deterministic permanent failure). + if len(b.pending) > 0 && + (b.pendingOps+len(g.ops) > b.maxOps || b.pendingBytes+gBytes > b.maxBytes || + b.overlapsPending(g)) { + if fs := b.flush(); fs != nil { + out = append(out, *fs) + } + } + + b.pending = append(b.pending, g) + if b.pendingKeys == nil { + b.pendingKeys = make(map[string]struct{}, len(g.ops)) + } + for _, op := range g.ops { + b.pendingKeys[op.key] = struct{}{} + } + b.pendingOps += len(g.ops) + b.pendingBytes += gBytes + + // Flush immediately once the watermarks are reached — including the + // oversized single-group case. + if b.pendingOps >= b.maxOps || b.pendingBytes >= b.maxBytes { + if fs := b.flush(); fs != nil { + out = append(out, *fs) + } + } + return out +} + +// flush drains whatever is pending into one flush set (nil when empty). +// Called by add at watermarks and by the apply loop at the end of each +// watch response / scan page so writes are never held waiting for more +// input. +func (b *batcher) flush() *flushSet { + if len(b.pending) == 0 { + return nil + } + fs := flushSet{ + groups: b.pending, + watermark: b.pending[len(b.pending)-1].rev, + oversized: b.pendingOps > b.maxOps || b.pendingBytes > b.maxBytes, + } + fs.ops = make([]kvOp, 0, b.pendingOps) + for _, g := range b.pending { + fs.ops = append(fs.ops, g.ops...) + } + fs.lastSrcKey = fs.ops[len(fs.ops)-1].srcKey + b.pending = nil + b.pendingKeys = nil + b.pendingOps = 0 + b.pendingBytes = 0 + return &fs +} + +// overlapsPending reports whether any of g's keys is already pending. +func (b *batcher) overlapsPending(g revGroup) bool { + for _, op := range g.ops { + if _, ok := b.pendingKeys[op.key]; ok { + return true + } + } + return false +} + +// groupByRevision converts an ordered event stream (already rewritten and +// filtered) into revision groups, preserving order. Events of one revision +// are always contiguous in an etcd watch stream. +func groupByRevision(ops []kvOp, revs []int64) []revGroup { + if len(ops) != len(revs) || len(ops) == 0 { + return nil + } + var groups []revGroup + cur := revGroup{rev: revs[0]} + for i, op := range ops { + if revs[i] != cur.rev { + groups = append(groups, cur) + cur = revGroup{rev: revs[i]} + } + cur.ops = append(cur.ops, op) + } + groups = append(groups, cur) + return groups +} diff --git a/pkg/mirroragent/batch_test.go b/pkg/mirroragent/batch_test.go new file mode 100644 index 00000000..0cefbad7 --- /dev/null +++ b/pkg/mirroragent/batch_test.go @@ -0,0 +1,192 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mkGroup builds a revGroup of n one-byte-key/valueLen-byte-value ops. +func mkGroup(rev int64, n, valueLen int) revGroup { + g := revGroup{rev: rev} + for i := range n { + g.ops = append(g.ops, kvOp{ + key: fmt.Sprintf("k%d-%d", rev, i), + value: string(make([]byte, valueLen)), + srcKey: fmt.Sprintf("s%d-%d", rev, i), + }) + } + return g +} + +// TestBatcherReservesCheckpointSlot pins the MaxTxnOps-1 invariant: with +// MaxTxnOps=5 the batcher flushes at exactly 4 data ops — one op slot is +// always reserved for the checkpoint write riding the same Txn. +func TestBatcherReservesCheckpointSlot(t *testing.T) { + b := newBatcher(5, 1<<20) + var flushed []flushSet + for rev := int64(1); rev <= 3; rev++ { + flushed = append(flushed, b.add(mkGroup(rev, 1, 1))...) + } + require.Empty(t, flushed, "3 ops must still be pending below the 4-op watermark") + + flushed = b.add(mkGroup(4, 1, 1)) + require.Len(t, flushed, 1, "the 4th op hits MaxTxnOps-1 exactly and must flush") + assert.Len(t, flushed[0].ops, 4) + assert.False(t, flushed[0].oversized) + assert.EqualValues(t, 4, flushed[0].watermark, "watermark is the last complete revision") + assert.Nil(t, b.flush(), "nothing may remain pending after the watermark flush") +} + +// TestBatcherNeverSplitsRevision: a revision group that does not fit on top +// of the pending set flushes the pending set first and stays whole. +func TestBatcherNeverSplitsRevision(t *testing.T) { + b := newBatcher(6, 1<<20) // 5 data op slots + first := b.add(mkGroup(10, 3, 1)) + require.Empty(t, first) + + // 3 pending + 3 incoming > 5: pending flushes alone, the new group pends. + flushed := b.add(mkGroup(11, 3, 1)) + require.Len(t, flushed, 1) + assert.Len(t, flushed[0].ops, 3) + assert.EqualValues(t, 10, flushed[0].watermark) + for _, op := range flushed[0].ops { + assert.Contains(t, op.srcKey, "s10-", "revision 11 ops must not leak into revision 10's flush") + } + + rest := b.flush() + require.NotNil(t, rest) + assert.Len(t, rest.ops, 3) + assert.EqualValues(t, 11, rest.watermark) +} + +// TestBatcherByteWatermark: the TxnFlushBytes boundary triggers a flush even +// when the op count is far below MaxTxnOps — the pending set flushes alone +// (at its revision boundary) when the next revision would push it past the +// byte watermark. +func TestBatcherByteWatermark(t *testing.T) { + b := newBatcher(100, 1000) + require.Empty(t, b.add(mkGroup(1, 1, 400))) + + flushed := b.add(mkGroup(2, 1, 700)) + require.Len(t, flushed, 1, "revision 2 does not fit on top: revision 1 must flush first") + assert.Len(t, flushed[0].ops, 1) + assert.EqualValues(t, 1, flushed[0].watermark) + assert.False(t, flushed[0].oversized) + + rest := b.flush() + require.NotNil(t, rest) + assert.Len(t, rest.ops, 1) + assert.EqualValues(t, 2, rest.watermark) +} + +// TestBatcherOversizedSingleRevision: one revision alone above both +// watermarks becomes its own flush set, marked oversized, never split. +func TestBatcherOversizedSingleRevision(t *testing.T) { + b := newBatcher(4, 100) // 3 data op slots + flushed := b.add(mkGroup(7, 9, 50)) + require.Len(t, flushed, 1, "an oversized revision must flush immediately as one set") + fs := flushed[0] + assert.True(t, fs.oversized) + assert.Len(t, fs.ops, 9, "all 9 ops of the revision stay in ONE Txn") + assert.EqualValues(t, 7, fs.watermark) + assert.Equal(t, "s7-8", fs.lastSrcKey) + assert.Nil(t, b.flush()) +} + +// TestBatcherOversizedDoesNotDragNeighbors: pending small revisions flush +// separately before an oversized revision arrives. +func TestBatcherOversizedDoesNotDragNeighbors(t *testing.T) { + b := newBatcher(10, 1<<20) + require.Empty(t, b.add(mkGroup(1, 2, 1))) + + flushed := b.add(mkGroup(2, 20, 1)) + require.Len(t, flushed, 2, "pending set flushes first, then the oversized revision alone") + assert.Len(t, flushed[0].ops, 2) + assert.False(t, flushed[0].oversized) + assert.Len(t, flushed[1].ops, 20) + assert.True(t, flushed[1].oversized) +} + +// TestBatcherFlushesOnDuplicateKey: etcd rejects duplicate keys within one +// Txn, and an unsynced-watcher catch-up response batches up to 1000 +// revisions — so a key modified twice in the window must split the flush at +// the revision boundary instead of producing a poison Txn. +func TestBatcherFlushesOnDuplicateKey(t *testing.T) { + b := newBatcher(100, 1<<20) + g1 := revGroup{rev: 5, ops: []kvOp{{key: "/dst/a", value: "v1"}, {key: "/dst/b", value: "v1"}}} + require.Empty(t, b.add(g1)) + + // Revision 6 touches /dst/a again: revision 5 must flush alone first. + g2 := revGroup{rev: 6, ops: []kvOp{{key: "/dst/a", value: "v2"}}} + flushed := b.add(g2) + require.Len(t, flushed, 1, "a duplicate key must force a flush at the revision boundary") + assert.EqualValues(t, 5, flushed[0].watermark) + assert.Len(t, flushed[0].ops, 2) + + rest := b.flush() + require.NotNil(t, rest) + assert.Len(t, rest.ops, 1) + assert.EqualValues(t, 6, rest.watermark) + + // Disjoint keys still coalesce. + require.Empty(t, b.add(revGroup{rev: 7, ops: []kvOp{{key: "/dst/c"}}})) + require.Empty(t, b.add(revGroup{rev: 8, ops: []kvOp{{key: "/dst/d"}}})) + both := b.flush() + require.NotNil(t, both) + assert.Len(t, both.ops, 2, "distinct keys must keep coalescing across revisions") +} + +// TestBatcherFlushSetRetainsGroups: revision boundaries survive into the +// flush set, so a target-limit rejection can be re-committed at revision +// granularity. +func TestBatcherFlushSetRetainsGroups(t *testing.T) { + b := newBatcher(100, 1<<20) + require.Empty(t, b.add(mkGroup(1, 2, 1))) + require.Empty(t, b.add(mkGroup(2, 3, 1))) + fs := b.flush() + require.NotNil(t, fs) + require.Len(t, fs.groups, 2) + assert.EqualValues(t, 1, fs.groups[0].rev) + assert.Len(t, fs.groups[0].ops, 2) + assert.EqualValues(t, 2, fs.groups[1].rev) + assert.Len(t, fs.groups[1].ops, 3) +} + +func TestGroupByRevision(t *testing.T) { + ops := []kvOp{ + {key: "a"}, {key: "b"}, // rev 5 + {key: "c"}, // rev 6 + {key: "d"}, {key: "e"}, // rev 9 + } + revs := []int64{5, 5, 6, 9, 9} + groups := groupByRevision(ops, revs) + require.Len(t, groups, 3) + assert.EqualValues(t, 5, groups[0].rev) + assert.Len(t, groups[0].ops, 2) + assert.EqualValues(t, 6, groups[1].rev) + assert.Len(t, groups[1].ops, 1) + assert.EqualValues(t, 9, groups[2].rev) + assert.Len(t, groups[2].ops, 2) + + assert.Nil(t, groupByRevision(nil, nil)) + assert.Nil(t, groupByRevision(ops, revs[:2]), "length mismatch must yield nothing") +} diff --git a/pkg/mirroragent/client.go b/pkg/mirroragent/client.go new file mode 100644 index 00000000..6974c7de --- /dev/null +++ b/pkg/mirroragent/client.go @@ -0,0 +1,62 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "crypto/tls" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// Client is the subset of *clientv3.Client the engine uses on each side. +// The source side needs KV (Get) + Watcher + Status; the target side needs +// KV (Get/Txn) + Status. *clientv3.Client satisfies it directly. +type Client interface { + clientv3.KV + clientv3.Watcher + Status(ctx context.Context, endpoint string) (*clientv3.StatusResponse, error) + Endpoints() []string +} + +var _ Client = (*clientv3.Client)(nil) + +// Keepalive settings the engine's liveness machinery assumes: without +// client-driven keepalives, NLB (350s) and Cloud NAT (1200s) idle timeouts +// silently kill quiet watches on the cross-cloud path this engine exists +// for. +const ( + DialKeepAliveTime = 25 * time.Second + DialKeepAliveTimeout = 10 * time.Second +) + +// NewClientConfig returns a clientv3.Config wired the way the engine +// requires: keepalives on (including without active streams) and a bounded +// dial. Callers own TLS/auth material — this library never reads Secrets. +// Per-unary request deadlines are applied inside the engine from +// Config.RequestTimeout, not here. +func NewClientConfig(endpoints []string, tlsConfig *tls.Config, dialTimeout time.Duration) clientv3.Config { + return clientv3.Config{ + Endpoints: endpoints, + DialTimeout: dialTimeout, + DialKeepAliveTime: DialKeepAliveTime, + DialKeepAliveTimeout: DialKeepAliveTimeout, + PermitWithoutStream: true, + TLS: tlsConfig, + } +} diff --git a/pkg/mirroragent/config.go b/pkg/mirroragent/config.go new file mode 100644 index 00000000..10f5a312 --- /dev/null +++ b/pkg/mirroragent/config.go @@ -0,0 +1,331 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Mode selects the agent's operating mode. Mirrors EtcdMirrorMode in +// api/v1alpha1. +type Mode string + +const ( + // ModeSync is normal continuous replication. + ModeSync Mode = "Sync" + // ModeDrain prepares for cutover: the agent records the source revision + // observed when the drain starts, keeps replicating until the checkpoint + // watermark reaches it, runs a verification pass, then flips the fence + // key's role to Primary so any straggler apply fails its mod-revision + // compare loudly, and returns from Run. + ModeDrain Mode = "Drain" +) + +// InitialSyncMode governs pre-existing keys under the effective destination +// prefix at genesis. Mirrors EtcdMirrorInitialSyncMode in api/v1alpha1. +type InitialSyncMode string + +const ( + // InitialSyncRequireEmpty refuses to start if the destination prefix + // already holds any key (the reserved checkpoint key is excluded by + // exact match). Re-arms whenever the checkpoint is invalidated by a + // cluster-identity mismatch. + InitialSyncRequireEmpty InitialSyncMode = "RequireEmpty" + // InitialSyncOverwrite scans and writes over whatever is there. Keys + // present on the target but absent on the source are left alone. + InitialSyncOverwrite InitialSyncMode = "Overwrite" + // InitialSyncOverwriteAndPrune is Overwrite plus one mandatory + // orphan-prune pass after the scan, making reversal onto a + // previously-populated prefix (failback) a first-class correct operation. + InitialSyncOverwriteAndPrune InitialSyncMode = "OverwriteAndPrune" +) + +// DefaultCheckpointKeySuffix is appended to the effective destination prefix +// to form the default reserved checkpoint key. The \x00 byte after the +// prefix cannot collide with any real key under it. +const DefaultCheckpointKeySuffix = "\x00etcdmirror-checkpoint" + +// Defaults, matching the CRD field defaults in api/v1alpha1. +const ( + DefaultMaxTxnOps = 128 + DefaultTxnFlushBytes = 1 << 20 // 1Mi + DefaultPageKeyLimit = 512 + DefaultPageBytes = 1 << 20 // 1Mi + DefaultRequestTimeout = 30 * time.Second + DefaultBackoffInitial = 1 * time.Second + DefaultBackoffMax = 30 * time.Second + DefaultReconcilePeriod = time.Hour + + // DefaultWatchBufferBytes bounds the in-memory replay buffer for watch + // events observed from R0 while the genesis scan runs. Must stay in + // lockstep with the EtcdMirror CRD's spec.sync.watchBufferBytes default. + DefaultWatchBufferBytes = 16 << 20 // 16Mi + // DefaultProgressInterval is how often the agent issues a client-driven + // RequestProgress on the source watch (server-side notify intervals are + // uncontrollable on foreign clusters). + DefaultProgressInterval = 45 * time.Second + // DefaultResyncLoopThreshold is how many consecutive forced resyncs + // (without reaching steady state in between) trip the livelock detector + // — the signature of source retention < scan+drain time. + DefaultResyncLoopThreshold = 3 + // DefaultQuotaProbeInterval is how often a quota-exhausted (NOSPACE) + // agent re-probes the target. Deliberately a slow flat poll, not + // backoff: quota exhaustion only heals when an operator acts. + DefaultQuotaProbeInterval = time.Minute +) + +// Config is the engine's plain-Go configuration. Fields mirror the +// EtcdMirror CRD spec (api/v1alpha1/etcdmirror_types.go); doc comments here +// and there are the PR1<->PR2 alignment contract. +type Config struct { + // LinkUID uniquely identifies this mirror link (source, target, prefix + // tuple); typically the EtcdMirror object's UID. Stamped into the fence + // key: a checkpoint carrying a different LinkUID is another link's fence + // and the agent refuses to touch it. Required. + LinkUID string + + // Epoch is this agent generation within the link, monotonically + // increased by the supervisor on each re-deploy. An agent that finds a + // higher epoch in the fence stops permanently (a newer generation owns + // the link); a lower stored epoch is taken over via the fenced write + // path. Must be >= 1. + Epoch int64 + + // Mode selects continuous replication (Sync, the default) or a cutover + // drain (Drain). See ModeDrain; RequestDrain flips a running agent. + Mode Mode + + // SourcePrefix scopes which source keys are mirrored; empty means the + // whole keyspace. + SourcePrefix string + // TargetPrefix is the prefix under which mirrored keys land. + TargetPrefix string + // DestPrefix is the middle term of the rewrite formula + // + // key' = TargetPrefix + DestPrefix + TrimPrefix(key, SourcePrefix) + // + // Default "" means the source prefix is stripped and key remainders land + // directly under TargetPrefix. + DestPrefix string + // ExcludePrefixes lists source key prefixes (full source-side keys) + // skipped entirely: not scanned, not watched, not counted, not pruned. + // Nested or duplicate entries are normalized away at defaulting time (a + // prefix covered by another is dropped) so range subtraction and count + // corrections each see a disjoint set. + ExcludePrefixes []string + + // InitialSyncMode governs pre-existing destination keys at genesis. + // Defaults to RequireEmpty. + InitialSyncMode InitialSyncMode + // StartRevision, when > 0, skips the genesis scan entirely and starts + // watching from StartRevision+1 (for fidelity-preserving snapshot + // seeds). Requires InitialSyncMode Overwrite or OverwriteAndPrune. + StartRevision int64 + + // CheckpointKey overrides the reserved checkpoint/fence key on the + // target. Defaults to the effective destination prefix + + // DefaultCheckpointKeySuffix. The key is excluded by exact match from + // scans, counts, prune passes, and the RequireEmpty check; the target + // RBAC grant must cover it. + CheckpointKey string + + // MaxTxnOps bounds how many operations the agent batches into a single + // target Txn, including the reserved checkpoint-write slot. Must not + // exceed the target's --max-txn-ops. Defaults to 128; minimum 2. + MaxTxnOps int + // TxnFlushBytes is the byte watermark at which a batch is flushed (at + // the next source-revision boundary). Defaults to 1Mi. + TxnFlushBytes int64 + // PageKeyLimit bounds keys per source scan page. The scan is pull-based, + // one page in flight — no read-ahead. Defaults to 512. + PageKeyLimit int + // PageBytes bounds bytes per source scan page. etcd Range has no byte + // limit, so this is enforced adaptively: the next page's key limit is + // derived from the observed bytes/key of the previous page. Defaults + // to 1Mi. + PageBytes int64 + // MaxOpsPerSecond rate-limits target writes (puts+deletes/sec, token + // bucket), applied to both the genesis scan and watch-driven applies. + // Zero means unlimited. + MaxOpsPerSecond int + // RequestTimeout is the per-RPC context deadline applied to every unary + // call on both sides (watches excluded — they are long-lived by design + // and covered by progress-notification liveness instead). Defaults + // to 30s. + RequestTimeout time.Duration + + // BackoffInitialDelay/BackoffMaxDelay bound the retry loop for + // connection-class errors. Throttling-class errors use a more + // conservative curve derived from the same bounds; quota exhaustion and + // permanent errors are never retried through this loop. Defaults: + // 1s to 30s. + BackoffInitialDelay time.Duration + BackoffMaxDelay time.Duration + + // ReconcileInterval will enable the periodic full diff-and-repair pass + // when > 0. NOT YET WIRED in this rung: no periodic scheduler exists + // until the reconciliation-promotion rung lands, so setting it is + // currently a no-op. Independent of this, one reconciliation-with-delete + // pass always runs after any forced resync (mark-and-sweep), as the + // OverwriteAndPrune genesis pass, and before the Drain verification. + ReconcileInterval time.Duration + // ReconcileDeleteOrphans will allow the PERIODIC pass to delete target + // keys with no source counterpart. NOT YET WIRED (see ReconcileInterval). + // Forced-resync sweeps and OverwriteAndPrune always delete orphans. + ReconcileDeleteOrphans bool + + // WatchBufferBytes bounds the memory used to buffer watch events + // observed from R0 while the genesis scan runs (the reflector replay + // buffer). On overflow the agent cancels the source watch and restarts + // the scan from a fresh R0 — a bounded retry instead of unbounded growth + // when source churn outruns scan+apply throughput. The restart is + // surfaced as Snapshot.LastScanRestartCause WatchBufferOverflow (the + // controller maps it to the InitialSyncCompactionRaced event) and + // repeated overflows count toward the resync-loop detector. Defaults to + // DefaultWatchBufferBytes; must stay in lockstep with the CRD's + // spec.sync.watchBufferBytes. Must be >= 0 (0 = default). + WatchBufferBytes int64 + + // Agent-internal knobs (not part of the CRD in v1). + ProgressInterval time.Duration + ResyncLoopThreshold int + QuotaProbeInterval time.Duration +} + +// withDefaults returns a copy with zero fields replaced by defaults. +func (c Config) withDefaults() Config { + if c.Mode == "" { + c.Mode = ModeSync + } + if c.InitialSyncMode == "" { + c.InitialSyncMode = InitialSyncRequireEmpty + } + if c.CheckpointKey == "" { + c.CheckpointKey = c.EffectiveDestPrefix() + DefaultCheckpointKeySuffix + } + if c.MaxTxnOps == 0 { + c.MaxTxnOps = DefaultMaxTxnOps + } + if c.TxnFlushBytes == 0 { + c.TxnFlushBytes = DefaultTxnFlushBytes + } + if c.PageKeyLimit == 0 { + c.PageKeyLimit = DefaultPageKeyLimit + } + if c.PageBytes == 0 { + c.PageBytes = DefaultPageBytes + } + if c.RequestTimeout == 0 { + c.RequestTimeout = DefaultRequestTimeout + } + if c.BackoffInitialDelay == 0 { + c.BackoffInitialDelay = DefaultBackoffInitial + } + if c.BackoffMaxDelay == 0 { + c.BackoffMaxDelay = DefaultBackoffMax + } + if c.WatchBufferBytes == 0 { + c.WatchBufferBytes = DefaultWatchBufferBytes + } + if c.ProgressInterval == 0 { + c.ProgressInterval = DefaultProgressInterval + } + if c.ResyncLoopThreshold == 0 { + c.ResyncLoopThreshold = DefaultResyncLoopThreshold + } + if c.QuotaProbeInterval == 0 { + c.QuotaProbeInterval = DefaultQuotaProbeInterval + } + c.ExcludePrefixes = normalizePrefixes(c.ExcludePrefixes) + return c +} + +// normalizePrefixes sorts prefixes and drops any entry covered by another +// (nested or duplicate). Both the scan-range subtraction and the per-prefix +// count corrections assume a disjoint set: a key covered by two overlapping +// entries must never be subtracted from a count twice. +func normalizePrefixes(in []string) []string { + if len(in) < 2 { + return in + } + sorted := make([]string, len(in)) + copy(sorted, in) + sort.Strings(sorted) + out := sorted[:0] + for _, p := range sorted { + if len(out) > 0 && strings.HasPrefix(p, out[len(out)-1]) { + continue + } + out = append(out, p) + } + return out +} + +// Validate checks the configuration after defaulting. +func (c Config) Validate() error { + if c.LinkUID == "" { + return fmt.Errorf("linkUID is required") + } + if c.Epoch < 1 { + return fmt.Errorf("epoch must be >= 1, got %d", c.Epoch) + } + if c.Mode != ModeSync && c.Mode != ModeDrain { + return fmt.Errorf("invalid mode %q", c.Mode) + } + switch c.InitialSyncMode { + case InitialSyncRequireEmpty, InitialSyncOverwrite, InitialSyncOverwriteAndPrune: + default: + return fmt.Errorf("invalid initialSyncMode %q", c.InitialSyncMode) + } + if c.StartRevision < 0 { + return fmt.Errorf("startRevision must be >= 0, got %d", c.StartRevision) + } + if c.StartRevision > 0 && c.InitialSyncMode == InitialSyncRequireEmpty { + return fmt.Errorf("startRevision requires initialSyncMode Overwrite or OverwriteAndPrune") + } + if c.MaxTxnOps < 2 { + return fmt.Errorf("maxTxnOps must be >= 2 (one op slot is reserved for the checkpoint), got %d", c.MaxTxnOps) + } + if c.TxnFlushBytes < 1 || c.PageBytes < 1 || c.PageKeyLimit < 1 { + return fmt.Errorf("txnFlushBytes, pageBytes and pageKeyLimit must be positive") + } + if c.MaxOpsPerSecond < 0 { + return fmt.Errorf("maxOpsPerSecond must be >= 0, got %d", c.MaxOpsPerSecond) + } + if c.WatchBufferBytes < 0 { + return fmt.Errorf("watchBufferBytes must be >= 0, got %d", c.WatchBufferBytes) + } + if !strings.HasPrefix(c.CheckpointKey, c.EffectiveDestPrefix()) { + return fmt.Errorf("checkpointKey must live under the effective destination prefix") + } + for _, p := range c.ExcludePrefixes { + if p == "" { + return fmt.Errorf("excludePrefixes entries must be non-empty") + } + } + return nil +} + +// EffectiveDestPrefix is TargetPrefix + DestPrefix: the target-side prefix +// every mirrored key lands under, and the range RequireEmpty and prune +// passes operate on. +func (c Config) EffectiveDestPrefix() string { + return c.TargetPrefix + c.DestPrefix +} diff --git a/pkg/mirroragent/config_test.go b/pkg/mirroragent/config_test.go new file mode 100644 index 00000000..29f82ca4 --- /dev/null +++ b/pkg/mirroragent/config_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func minimalCfg() Config { + return Config{LinkUID: "l", Epoch: 1, SourcePrefix: "/s/", TargetPrefix: "/d/"} +} + +func TestConfigDefaults(t *testing.T) { + c := minimalCfg().withDefaults() + require.NoError(t, c.Validate()) + + assert.Equal(t, ModeSync, c.Mode) + assert.Equal(t, InitialSyncRequireEmpty, c.InitialSyncMode) + assert.Equal(t, "/d/"+DefaultCheckpointKeySuffix, c.CheckpointKey) + assert.Equal(t, DefaultMaxTxnOps, c.MaxTxnOps) + assert.EqualValues(t, DefaultTxnFlushBytes, c.TxnFlushBytes) + assert.Equal(t, DefaultPageKeyLimit, c.PageKeyLimit) + assert.EqualValues(t, DefaultPageBytes, c.PageBytes) + assert.Equal(t, DefaultRequestTimeout, c.RequestTimeout) + assert.EqualValues(t, DefaultWatchBufferBytes, c.WatchBufferBytes) + assert.Equal(t, DefaultResyncLoopThreshold, c.ResyncLoopThreshold) + assert.Equal(t, DefaultProgressInterval, c.ProgressInterval) + assert.Equal(t, DefaultQuotaProbeInterval, c.QuotaProbeInterval) +} + +func TestConfigValidate(t *testing.T) { + cases := []struct { + name string + mutate func(*Config) + wantErr string + }{ + {name: "valid", mutate: func(*Config) {}}, + {name: "missing linkUID", mutate: func(c *Config) { c.LinkUID = "" }, + wantErr: "linkUID"}, + {name: "epoch below one", mutate: func(c *Config) { c.Epoch = 0 }, + wantErr: "epoch"}, + {name: "bad mode", mutate: func(c *Config) { c.Mode = "Paused" }, + wantErr: "mode"}, + {name: "bad initialSyncMode", mutate: func(c *Config) { c.InitialSyncMode = "Merge" }, + wantErr: "initialSyncMode"}, + {name: "negative startRevision", mutate: func(c *Config) { c.StartRevision = -1 }, + wantErr: "startRevision"}, + // Defense-in-depth mirror of the CRD's CEL rule: a startRevision + // seed skips the scan, so RequireEmpty could never be satisfied + // meaningfully. + {name: "startRevision requires overwrite", mutate: func(c *Config) { c.StartRevision = 10 }, + wantErr: "startRevision requires initialSyncMode Overwrite"}, + {name: "startRevision with overwrite ok", mutate: func(c *Config) { + c.StartRevision = 10 + c.InitialSyncMode = InitialSyncOverwrite + }}, + {name: "maxTxnOps below two", mutate: func(c *Config) { c.MaxTxnOps = 1 }, + wantErr: "maxTxnOps must be >= 2"}, + {name: "negative txnFlushBytes", mutate: func(c *Config) { c.TxnFlushBytes = -1 }, + wantErr: "positive"}, + {name: "negative pageBytes", mutate: func(c *Config) { c.PageBytes = -1 }, + wantErr: "positive"}, + {name: "negative pageKeyLimit", mutate: func(c *Config) { c.PageKeyLimit = -1 }, + wantErr: "positive"}, + {name: "negative maxOpsPerSecond", mutate: func(c *Config) { c.MaxOpsPerSecond = -1 }, + wantErr: "maxOpsPerSecond"}, + {name: "negative watchBufferBytes", mutate: func(c *Config) { c.WatchBufferBytes = -1 }, + wantErr: "watchBufferBytes"}, + {name: "checkpoint key outside dest prefix", mutate: func(c *Config) { c.CheckpointKey = "/elsewhere" }, + wantErr: "checkpointKey"}, + {name: "empty exclude entry", mutate: func(c *Config) { c.ExcludePrefixes = []string{""} }, + wantErr: "excludePrefixes"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := minimalCfg() + tc.mutate(&c) + err := c.withDefaults().Validate() + if tc.wantErr == "" { + assert.NoError(t, err) + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// TestNormalizePrefixes: nested/duplicate exclude entries are collapsed at +// defaulting time — count corrections assume a disjoint set, and a key +// covered by two overlapping entries must never be subtracted twice +// (previously a byte-exact converged drain could fail verification). +func TestNormalizePrefixes(t *testing.T) { + c := minimalCfg() + c.ExcludePrefixes = []string{"/s/tmp/cache/", "/s/tmp/", "/s/other/", "/s/tmp/"} + c = c.withDefaults() + require.NoError(t, c.Validate()) + assert.Equal(t, []string{"/s/other/", "/s/tmp/"}, c.ExcludePrefixes) + + c2 := minimalCfg() + c2.ExcludePrefixes = []string{"/s/a/"} + assert.Equal(t, []string{"/s/a/"}, c2.withDefaults().ExcludePrefixes) + c3 := minimalCfg() + assert.Empty(t, c3.withDefaults().ExcludePrefixes) +} + +// TestCheckpointKeyConvention: the default reserved key uses the +// \x00-after-prefix convention, which no real key under the prefix can +// collide with, and lives under the effective destination prefix. +func TestCheckpointKeyConvention(t *testing.T) { + c := Config{LinkUID: "l", Epoch: 1, SourcePrefix: "/s/", TargetPrefix: "/d/", DestPrefix: "sub/"} + c = c.withDefaults() + require.NoError(t, c.Validate()) + assert.Equal(t, "/d/sub/", c.EffectiveDestPrefix()) + assert.Equal(t, "/d/sub/\x00etcdmirror-checkpoint", c.CheckpointKey) +} diff --git a/pkg/mirroragent/doc.go b/pkg/mirroragent/doc.go new file mode 100644 index 00000000..95fa3427 --- /dev/null +++ b/pkg/mirroragent/doc.go @@ -0,0 +1,101 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package mirroragent implements the EtcdMirror replication engine: a +// continuous, one-way key-range sync from a source etcd cluster into a +// target etcd cluster. It is a pure library — no Kubernetes API types, no +// binary, no metrics endpoint; progress is exposed through [Agent.Snapshot]. +// +// Engine invariants (the doc comments in api/v1alpha1/etcdmirror_types.go +// state the same contracts on the CRD side; keep both in sync): +// +// - Genesis is an UNPINNED chunked scan with the watch already open from +// the revision observed before the scan started; buffered events are +// replayed over the scanned base (reflector pattern). Pages read at the +// current revision, so mid-scan compaction cannot fail the scan. +// - The checkpoint (source-revision watermark plus {linkUID, epoch, role}) +// lives IN THE TARGET etcd at a reserved key, written in the SAME Txn as +// every applied batch and fenced with a mod-revision compare on EVERY +// write path (applies, reconciliation repairs, prune deletes). The +// reserved key is excluded by exact match from scans, counts, prune +// passes, and the RequireEmpty check. +// - Target Txns flush ONLY at source-revision boundaries: a source +// revision's events are never split across Txns, whole revisions are +// coalesced up to the MaxTxnOps/TxnFlushBytes watermarks, and one op +// slot is always reserved for the checkpoint write. A single revision +// larger than MaxTxnOps is applied as one oversized Txn with the +// checkpoint held until it lands. +// - Key rewrite is one formula, anchored, never a substring replace: +// key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix). +// - Errors are classified (see [Classify]): compaction forces a resync +// (with a livelock detector), NOSPACE parks the agent until an operator +// acts, oversized requests are permanent with the offending key surfaced +// redacted (see [RedactKey]; values never logged), throttling backs off +// on its own curve. +// - Memory is bounded: one in-flight scan page (byte-bounded, adaptive) +// and a byte-bounded replay buffer for the watch opened before the scan; +// on overflow the agent cancels the source watch and restarts the scan +// from a fresh R0 (a bounded retry, surfaced with cause +// WatchBufferOverflow) instead of growing. +// +// # Why mid-scan compaction cannot wedge the engine +// +// 1. At scan start one linearizable Get (WithCountOnly) returns +// Header.Revision = R0 and the total Count in a single RPC. +// 2. The watch opens at R0+1 BEFORE any scan page is read. R0 was observed +// this instant by a linearizable read, so it cannot already be compacted. +// 3. Every scan page is an UNPINNED Get (no WithRev). ErrCompacted is a +// property of reads pinned below the compact revision; an unpinned read +// is immune by construction, at every point during the scan, regardless +// of concurrent compactions. The failure class is removed, not detected +// and retried. +// 4. The only remaining compaction hazard is the watch stream itself going +// quiet long enough that a re-Watch(WithRev(watermark+1)) lands below +// the compact revision — identical in shape during InitialSync and +// steady state, handled by one mechanism: forced resync with a +// mandatory mark-and-sweep prune. +// +// Scan and watch are not sequential phases with a handoff race: the watch is +// live for the entire scan, and scan writes may interleave with replayed +// watch writes because both write the same idempotent final value for a +// given key — convergence to the last-write value; a duplicate Put of +// identical content is a correctness no-op, only a bounded efficiency cost. +// +// # Why the fence needs only a mod_revision compare +// +// etcd's Compare supports whole-value/mod_revision/version/create_revision +// predicates only — no field-level JSON predicates. All safety rests on one +// discipline, identical on EVERY write path (apply, reconcile repair, prune, +// cutover role-flip): +// +// If(Compare(ModRevision(fenceKey), "=", observedModRev)). +// Then(dataOps..., Put(fenceKey, next)) +// // on !Succeeded: re-read, recompute, retry — never blind re-Commit +// +// linkUID/epoch/role are payload for humans and the engine's state machine, +// never comparison predicates. Cutover safety falls out for free: the +// role-flip Txn bumps ModRevision, so any writer holding a pre-flip +// observedModRev fails its next compare loudly — indistinguishable from an +// ordinary concurrent-writer collision, no special role-check code needed. +// +// # Retry ownership +// +// clientv3 auto-retries Get only on codes.Unavailable; Txn/Put/Delete are +// write-at-most-once (client-retried only when no connection was ever +// established). The engine owns 100% of write-path retry/backoff. A +// fenced-Txn retry after an ambiguous timeout must re-read the fence first — +// the Txn's own success bumped the fence ModRevision. +package mirroragent diff --git a/pkg/mirroragent/errors.go b/pkg/mirroragent/errors.go new file mode 100644 index 00000000..8d2a2ca0 --- /dev/null +++ b/pkg/mirroragent/errors.go @@ -0,0 +1,290 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" +) + +// Class is the engine's error taxonomy. Every failure the retry loop sees is +// classified into exactly one class, and each class has its own recovery +// policy — misclassification (e.g. labelling an oversized Txn "throttling") +// is itself a bug class this taxonomy exists to eliminate. +type Class string + +const ( + // ClassTransient covers connection-class errors (unavailable, timeouts, + // leader loss): retried with the standard exponential backoff. + ClassTransient Class = "Transient" + // ClassThrottle covers the target rejecting the write rate + // (rpctypes.ErrTooManyRequests and other rate-flavored + // ResourceExhausted): retried on a more conservative curve, distinct + // from ClassTransient and never conflated with ClassQuota. + ClassThrottle Class = "Throttle" + // ClassResync covers conditions that invalidate the watch/checkpoint + // position (source compaction outran the watch, a bound cluster identity + // changed): the agent runs a forced resync (scan + mandatory + // mark-and-sweep), counted by the resync-loop livelock detector. + ClassResync Class = "Resync" + // ClassQuota is rpctypes.ErrNoSpace from the target: permanent until an + // operator compacts/defrags/disarms. The agent parks on a slow flat + // probe instead of burning backoff against a full quota. + ClassQuota Class = "Quota" + // ClassPermanent is never retried: oversized requests (server + // "request is too large"/"too many operations" and the client 2MiB + // send cap), auth/permission misconfiguration, fence violations, + // corrupt/unknown-version checkpoints, version-floor failures, + // RequireEmpty violations. + ClassPermanent Class = "Permanent" +) + +// ResyncReason distinguishes why a forced resync was required. +type ResyncReason string + +const ( + // ResyncReasonCompacted: source compaction outran the watch (restart or + // pause longer than retention). + ResyncReasonCompacted ResyncReason = "Compacted" + // ResyncReasonClusterIDMismatch: a bound cluster ID (source or target) + // no longer matches the probed cluster; genesis is forced and + // RequireEmpty re-arms. + ResyncReasonClusterIDMismatch ResyncReason = "ClusterIDMismatch" +) + +// ResyncError forces a full resync (genesis scan + mark-and-sweep). +type ResyncError struct { + Reason ResyncReason + Cause error +} + +func (e *ResyncError) Error() string { + return fmt.Sprintf("forced resync required (%s): %v", e.Reason, e.Cause) +} +func (e *ResyncError) Unwrap() error { return e.Cause } + +// CheckpointInvalidError reports a corrupt or unknown-version checkpoint. +// Distinct from an absent checkpoint (plain genesis). PERMANENT: an +// undecodable fence cannot prove link ownership, epoch ordering, or role, so +// overwriting it (and running a resync's mandatory prune over data it may be +// protecting) is never safe. The operator must inspect the reserved key and +// delete it to recover. +type CheckpointInvalidError struct { + Reason string +} + +func (e *CheckpointInvalidError) Error() string { + return "checkpoint invalid: " + e.Reason +} + +// ConfigError reports a client/engine configuration defect detected at +// runtime (e.g. a client with no endpoints). Permanent. +type ConfigError struct { + Detail string +} + +func (e *ConfigError) Error() string { return "configuration error: " + e.Detail } + +// FenceError is a fence violation: the reserved key's mod revision moved +// under us (another agent generation took over, or the role flipped to +// Primary at cutover). Permanent — this agent must never write again. +type FenceError struct { + Detail string +} + +func (e *FenceError) Error() string { return "fence violation: " + e.Detail } + +// RedactKey returns a safe display form for a key surfaced in status, +// events, or logs: the configured destination prefix (already public in +// the spec) + "…" + the first 8 hex chars of sha256(key). Key bytes beyond +// the prefix are never surfaced; values never at all. +func RedactKey(prefix string, key []byte) string { + sum := sha256.Sum256(key) + return prefix + "…" + hex.EncodeToString(sum[:])[:8] +} + +// TooLargeError is an oversized request: the server's Txn size or op-count +// limit, or the gRPC client send cap. Permanent; carries the offending key +// (never the value) so operators can find the poison key. +type TooLargeError struct { + // Key is the redacted form (RedactKey) of the first key of the offending + // batch (target-side): the destination prefix plus a hash — raw key + // suffixes and values are deliberately never carried. + Key string + Ops int + Bytes int64 + Cause error +} + +func (e *TooLargeError) Error() string { + return fmt.Sprintf("request too large (%d ops, %d bytes, first key %q): %v", + e.Ops, e.Bytes, e.Key, e.Cause) +} +func (e *TooLargeError) Unwrap() error { return e.Cause } + +// EmptyTargetViolationError reports a non-empty destination prefix under +// InitialSyncRequireEmpty. Permanent. The range identifies exactly what an +// operator must clear (`etcdctl del` over [RangeStart, RangeEnd)); the +// reserved checkpoint key was excluded from the count. +type EmptyTargetViolationError struct { + RangeStart string + RangeEnd string + KeyCount int64 +} + +func (e *EmptyTargetViolationError) Error() string { + return fmt.Sprintf( + "destination prefix not empty: %d pre-existing keys in [%q, %q) and initialSyncMode is RequireEmpty", + e.KeyCount, e.RangeStart, e.RangeEnd) +} + +// PrefixConflictError reports another EtcdMirror link's reserved fence key +// found inside this link's effective destination prefix during a prune pass: +// two links target overlapping destination ranges on the same cluster. +// Deleting the sibling's fence (and its data, as "orphans") would silently +// destroy the other link, so the pass stops loudly instead. Permanent until +// the operator resolves the overlap. +type PrefixConflictError struct { + // Key is the redacted form (RedactKey) of the foreign reserved key. + Key string + // OwnerLinkUID is the link that owns the foreign fence. + OwnerLinkUID string +} + +func (e *PrefixConflictError) Error() string { + return fmt.Sprintf( + "destination prefix conflict: reserved fence key %q under this link's destination prefix belongs to link %q", + e.Key, e.OwnerLinkUID) +} + +// DrainVerificationError reports a post-drain per-side key-count mismatch +// that one repair pass did not resolve. Permanent — cutover must not proceed +// on divergent data. +type DrainVerificationError struct { + SourceKeys int64 + TargetKeys int64 +} + +func (e *DrainVerificationError) Error() string { + return fmt.Sprintf("drain verification failed: source has %d keys, target has %d", + e.SourceKeys, e.TargetKeys) +} + +// UnsupportedVersionError reports an etcd server below the declared >=3.4 +// hard floor. Permanent. +type UnsupportedVersionError struct { + Side string // "source" or "target" + Version string +} + +func (e *UnsupportedVersionError) Error() string { + return fmt.Sprintf("%s etcd version %s is below the supported floor %s", e.Side, e.Version, hardVersionFloor) +} + +// Classify maps any error the engine encounters to its taxonomy class. +// Typed engine errors win; then etcd's typed rpc errors; then gRPC status +// codes; unknown errors default to transient (retrying an unknown error is +// recoverable, silently dropping a permanent one is not). +// +// The engine owns 100% of write-path retry per the class returned here — +// see the retry-ownership contract on [backoff]. Never classify by gRPC +// code alone: ErrNoSpace (quota), ErrTooManyRequests (throttle), and the +// client send cap (permanent) all share codes.ResourceExhausted. +func Classify(err error) Class { + if err == nil { + return ClassTransient + } + + // Engine-typed errors first. + var ( + resyncErr *ResyncError + cpInvalid *CheckpointInvalidError + fenceErr *FenceError + tooLarge *TooLargeError + emptyTarget *EmptyTargetViolationError + unsupportedV *UnsupportedVersionError + drainVerify *DrainVerificationError + configErr *ConfigError + prefixErr *PrefixConflictError + ) + switch { + case errors.As(err, &resyncErr): + return ClassResync + case errors.As(err, &cpInvalid), + errors.As(err, &fenceErr), + errors.As(err, &tooLarge), + errors.As(err, &emptyTarget), + errors.As(err, &unsupportedV), + errors.As(err, &drainVerify), + errors.As(err, &configErr), + errors.As(err, &prefixErr): + return ClassPermanent + } + + // etcd-typed errors: normalize the raw gRPC error to its canonical + // rpctypes singleton where one exists, so both wire and pre-converted + // forms classify identically. + switch rpctypes.Error(err) { + case rpctypes.ErrNoSpace: + return ClassQuota + case rpctypes.ErrTooManyRequests: + return ClassThrottle + case rpctypes.ErrCompacted, rpctypes.ErrFutureRev: + return ClassResync + case rpctypes.ErrTooManyOps, rpctypes.ErrRequestTooLarge: + return ClassPermanent + case rpctypes.ErrPermissionDenied, rpctypes.ErrUserEmpty, rpctypes.ErrAuthFailed: + return ClassPermanent + } + + // Context errors: cancellation/deadline of our own contexts. + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return ClassTransient + } + + // Remaining gRPC status codes. + if s, ok := status.FromError(err); ok { + switch s.Code() { + case codes.ResourceExhausted: + // The client-side send cap surfaces as ResourceExhausted + // "trying to send message larger than max": that is an oversized + // request, NOT throttling — mislabelling it throttling retries a + // poison batch forever. + if strings.Contains(s.Message(), "larger than max") { + return ClassPermanent + } + return ClassThrottle + case codes.InvalidArgument: + return ClassPermanent + case codes.PermissionDenied, codes.Unauthenticated: + return ClassPermanent + case codes.OutOfRange: + return ClassResync + } + } + + return ClassTransient +} diff --git a/pkg/mirroragent/errors_test.go b/pkg/mirroragent/errors_test.go new file mode 100644 index 00000000..62c6d745 --- /dev/null +++ b/pkg/mirroragent/errors_test.go @@ -0,0 +1,135 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" +) + +func TestClassify(t *testing.T) { + cases := []struct { + name string + err error + want Class + }{ + // Engine-typed errors. + {name: "resync compacted", err: &ResyncError{Reason: ResyncReasonCompacted}, want: ClassResync}, + {name: "resync cluster id", err: &ResyncError{Reason: ResyncReasonClusterIDMismatch}, want: ClassResync}, + {name: "wrapped resync", err: fmt.Errorf("cycle: %w", &ResyncError{Reason: ResyncReasonCompacted}), + want: ClassResync}, + {name: "checkpoint invalid fails closed", err: &CheckpointInvalidError{Reason: "garbage"}, + want: ClassPermanent}, + {name: "fence violation", err: &FenceError{Detail: "taken over"}, want: ClassPermanent}, + {name: "too large", err: &TooLargeError{Key: "/dst/…deadbeef"}, want: ClassPermanent}, + {name: "empty target violation", err: &EmptyTargetViolationError{KeyCount: 3}, want: ClassPermanent}, + {name: "unsupported version", err: &UnsupportedVersionError{Side: "source", Version: "3.3.0"}, + want: ClassPermanent}, + {name: "drain verification", err: &DrainVerificationError{SourceKeys: 1, TargetKeys: 2}, + want: ClassPermanent}, + {name: "config error", err: &ConfigError{Detail: "no endpoints"}, want: ClassPermanent}, + {name: "prefix conflict", err: &PrefixConflictError{Key: "/dst/…deadbeef", OwnerLinkUID: "other"}, + want: ClassPermanent}, + + // etcd-typed rpc errors (client-side singletons). + {name: "no space", err: rpctypes.ErrNoSpace, want: ClassQuota}, + {name: "too many requests", err: rpctypes.ErrTooManyRequests, want: ClassThrottle}, + {name: "compacted", err: rpctypes.ErrCompacted, want: ClassResync}, + {name: "future rev", err: rpctypes.ErrFutureRev, want: ClassResync}, + {name: "too many ops", err: rpctypes.ErrTooManyOps, want: ClassPermanent}, + {name: "request too large", err: rpctypes.ErrRequestTooLarge, want: ClassPermanent}, + {name: "permission denied", err: rpctypes.ErrPermissionDenied, want: ClassPermanent}, + + // etcd-typed rpc errors (gRPC wire form). + {name: "no space wire", err: rpctypes.ErrGRPCNoSpace, want: ClassQuota}, + {name: "compacted wire", err: rpctypes.ErrGRPCCompacted, want: ClassResync}, + + // The three-way codes.ResourceExhausted disambiguation: identical + // gRPC code, three different classes — classification must never be + // by code alone. + {name: "resource exhausted quota", + err: status.Error(codes.ResourceExhausted, "etcdserver: mvcc: database space exceeded"), + want: ClassQuota}, + {name: "resource exhausted throttle", + err: status.Error(codes.ResourceExhausted, "etcdserver: too many requests"), + want: ClassThrottle}, + {name: "resource exhausted client send cap", + err: status.Error(codes.ResourceExhausted, + "trying to send message larger than max (3145728 vs. 2097152)"), + want: ClassPermanent}, + {name: "resource exhausted unknown rate flavor", + err: status.Error(codes.ResourceExhausted, "some proxy rate limit"), + want: ClassThrottle}, + + // Context and transport errors. + {name: "deadline exceeded", err: context.DeadlineExceeded, want: ClassTransient}, + {name: "canceled", err: context.Canceled, want: ClassTransient}, + {name: "no leader", err: rpctypes.ErrNoLeader, want: ClassTransient}, + {name: "unavailable", err: status.Error(codes.Unavailable, "connection refused"), want: ClassTransient}, + {name: "invalid argument", err: status.Error(codes.InvalidArgument, "etcdserver: request is too large"), + want: ClassPermanent}, + {name: "unauthenticated", err: status.Error(codes.Unauthenticated, "invalid auth token"), + want: ClassPermanent}, + {name: "out of range", err: status.Error(codes.OutOfRange, "required revision has been compacted"), + want: ClassResync}, + + // Unknowns default to transient: retrying an unknown error is + // recoverable, silently dropping a permanent one is not. + {name: "unknown error", err: errors.New("weather is bad"), want: ClassTransient}, + {name: "nil", err: nil, want: ClassTransient}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, Classify(tc.err)) + }) + } +} + +func TestRedactKey(t *testing.T) { + secret := []byte("/dst/tenants/acme/api-token-primary") + got := RedactKey("/dst/", secret) + + assert.Equal(t, got, RedactKey("/dst/", secret), "redaction must be deterministic") + assert.True(t, strings.HasPrefix(got, "/dst/…"), "the public prefix survives: %q", got) + assert.Len(t, got, len("/dst/…")+8, "prefix + ellipsis + 8 hex chars") + assert.NotContains(t, got, "tenants", "no raw key bytes beyond the prefix may surface") + assert.NotContains(t, got, "acme") + assert.NotEqual(t, RedactKey("/dst/", []byte("/dst/other")), got) +} + +func TestTooLargeErrorNeverLeaksRawKey(t *testing.T) { + // Mirrors the construction sites: Key is always the RedactKey form. + e := &TooLargeError{ + Key: RedactKey("/dst/", []byte("/dst/tenants/acme/api-token-primary")), + Ops: 2, + Bytes: 3 << 20, + Cause: errors.New("etcdserver: request is too large"), + } + msg := e.Error() + assert.NotContains(t, msg, "acme", "error text must not carry raw key bytes") + assert.Contains(t, msg, "/dst/…", "error text must carry the redacted key for operators") + assert.Equal(t, ClassPermanent, Classify(e)) +} diff --git a/pkg/mirroragent/fence.go b/pkg/mirroragent/fence.go new file mode 100644 index 00000000..ee672a7d --- /dev/null +++ b/pkg/mirroragent/fence.go @@ -0,0 +1,150 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "encoding/json" + "fmt" +) + +// FenceRole is the role stamped into the fence key. Applies are only legal +// while the role is Mirror; the drain flow flips it to Primary at cutover so +// a straggler mirror apply fails its mod-revision compare loudly. +type FenceRole string + +const ( + // RoleMirror means the mirror owns the destination prefix and applies + // are in flight. + RoleMirror FenceRole = "Mirror" + // RolePrimary means cutover completed: the destination prefix is now + // authoritative and no mirror may write under it. + RolePrimary FenceRole = "Primary" +) + +// FenceVersion is the current checkpoint wire-format version. Decoding fails +// closed on any other version (distinct from an absent checkpoint). +const FenceVersion = 1 + +// FenceValue is the checkpoint/fence document stored at the reserved key in +// the target etcd. It is written in the same Txn as every applied batch; +// its mod revision is the compare every write path is fenced on. +// +// State encoding — the tuple (Scanning, PrunePending, Role) subsumes an +// explicit phase enum: +// +// Scanning=true genesis scan in flight: Watermark is the +// scan's watch-start revision R0 (the replay +// base), NOT a caught-up-through claim; +// progress lives in ScanCursor/SubRevision. +// Consumers must never read Watermark as +// replication progress while Scanning is true. +// Scanning=false, Role=Mirror steady state: Watermark is the source +// revision fully applied through. +// PrunePending=true a forced resync's mandatory mark-and-sweep +// is owed; survives crashes. +// Role=Primary cutover complete: no mirror may write under +// the destination prefix; stragglers fail +// their mod-revision compare loudly. +type FenceValue struct { + // Version is the wire-format version (FenceVersion). + Version int `json:"v"` + // LinkUID identifies the mirror link that owns this fence. + LinkUID string `json:"linkUID"` + // Epoch is the agent generation that last wrote the checkpoint. + Epoch int64 `json:"epoch"` + // Role is Mirror until cutover flips it to Primary. + Role FenceRole `json:"role"` + // Watermark is the source revision through which the target is caught + // up. While a genesis scan is in flight it is the scan's watch-start + // revision (the base the buffered watch replays over), not a fully + // applied revision — Scanning distinguishes the two. + Watermark int64 `json:"watermark"` + // SubRevision is the ordinal progress marker within a Watermark that is + // not yet revision-complete: the genesis scan stamps its page ordinal + // here. Zero on every revision-complete checkpoint. + SubRevision int64 `json:"subRevision,omitempty"` + // Scanning is true while a genesis scan is in flight; ScanCursor is then + // the last source key whose page has been applied, so a restarted agent + // resumes the scan instead of starting over. + Scanning bool `json:"scanning,omitempty"` + ScanCursor string `json:"scanCursor,omitempty"` + // PrunePending is true from the moment a forced resync claims the fence + // until its mandatory mark-and-sweep prune pass has completed. It makes + // the owed sweep durable: an agent that crashes mid-forced-resync and + // resumes the scan still runs the prune, so deletes from the blind window + // that triggered the resync cannot silently resurrect on the target. + PrunePending bool `json:"prunePending,omitempty"` + // SourceClusterID / TargetClusterID bind the checkpoint to BOTH cluster + // identities. Either changing means an endpoint now points at a + // different cluster than the checkpoint was taken against: the + // checkpoint is invalidated, genesis is forced, and RequireEmpty + // re-arms. String-encoded: cluster IDs use the full uint64 range. + SourceClusterID uint64 `json:"sourceClusterID,string"` + TargetClusterID uint64 `json:"targetClusterID,string"` +} + +// Encode serializes the fence value for storage at the reserved key. The +// wire-format version is stamped unconditionally. +func (f FenceValue) Encode() (string, error) { + f.Version = FenceVersion + if err := f.validate(); err != nil { + return "", err + } + b, err := json.Marshal(f) + if err != nil { + return "", err + } + return string(b), nil +} + +// DecodeFenceValue parses a stored checkpoint. Corrupt content or an unknown +// version returns a *CheckpointInvalidError, which classifies PERMANENT: an +// undecodable fence proves nothing about ownership, epoch ordering, or role +// (it may be a newer agent generation's format, or a corrupted post-cutover +// Primary fence), so no write — least of all a resync's prune — is provably +// safe. The operator must inspect and delete the reserved key to recover. +func DecodeFenceValue(raw []byte) (FenceValue, error) { + var f FenceValue + if err := json.Unmarshal(raw, &f); err != nil { + return FenceValue{}, &CheckpointInvalidError{Reason: fmt.Sprintf("undecodable checkpoint: %v", err)} + } + if f.Version != FenceVersion { + return FenceValue{}, &CheckpointInvalidError{ + Reason: fmt.Sprintf("unknown checkpoint version %d (agent supports %d)", f.Version, FenceVersion), + } + } + if err := f.validate(); err != nil { + return FenceValue{}, &CheckpointInvalidError{Reason: err.Error()} + } + return f, nil +} + +func (f FenceValue) validate() error { + if f.LinkUID == "" { + return fmt.Errorf("checkpoint has empty linkUID") + } + if f.Epoch < 1 { + return fmt.Errorf("checkpoint has invalid epoch %d", f.Epoch) + } + if f.Role != RoleMirror && f.Role != RolePrimary { + return fmt.Errorf("checkpoint has invalid role %q", f.Role) + } + if f.Watermark < 0 || f.SubRevision < 0 { + return fmt.Errorf("checkpoint has negative revision fields") + } + return nil +} diff --git a/pkg/mirroragent/fence_test.go b/pkg/mirroragent/fence_test.go new file mode 100644 index 00000000..be53db29 --- /dev/null +++ b/pkg/mirroragent/fence_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func validFence() FenceValue { + return FenceValue{ + LinkUID: "link-1", + Epoch: 3, + Role: RoleMirror, + Watermark: 42, + SubRevision: 7, + Scanning: true, + ScanCursor: "/src/key-0100", + PrunePending: true, + SourceClusterID: 0xdeadbeefcafef00d, + TargetClusterID: 0x0123456789abcdef, + } +} + +func TestFenceValueRoundTrip(t *testing.T) { + in := validFence() + raw, err := in.Encode() + require.NoError(t, err) + + out, err := DecodeFenceValue([]byte(raw)) + require.NoError(t, err) + in.Version = FenceVersion // Encode stamps it + assert.Equal(t, in, out) + assert.True(t, out.PrunePending, "PrunePending must survive the round trip") + assert.Equal(t, uint64(0xdeadbeefcafef00d), out.SourceClusterID, + "cluster IDs must survive the full uint64 range (string-encoded)") +} + +// TestDecodeFenceValueFailsClosed pins the WIP semantics: corrupt content or +// an unknown wire version is a *CheckpointInvalidError AND classifies +// Permanent — never a resync, never a resume on a guess. +func TestDecodeFenceValueFailsClosed(t *testing.T) { + cases := []struct { + name string + raw string + }{ + {name: "garbage", raw: "not json at all {"}, + {name: "empty", raw: ""}, + {name: "future version", raw: `{"v":99,"linkUID":"l","epoch":1,"role":"Mirror",` + + `"watermark":1,"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "version zero", raw: `{"linkUID":"l","epoch":1,"role":"Mirror",` + + `"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "empty linkUID", raw: `{"v":1,"linkUID":"","epoch":1,"role":"Mirror",` + + `"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "epoch below one", raw: `{"v":1,"linkUID":"l","epoch":0,"role":"Mirror",` + + `"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "unknown role", raw: `{"v":1,"linkUID":"l","epoch":1,"role":"Standby",` + + `"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "negative watermark", raw: `{"v":1,"linkUID":"l","epoch":1,"role":"Mirror",` + + `"watermark":-5,"sourceClusterID":"1","targetClusterID":"2"}`}, + {name: "negative subrevision", raw: `{"v":1,"linkUID":"l","epoch":1,"role":"Mirror",` + + `"subRevision":-1,"sourceClusterID":"1","targetClusterID":"2"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := DecodeFenceValue([]byte(tc.raw)) + var ci *CheckpointInvalidError + require.ErrorAs(t, err, &ci) + assert.Equal(t, ClassPermanent, Classify(err), + "an undecodable checkpoint must fail closed (permanent), not resync") + }) + } +} + +func TestFenceValueEncodeRejectsInvalid(t *testing.T) { + cases := []struct { + name string + mutate func(*FenceValue) + }{ + {name: "empty linkUID", mutate: func(f *FenceValue) { f.LinkUID = "" }}, + {name: "epoch below one", mutate: func(f *FenceValue) { f.Epoch = 0 }}, + {name: "bad role", mutate: func(f *FenceValue) { f.Role = "Replica" }}, + {name: "negative watermark", mutate: func(f *FenceValue) { f.Watermark = -1 }}, + {name: "negative subrevision", mutate: func(f *FenceValue) { f.SubRevision = -2 }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := validFence() + tc.mutate(&f) + _, err := f.Encode() + assert.Error(t, err) + }) + } +} diff --git a/pkg/mirroragent/helpers_integration_test.go b/pkg/mirroragent/helpers_integration_test.go new file mode 100644 index 00000000..919d4841 --- /dev/null +++ b/pkg/mirroragent/helpers_integration_test.go @@ -0,0 +1,305 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent_test + +import ( + "context" + "fmt" + "net" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd-operator/pkg/mirroragent" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" +) + +// startEtcd boots one in-process embedded etcd on unique loopback ports and +// returns a client wired the way the engine requires (NewClientConfig). +func startEtcd(t *testing.T, mutate func(*embed.Config)) *clientv3.Client { + t.Helper() + cfg := embed.NewConfig() + cfg.Dir = t.TempDir() + cfg.Logger = "zap" + cfg.LogLevel = "error" + cu := freeURL(t) + pu := freeURL(t) + cfg.ListenClientUrls = []url.URL{*cu} + cfg.AdvertiseClientUrls = []url.URL{*cu} + cfg.ListenPeerUrls = []url.URL{*pu} + cfg.AdvertisePeerUrls = []url.URL{*pu} + cfg.InitialCluster = cfg.Name + "=" + pu.String() + if mutate != nil { + mutate(cfg) + } + e, err := embed.StartEtcd(cfg) + require.NoError(t, err) + t.Cleanup(e.Close) + select { + case <-e.Server.ReadyNotify(): + case <-time.After(60 * time.Second): + t.Fatal("embedded etcd took too long to start") + } + cli, err := clientv3.New(mirroragent.NewClientConfig([]string{cu.String()}, nil, 5*time.Second)) + require.NoError(t, err) + t.Cleanup(func() { _ = cli.Close() }) + return cli +} + +func freeURL(t *testing.T) *url.URL { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := l.Addr().String() + require.NoError(t, l.Close()) + u, err := url.Parse("http://" + addr) + require.NoError(t, err) + return u +} + +// baseCfg returns an engine config with intervals shrunk for test runtime. +func baseCfg(srcPrefix, dstPrefix string) mirroragent.Config { + return mirroragent.Config{ + LinkUID: "test-link", + Epoch: 1, + SourcePrefix: srcPrefix, + TargetPrefix: dstPrefix, + RequestTimeout: 5 * time.Second, + BackoffInitialDelay: 50 * time.Millisecond, + BackoffMaxDelay: 500 * time.Millisecond, + ProgressInterval: 200 * time.Millisecond, + QuotaProbeInterval: 250 * time.Millisecond, + } +} + +func checkpointKey(cfg mirroragent.Config) string { + return cfg.TargetPrefix + cfg.DestPrefix + mirroragent.DefaultCheckpointKeySuffix +} + +type agentRun struct { + agent *mirroragent.Agent + cancel context.CancelFunc + done chan error +} + +// startAgent builds and runs an Agent in a goroutine, cleaned up with the +// test. +func startAgent(t *testing.T, cfg mirroragent.Config, src, dst mirroragent.Client) *agentRun { + t.Helper() + agent, err := mirroragent.New(cfg, src, dst) + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- agent.Run(ctx) }() + r := &agentRun{agent: agent, cancel: cancel, done: done} + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(15 * time.Second): + t.Error("agent did not stop within 15s of cancel") + } + }) + return r +} + +// stop cancels the agent and waits for Run to return. +func (r *agentRun) stop(t *testing.T) error { + t.Helper() + r.cancel() + return r.waitErr(t, 15*time.Second) +} + +// waitErr waits for Run to return and hands back its error. +func (r *agentRun) waitErr(t *testing.T, timeout time.Duration) error { + t.Helper() + select { + case err := <-r.done: + r.done <- err // allow repeated reads / cleanup + return err + case <-time.After(timeout): + t.Fatalf("agent Run did not return within %v", timeout) + return nil + } +} + +// waitSnap polls Snapshot until cond holds. +func waitSnap( + t *testing.T, a *mirroragent.Agent, timeout time.Duration, + what string, cond func(mirroragent.Snapshot) bool, +) mirroragent.Snapshot { + t.Helper() + deadline := time.Now().Add(timeout) + var s mirroragent.Snapshot + for time.Now().Before(deadline) { + s = a.Snapshot() + if cond(s) { + return s + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("waiting for %s: condition not met within %v; last snapshot: %+v", what, timeout, s) + return s +} + +// putN writes n sequential keys under prefix and returns the resulting +// source data as the expected target map under dstPrefix. +func putN(t *testing.T, cli *clientv3.Client, srcPrefix, dstPrefix string, n int) map[string]string { + t.Helper() + want := make(map[string]string, n) + for i := range n { + k := fmt.Sprintf("key-%04d", i) + v := fmt.Sprintf("val-%04d", i) + _, err := cli.Put(t.Context(), srcPrefix+k, v) + require.NoError(t, err) + want[dstPrefix+k] = v + } + return want +} + +// targetData reads every key under the destination prefix except the +// reserved checkpoint key. +func targetData(t *testing.T, cli *clientv3.Client, cfg mirroragent.Config) map[string]string { + t.Helper() + resp, err := cli.Get(t.Context(), cfg.TargetPrefix+cfg.DestPrefix, clientv3.WithPrefix()) + require.NoError(t, err) + out := make(map[string]string, len(resp.Kvs)) + for _, kv := range resp.Kvs { + if string(kv.Key) == checkpointKey(cfg) { + continue + } + out[string(kv.Key)] = string(kv.Value) + } + return out +} + +// waitTargetData polls until the destination data (reserved key excluded) +// equals want exactly. +func waitTargetData( + t *testing.T, cli *clientv3.Client, cfg mirroragent.Config, + timeout time.Duration, want map[string]string, +) { + t.Helper() + deadline := time.Now().Add(timeout) + var got map[string]string + for time.Now().Before(deadline) { + got = targetData(t, cli, cfg) + if mapsEqual(got, want) { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatalf("target data never converged: got %d keys, want %d keys\ngot: %v\nwant: %v", + len(got), len(want), summarize(got), summarize(want)) +} + +func mapsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func summarize(m map[string]string) string { + if len(m) <= 12 { + return fmt.Sprintf("%v", m) + } + keys := make([]string, 0, 12) + for k := range m { + keys = append(keys, k) + if len(keys) == 12 { + break + } + } + return fmt.Sprintf("{%s, ...}", strings.Join(keys, ", ")) +} + +// readFence reads and decodes the reserved checkpoint key. +func readFence( + t *testing.T, cli *clientv3.Client, cfg mirroragent.Config, +) (mirroragent.FenceValue, int64) { + t.Helper() + resp, err := cli.Get(t.Context(), checkpointKey(cfg)) + require.NoError(t, err) + require.Len(t, resp.Kvs, 1, "reserved checkpoint key missing") + f, err := mirroragent.DecodeFenceValue(resp.Kvs[0].Value) + require.NoError(t, err) + return f, resp.Kvs[0].ModRevision +} + +// sourceRevision returns the source cluster's current revision. +func sourceRevision(t *testing.T, cli *clientv3.Client, prefix string) int64 { + t.Helper() + resp, err := cli.Get(t.Context(), prefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) + require.NoError(t, err) + return resp.Header.Revision +} + +// countingClient wraps a Client and counts Get/Txn calls, for asserting that +// resume paths do not rescan and quota parking does not hot-loop. +type countingClient struct { + mirroragent.Client + gets atomic.Int64 + txns atomic.Int64 +} + +func (c *countingClient) Get( + ctx context.Context, key string, opts ...clientv3.OpOption, +) (*clientv3.GetResponse, error) { + c.gets.Add(1) + return c.Client.Get(ctx, key, opts...) +} + +func (c *countingClient) Txn(ctx context.Context) clientv3.Txn { + c.txns.Add(1) + return c.Client.Txn(ctx) +} + +// watchRevRecordingClient records the OpOption-resolved start revision of +// every Watch call, for pinning resume revisions. +type watchRevRecordingClient struct { + mirroragent.Client + mu sync.Mutex + revs []int64 +} + +func (c *watchRevRecordingClient) Watch( + ctx context.Context, key string, opts ...clientv3.OpOption, +) clientv3.WatchChan { + op := clientv3.OpGet(key, opts...) + c.mu.Lock() + c.revs = append(c.revs, op.Rev()) + c.mu.Unlock() + return c.Client.Watch(ctx, key, opts...) +} + +func (c *watchRevRecordingClient) recorded() []int64 { + c.mu.Lock() + defer c.mu.Unlock() + return append([]int64(nil), c.revs...) +} diff --git a/pkg/mirroragent/integration_delta_test.go b/pkg/mirroragent/integration_delta_test.go new file mode 100644 index 00000000..a7eb10b0 --- /dev/null +++ b/pkg/mirroragent/integration_delta_test.go @@ -0,0 +1,1289 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Integration tests for the Design-3 gap-closure delta: replay-buffer +// overflow restarts, the durable PrunePending flag, cluster-ID re-arm, +// the resync-loop latch, exclude-range elision, oversize permanence, +// fail-closed checkpoints, the version floor, progress-notify liveness, +// and duplicate-replay idempotence. +package mirroragent_test + +import ( + "context" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "go.etcd.io/etcd-operator/pkg/mirroragent" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" +) + +// bigValueClient returns a client whose send cap admits values larger than +// the 2MiB clientv3 default, for seeding oversize test data. +func bigValueClient(t *testing.T, target *clientv3.Client) *clientv3.Client { + t.Helper() + cfg := mirroragent.NewClientConfig(target.Endpoints(), nil, 5*time.Second) + cfg.MaxCallSendMsgSize = 16 << 20 + cli, err := clientv3.New(cfg) + require.NoError(t, err) + t.Cleanup(func() { _ = cli.Close() }) + return cli +} + +// TestWatchBufferOverflowRestart covers the bounded replay buffer: with a +// tiny WatchBufferBytes and sustained mid-scan churn, genesis attempts are +// aborted and restarted from a fresh R0 (never unbounded growth), the cause +// is surfaced in the snapshot, repeated restarts trip the resync-loop +// detector, and once churn stops the mirror still converges byte-exact. +func TestWatchBufferOverflowRestart(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.WatchBufferBytes = 512 // a handful of events + cfg.MaxOpsPerSecond = 150 // keep each scan attempt slow enough to race + cfg.PageKeyLimit = 25 + cfg.ResyncLoopThreshold = 2 + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 150) + + r := startAgent(t, cfg, src, dst) + + // Churn during the scan: each write lands in the replay buffer. Values + // are sized so a SINGLE churn event overflows the 512-byte buffer — + // overflow must not depend on sustaining a wall-clock churn rate on a + // loaded runner. + churnValue := strings.Repeat("c", 1024) + churnCtx, stopChurn := context.WithCancel(t.Context()) + defer stopChurn() + churnDone := make(chan struct{}) + go func() { + defer close(churnDone) + for i := 0; churnCtx.Err() == nil; i++ { + k := fmt.Sprintf("churn-%04d", i) + if _, err := src.Put(churnCtx, cfg.SourcePrefix+k, churnValue); err != nil { + return + } + time.Sleep(5 * time.Millisecond) + } + }() + + snap := waitSnap(t, r.agent, 30*time.Second, "buffer overflow restarts", + func(s mirroragent.Snapshot) bool { return s.ScanRestartCount >= 2 }) + assert.Equal(t, mirroragent.ScanRestartWatchBufferOverflow, snap.LastScanRestartCause) + assert.EqualValues(t, 0, snap.ForcedResyncCount, + "a buffer-bound restart is not a forced resync — the checkpoint was never invalidated") + waitSnap(t, r.agent, 30*time.Second, "resync-loop detector tripped by repeated overflows", + func(s mirroragent.Snapshot) bool { return s.ResyncLoopDetected }) + + stopChurn() + <-churnDone + + // Byte-exact convergence against source truth, re-read each poll: a + // churn Put cancelled mid-flight can still land server-side after the + // stop signal, so the authority is whatever the source holds NOW. + var want map[string]string + deadline := time.Now().Add(90 * time.Second) + for { + sresp, err := src.Get(t.Context(), cfg.SourcePrefix, clientv3.WithPrefix()) + require.NoError(t, err) + want = make(map[string]string, len(sresp.Kvs)) + for _, kv := range sresp.Kvs { + want[cfg.TargetPrefix+strings.TrimPrefix(string(kv.Key), cfg.SourcePrefix)] = string(kv.Value) + } + if got := targetData(t, dst, cfg); mapsEqual(got, want) { + break + } + require.True(t, time.Now().Before(deadline), + "target never converged to source truth after churn stopped") + time.Sleep(100 * time.Millisecond) + } + + // Post-churn writes reach steady state and clear the detector. A write + // can still be swallowed into a final scan/replay (which must NOT clear + // the latch), so keep writing until one lands as a live tail apply. + deadline = time.Now().Add(30 * time.Second) + for i := 0; r.agent.Snapshot().ResyncLoopDetected; i++ { + require.True(t, time.Now().Before(deadline), + "detector never cleared at steady state despite live writes") + _, err := src.Put(t.Context(), fmt.Sprintf("/src/settled-%d", i), "yes") + require.NoError(t, err) + time.Sleep(150 * time.Millisecond) + } +} + +// TestPrunePendingCrashResume is THE new-mechanism test: an agent that is +// killed mid-forced-resync — after the fence records PrunePending=true but +// before the mark-and-sweep ran — must still run the prune after restart, +// so a delete from the blind window cannot resurrect on the target. +func TestPrunePendingCrashResume(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 40) + r1 := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r1.agent, 10*time.Second, "run 1 Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + require.ErrorIs(t, r1.stop(t), context.Canceled) + + // Blind window while the agent is down: delete one mirrored key, add + // two, then compact past the watermark so the re-watch is doomed. + ctx := t.Context() + _, err := src.Delete(ctx, "/src/key-0005") + require.NoError(t, err) + delete(want, "/dst/key-0005") + for i := range 2 { + k := fmt.Sprintf("late-%d", i) + _, perr := src.Put(ctx, cfg.SourcePrefix+k, "late") + require.NoError(t, perr) + want[cfg.TargetPrefix+k] = "late" + } + head, err := src.Get(ctx, "/src/", clientv3.WithPrefix(), clientv3.WithCountOnly()) + require.NoError(t, err) + _, err = src.Compact(ctx, head.Header.Revision, clientv3.WithCompactPhysical()) + require.NoError(t, err) + + // Run 2 hits ErrCompacted and starts the forced resync (slowed down and + // with small Txns so the fence records a MID-SCAN cursor). It is killed + // only once the fence shows PrunePending AND a non-empty scan cursor, so + // run 3 must exercise the cursor-resume branch, not the fresh-R0 path. + cfg2 := cfg + cfg2.Epoch = 2 + cfg2.MaxOpsPerSecond = 25 + cfg2.MaxTxnOps = 6 // 5 data slots: the cursor advances every 5 keys + r2 := startAgent(t, cfg2, src, dst) + deadline := time.Now().Add(20 * time.Second) + for { + require.True(t, time.Now().Before(deadline), + "fence never recorded PrunePending with a mid-scan cursor") + resp, gerr := dst.Get(ctx, checkpointKey(cfg)) + require.NoError(t, gerr) + if len(resp.Kvs) == 1 { + f, derr := mirroragent.DecodeFenceValue(resp.Kvs[0].Value) + require.NoError(t, derr) + if f.PrunePending && f.Scanning && f.ScanCursor != "" { + break + } + } + time.Sleep(5 * time.Millisecond) + } + r2.cancel() + require.ErrorIs(t, r2.waitErr(t, 15*time.Second), context.Canceled) + + // The kill must have landed mid-scan for the resume branch to be under + // test at all; the paced scan (40 keys at 25 ops/s) makes this window + // seconds wide. + killed, _ := readFence(t, dst, cfg) + require.True(t, killed.Scanning && killed.ScanCursor != "", + "test premise: run 2 must die MID-SCAN (fence: %+v) — widen the pacing if this trips", killed) + + // The blind-window delete has resurrection potential right now. + resp, err := dst.Get(ctx, "/dst/key-0005") + require.NoError(t, err) + require.Len(t, resp.Kvs, 1, + "test setup: the deleted key must still be on the target before the prune") + + // Run 3 resumes purely from the durable fence state: the scan continues + // from the recorded cursor (no re-count, no re-applied pre-cursor keys) + // and the owed sweep still runs without re-detecting the compaction. + cfg3 := cfg + cfg3.Epoch = 3 + rsrc := &rangeRecordingClient{Client: src} + r3 := startAgent(t, cfg3, rsrc, dst) + waitTargetData(t, dst, cfg, 30*time.Second, want) + snap := waitSnap(t, r3.agent, 20*time.Second, "run 3 Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + assert.EqualValues(t, 0, snap.ForcedResyncCount, + "run 3 resumed an owed prune; it did not have to re-detect the compaction") + f, _ := readFence(t, dst, cfg) + assert.False(t, f.PrunePending, "the flag clears once the sweep completed") + assert.False(t, f.Scanning) + + // Resume-branch pin: run 3's FIRST source read must be the scan page + // starting just after the recorded cursor. The fresh-R0 path would open + // with a CountOnly read at the range start instead — and a broken resume + // (panic/skip) could only be reached through that first read. + windows := rsrc.recorded() + require.NotEmpty(t, windows) + assert.Equal(t, killed.ScanCursor+"\x00", windows[0][0], + "run 3's first source read must resume at nextKey(fence.ScanCursor), not rescan from the start") +} + +// TestClusterIDMismatchRearm covers the dual-identity binding: a checkpoint +// bound to a different source OR target cluster forces genesis and RE-ARMS +// RequireEmpty (unlike an ordinary forced resync, which skips it because the +// fence proves ownership). +func TestClusterIDMismatchRearm(t *testing.T) { + t.Run("FreshTargetRearms", func(t *testing.T) { + src := startEtcd(t, nil) + dstA := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 5) + r1 := startAgent(t, cfg, src, dstA) + waitTargetData(t, dstA, cfg, 20*time.Second, want) + require.ErrorIs(t, r1.stop(t), context.Canceled) + + // A rebuilt target carrying the old fence (e.g. restored from a + // snapshot of another cluster) plus pre-existing data. + dstB := startEtcd(t, nil) + ctx := t.Context() + fresp, err := dstA.Get(ctx, checkpointKey(cfg)) + require.NoError(t, err) + require.Len(t, fresp.Kvs, 1) + staleFence := string(fresp.Kvs[0].Value) + _, err = dstB.Put(ctx, checkpointKey(cfg), staleFence) + require.NoError(t, err) + _, err = dstB.Put(ctx, "/dst/preexisting", "dirty") + require.NoError(t, err) + + cfg2 := cfg + cfg2.Epoch = 2 + r2 := startAgent(t, cfg2, src, dstB) + runErr := r2.waitErr(t, 20*time.Second) + var ev *mirroragent.EmptyTargetViolationError + require.ErrorAs(t, runErr, &ev, + "the re-armed RequireEmpty must trip on the non-empty fresh target, got: %v", runErr) + snap := r2.agent.Snapshot() + assert.Equal(t, mirroragent.ResyncReasonClusterIDMismatch, snap.LastResyncReason) + assert.EqualValues(t, 1, snap.ForcedResyncCount) + + // Failing the gate must write nothing — the stale fence is intact. + after, err := dstB.Get(ctx, checkpointKey(cfg)) + require.NoError(t, err) + require.Len(t, after.Kvs, 1) + assert.Equal(t, staleFence, string(after.Kvs[0].Value)) + }) + + t.Run("FreshSourceRearms", func(t *testing.T) { + srcA := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, srcA, cfg.SourcePrefix, cfg.TargetPrefix, 5) + r1 := startAgent(t, cfg, srcA, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + require.ErrorIs(t, r1.stop(t), context.Canceled) + + // Repointing at a different source cluster: the mirrored data on the + // target is no longer provably "this link's own" relative to the new + // source, so RequireEmpty re-arms and trips on it. + srcB := startEtcd(t, nil) + _, err := srcB.Put(t.Context(), "/src/other", "different-world") + require.NoError(t, err) + + cfg2 := cfg + cfg2.Epoch = 2 + r2 := startAgent(t, cfg2, srcB, dst) + runErr := r2.waitErr(t, 20*time.Second) + var ev *mirroragent.EmptyTargetViolationError + require.ErrorAs(t, runErr, &ev, "got: %v", runErr) + assert.Equal(t, mirroragent.ResyncReasonClusterIDMismatch, r2.agent.Snapshot().LastResyncReason) + }) + + t.Run("FreshSourceOverwriteAndPruneConverges", func(t *testing.T) { + srcA := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwrite + + want := putN(t, srcA, cfg.SourcePrefix, cfg.TargetPrefix, 5) + r1 := startAgent(t, cfg, srcA, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + require.ErrorIs(t, r1.stop(t), context.Canceled) + + srcB := startEtcd(t, nil) + _, err := srcB.Put(t.Context(), "/src/fresh", "new-world") + require.NoError(t, err) + + cfg2 := cfg + cfg2.Epoch = 2 + r2 := startAgent(t, cfg2, srcB, dst) + // The mismatch forces genesis with a mandatory mark-and-sweep: srcA's + // five keys are orphans against srcB and must be pruned even though + // the mode is plain Overwrite. + waitTargetData(t, dst, cfg, 30*time.Second, map[string]string{"/dst/fresh": "new-world"}) + snap := r2.agent.Snapshot() + assert.Equal(t, mirroragent.ResyncReasonClusterIDMismatch, snap.LastResyncReason) + f, _ := readFence(t, dst, cfg) + assert.False(t, f.PrunePending) + assert.Equal(t, snap.SourceClusterID, f.SourceClusterID, + "the checkpoint must re-bind to the new source cluster identity") + }) +} + +// breakableWatchClient wraps a source client so the test can (a) kill the +// live watch stream and (b) serve injected already-compacted watch responses +// — the deterministic stand-in for "retention outran the watch". +type breakableWatchClient struct { + mirroragent.Client + inject atomic.Bool + + mu sync.Mutex + cancels []context.CancelFunc +} + +func (c *breakableWatchClient) Watch( + ctx context.Context, key string, opts ...clientv3.OpOption, +) clientv3.WatchChan { + if c.inject.Load() { + ch := make(chan clientv3.WatchResponse, 1) + ch <- clientv3.WatchResponse{Canceled: true, CompactRevision: 3} + close(ch) + return ch + } + wctx, cancel := context.WithCancel(ctx) + c.mu.Lock() + c.cancels = append(c.cancels, cancel) + c.mu.Unlock() + return c.Client.Watch(wctx, key, opts...) +} + +func (c *breakableWatchClient) breakWatches() { + c.mu.Lock() + defer c.mu.Unlock() + for _, cancel := range c.cancels { + cancel() + } + c.cancels = nil +} + +// TestResyncLoopLatch covers the livelock detector: consecutive compaction +// failures without an intervening steady state latch ResyncLoopDetected, the +// latch does not self-clear while the loop continues, and it clears only +// when steady state is finally reached. +func TestResyncLoopLatch(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ResyncLoopThreshold = 2 + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 8) + bsrc := &breakableWatchClient{Client: src} + r := startAgent(t, cfg, bsrc, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 10*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + // Kill the live watch; every re-watch now lands below the "compact + // revision" — the retention < scan-time livelock signature. + bsrc.inject.Store(true) + bsrc.breakWatches() + + snap := waitSnap(t, r.agent, 20*time.Second, "first forced resync", + func(s mirroragent.Snapshot) bool { return s.ForcedResyncCount >= 1 }) + assert.Equal(t, mirroragent.ResyncReasonCompacted, snap.LastResyncReason) + waitSnap(t, r.agent, 20*time.Second, "livelock latch", + func(s mirroragent.Snapshot) bool { return s.ResyncLoopDetected }) + + // The latch must hold while the loop continues. + time.Sleep(400 * time.Millisecond) + assert.True(t, r.agent.Snapshot().ResyncLoopDetected, "the latch must not self-clear mid-loop") + + // Heal the source; the next attempt converges. + bsrc.inject.Store(false) + _, err := src.Put(t.Context(), "/src/healed", "ok") + require.NoError(t, err) + want["/dst/healed"] = "ok" + waitTargetData(t, dst, cfg, 30*time.Second, want) + + // Steady state = a successfully applied TAIL response; only that clears + // the latch (scan convergence alone does not). + _, err = src.Put(t.Context(), "/src/steady", "ok") + require.NoError(t, err) + want["/dst/steady"] = "ok" + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 20*time.Second, "latch clears at steady state", + func(s mirroragent.Snapshot) bool { return !s.ResyncLoopDetected && !s.Compacted }) +} + +// rangeRecordingClient records every Get's [start, end) window, to prove +// exclusion by range decomposition rather than client-side filtering. +type rangeRecordingClient struct { + mirroragent.Client + mu sync.Mutex + windows [][2]string +} + +func (c *rangeRecordingClient) Get( + ctx context.Context, key string, opts ...clientv3.OpOption, +) (*clientv3.GetResponse, error) { + op := clientv3.OpGet(key, opts...) + c.mu.Lock() + c.windows = append(c.windows, [2]string{string(op.KeyBytes()), string(op.RangeBytes())}) + c.mu.Unlock() + return c.Client.Get(ctx, key, opts...) +} + +func (c *rangeRecordingClient) recorded() [][2]string { + c.mu.Lock() + defer c.mu.Unlock() + return append([][2]string(nil), c.windows...) +} + +// TestExcludeRangeElision covers exclude handling at the RPC level: no +// source Get window may even overlap an excluded range — excluded data is +// never transferred, not fetched-then-dropped. The mode is deliberately +// OverwriteAndPrune so the assertion also covers the reconcile/prune pass +// (the code path every forced resync and drain repair runs): operators +// exclude e.g. /secrets/ precisely so those values never leave the source +// network, resyncs included. +func TestExcludeRangeElision(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwriteAndPrune + cfg.ExcludePrefixes = []string{"/src/skip/"} + cfg.PageKeyLimit = 3 // several pages, several windows + + ctx := t.Context() + want := map[string]string{} + for i := range 10 { + a := fmt.Sprintf("/src/aa-%02d", i) + z := fmt.Sprintf("/src/zz-%02d", i) + s := fmt.Sprintf("/src/skip/%02d", i) + for _, kv := range [][2]string{{a, "a"}, {z, "z"}, {s, "never"}} { + _, err := src.Put(ctx, kv[0], kv[1]) + require.NoError(t, err) + } + want["/dst/"+strings.TrimPrefix(a, "/src/")] = "a" + want["/dst/"+strings.TrimPrefix(z, "/src/")] = "z" + } + + rsrc := &rangeRecordingClient{Client: src} + r := startAgent(t, cfg, rsrc, dst) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, want) + assert.EqualValues(t, 20, snap.InitialSyncTotalKeyCount, + "excluded keys must not appear in the InitialSync denominator") + + // A live write under the excluded prefix must not arrive either (the + // watch filters client-side; only range reads decompose). + _, err := src.Put(ctx, "/src/skip/live", "never") + require.NoError(t, err) + _, err = src.Put(ctx, "/src/aa-live", "a") + require.NoError(t, err) + want["/dst/aa-live"] = "a" + waitTargetData(t, dst, cfg, 10*time.Second, want) + + exclStart, exclEnd := "/src/skip/", "/src/skip0" + for _, w := range rsrc.recorded() { + start, end := w[0], w[1] + if end == "" { + end = start + "\x00" // point Get + } + overlaps := start < exclEnd && (end == "\x00" || end > exclStart) + assert.False(t, overlaps, + "source Get window [%q, %q) overlaps the excluded range — exclusion must be server-side elision", + w[0], w[1]) + } +} + +// TestOversizedPermanentBands covers both oversize failure bands: the +// server's request-size reject and the gRPC client send cap. Both classify +// Permanent with distinct causes and a redacted key — never throttling, +// never a retry loop. +func TestOversizedPermanentBands(t *testing.T) { + //nolint:dupl // the two bands intentionally mirror each other with different servers/causes + t.Run("ServerRejectBand", func(t *testing.T) { + src := startEtcd(t, func(c *embed.Config) { c.MaxRequestBytes = 8 << 20 }) + dst := startEtcd(t, nil) // default ~1.5MiB request ceiling + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwrite + + seed := bigValueClient(t, src) + _, err := seed.Put(t.Context(), "/src/poison-key-server", strings.Repeat("v", 1600*1024)) + require.NoError(t, err) + + countingDst := &countingClient{Client: dst} + r := startAgent(t, cfg, src, countingDst) + runErr := r.waitErr(t, 30*time.Second) + var tle *mirroragent.TooLargeError + require.ErrorAs(t, runErr, &tle, "got: %v", runErr) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Contains(t, runErr.Error(), "request is too large", + "the server reject band must surface its distinct cause") + assert.Contains(t, tle.Key, "/dst/…", "the offending key must be surfaced redacted") + assert.NotContains(t, tle.Key, "poison", "raw key bytes must never surface") + + // Permanent means no retry loop: fence claim + the poison attempt + // (single-revision set — no shrink) + slack, counted over the WHOLE + // run (Run has returned, so this bounds every commit ever made). + assert.LessOrEqual(t, countingDst.txns.Load(), int64(4), + "a permanent oversize must never be retried") + assert.Equal(t, mirroragent.PhaseFailed, r.agent.Snapshot().Phase) + }) + + //nolint:dupl // the two bands intentionally mirror each other with different servers/causes + t.Run("ClientSendCapBand", func(t *testing.T) { + src := startEtcd(t, func(c *embed.Config) { c.MaxRequestBytes = 8 << 20 }) + // The target server would accept it; the CLIENT's 2MiB send cap is + // the limit under test. + dst := startEtcd(t, func(c *embed.Config) { c.MaxRequestBytes = 8 << 20 }) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwrite + + seed := bigValueClient(t, src) + _, err := seed.Put(t.Context(), "/src/poison-key-client", strings.Repeat("v", 3<<20)) + require.NoError(t, err) + + countingDst := &countingClient{Client: dst} + r := startAgent(t, cfg, src, countingDst) + runErr := r.waitErr(t, 30*time.Second) + var tle *mirroragent.TooLargeError + require.ErrorAs(t, runErr, &tle, "got: %v", runErr) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Contains(t, runErr.Error(), "larger than max", + "the client send cap band must surface its distinct cause") + assert.NotContains(t, tle.Key, "poison") + assert.LessOrEqual(t, countingDst.txns.Load(), int64(4), + "a permanent client-cap oversize must never be retried") + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.ClassPermanent, snap.LastErrorClass, + "the send cap shares ResourceExhausted with throttling and must NOT classify Throttle") + assert.False(t, snap.Throttled) + }) +} + +// TestCorruptCheckpointFailsClosed covers the fail-closed contract: garbage +// or an unknown-version document at the reserved key stops the agent +// permanently — no genesis, no resync, zero writes. +func TestCorruptCheckpointFailsClosed(t *testing.T) { + cases := []struct { + name string + raw string + }{ + {name: "Garbage", raw: "\x00\x01 not a checkpoint"}, + {name: "FutureVersion", raw: `{"v":99,"linkUID":"test-link","epoch":1,"role":"Mirror",` + + `"watermark":10,"sourceClusterID":"1","targetClusterID":"2"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + + ctx := t.Context() + _, err := dst.Put(ctx, checkpointKey(cfg), tc.raw) + require.NoError(t, err) + preRev, err := dst.Get(ctx, "\x00", clientv3.WithFromKey(), clientv3.WithCountOnly()) + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + runErr := r.waitErr(t, 20*time.Second) + var ci *mirroragent.CheckpointInvalidError + require.ErrorAs(t, runErr, &ci, "got: %v", runErr) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.PhaseFailed, snap.Phase) + assert.EqualValues(t, 0, snap.ForcedResyncCount, "fail closed means NO resync") + + // Zero writes of any kind: the target revision did not move and + // the reserved key still holds the exact original bytes. + post, err := dst.Get(ctx, checkpointKey(cfg)) + require.NoError(t, err) + require.Len(t, post.Kvs, 1) + assert.Equal(t, tc.raw, string(post.Kvs[0].Value)) + assert.Equal(t, preRev.Header.Revision, post.Header.Revision, + "the agent must not have written anything to the target") + }) + } +} + +// versionStubClient overrides the maintenance Status version — the seam for +// the version-floor probe. +type versionStubClient struct { + mirroragent.Client + version string +} + +func (c *versionStubClient) Status( + ctx context.Context, endpoint string, +) (*clientv3.StatusResponse, error) { + resp, err := c.Client.Status(ctx, endpoint) + if err != nil { + return nil, err + } + resp.Version = c.version + return resp, nil +} + +// TestVersionFloor covers the declared >=3.4 hard floor: a source below it +// fails permanently at connect, before any scan read or target write. +func TestVersionFloor(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + + countingSrc := &countingClient{Client: src} + countingDst := &countingClient{Client: dst} + stubSrc := &versionStubClient{Client: countingSrc, version: "3.3.0"} + + r := startAgent(t, cfg, stubSrc, countingDst) + runErr := r.waitErr(t, 20*time.Second) + var uv *mirroragent.UnsupportedVersionError + require.ErrorAs(t, runErr, &uv, "got: %v", runErr) + assert.Equal(t, "source", uv.Side) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Equal(t, mirroragent.PhaseFailed, r.agent.Snapshot().Phase) + + assert.EqualValues(t, 0, countingSrc.gets.Load(), "no scan read may precede the version gate") + assert.EqualValues(t, 0, countingDst.txns.Load(), "no target write may precede the version gate") +} + +// TestProgressNotifyAdvancesIdleWatermark is the RequestProgress metadata +// regression test: on an idle prefix the watermark must still advance via +// client-driven progress requests. It fails if the RequestProgress context +// metadata ever diverges from the Watch context (watcher gRPC streams are +// keyed by outgoing metadata, and the engine watches WithRequireLeader). +func TestProgressNotifyAdvancesIdleWatermark(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ProgressInterval = 150 * time.Millisecond + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + r := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + + // The mirrored prefix goes idle; only out-of-prefix writes move the + // cluster revision. + ctx := t.Context() + var outRev int64 + for i := range 5 { + resp, err := src.Put(ctx, fmt.Sprintf("/elsewhere/%d", i), "x") + require.NoError(t, err) + outRev = resp.Header.Revision + } + + snap := waitSnap(t, r.agent, 20*time.Second, "watermark advance on an idle prefix", + func(s mirroragent.Snapshot) bool { return s.Watermark >= outRev }) + assert.GreaterOrEqual(t, snap.SourceRevision, outRev) + f, _ := readFence(t, dst, cfg) + assert.GreaterOrEqual(t, f.Watermark, outRev, + "the fenced checkpoint — not just the snapshot — must carry the progress watermark") +} + +// duplicatingWatchClient delivers every data-bearing watch response twice. +type duplicatingWatchClient struct { + mirroragent.Client +} + +func (c *duplicatingWatchClient) Watch( + ctx context.Context, key string, opts ...clientv3.OpOption, +) clientv3.WatchChan { + in := c.Client.Watch(ctx, key, opts...) + out := make(chan clientv3.WatchResponse) + go func() { + defer close(out) + for wr := range in { + out <- wr + if len(wr.Events) > 0 && wr.Err() == nil { + out <- wr + } + } + }() + return out +} + +// TestDuplicateReplayIdempotent covers replay idempotence: duplicate event +// delivery (the reflector's overlap case, forced here for every response) +// re-puts value-identical keys — no drift, no divergence, and the drain +// verification still proves per-side equality. +func TestDuplicateReplayIdempotent(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + dupSrc := &duplicatingWatchClient{Client: src} + r := startAgent(t, cfg, dupSrc, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + ctx := t.Context() + want := map[string]string{} + for i := range 10 { + k := fmt.Sprintf("dup-%02d", i) + _, err := src.Put(ctx, cfg.SourcePrefix+k, "v") + require.NoError(t, err) + want[cfg.TargetPrefix+k] = "v" + } + _, err := src.Delete(ctx, "/src/dup-03") + require.NoError(t, err) + delete(want, "/dst/dup-03") + + waitTargetData(t, dst, cfg, 20*time.Second, want) + snap := r.agent.Snapshot() + assert.EqualValues(t, 0, snap.ForcedResyncCount) + assert.Empty(t, snap.LastError, "duplicate delivery must not surface any error") + + // The strongest idempotence proof: drain verification counts both sides + // and only cuts over on exact equality. + r.agent.RequestDrain() + require.NoError(t, r.waitErr(t, 30*time.Second), "a completed drain returns nil") + final := r.agent.Snapshot() + require.NotNil(t, final.Cutover) + assert.EqualValues(t, 9, final.Cutover.SourceKeyCount) + assert.EqualValues(t, 9, final.Cutover.TargetKeyCount) + assert.EqualValues(t, 9, final.SourceKeyCount, + "the always-on per-side key counts must be populated by the verification pass") + assert.EqualValues(t, 9, final.TargetKeyCount) + assert.True(t, final.CutoverReady) +} + +// ambiguousTxnClient makes the next successful Commit report an ambiguous +// timeout: the Txn commits server-side but the caller sees DeadlineExceeded +// — the classic blackholed-NLB lost-response scenario. +type ambiguousTxnClient struct { + mirroragent.Client + arm atomic.Bool +} + +func (c *ambiguousTxnClient) Txn(ctx context.Context) clientv3.Txn { + return &ambiguousTxn{inner: c.Client.Txn(ctx), c: c} +} + +type ambiguousTxn struct { + inner clientv3.Txn + c *ambiguousTxnClient +} + +func (t *ambiguousTxn) If(cs ...clientv3.Cmp) clientv3.Txn { t.inner = t.inner.If(cs...); return t } +func (t *ambiguousTxn) Then(ops ...clientv3.Op) clientv3.Txn { + t.inner = t.inner.Then(ops...) + return t +} +func (t *ambiguousTxn) Else(ops ...clientv3.Op) clientv3.Txn { + t.inner = t.inner.Else(ops...) + return t +} +func (t *ambiguousTxn) Commit() (*clientv3.TxnResponse, error) { + resp, err := t.inner.Commit() + if err == nil && resp.Succeeded && t.c.arm.CompareAndSwap(true, false) { + return nil, context.DeadlineExceeded + } + return resp, err +} + +// TestAmbiguousCommitAdopted covers the spec's re-read-recompute-retry rule +// for fenced Txns: when a committed Txn's response is lost, the retry's +// compare fails against the agent's OWN write — the engine must recognize +// the stored fence as this attempt's value and adopt it, never misreport a +// permanent fence violation. +func TestAmbiguousCommitAdopted(t *testing.T) { + t.Run("SteadyApply", func(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + amb := &ambiguousTxnClient{Client: dst} + r := startAgent(t, cfg, src, amb) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + amb.arm.Store(true) + putResp, err := src.Put(t.Context(), "/src/ambiguous", "survives") + require.NoError(t, err) + waitTargetData(t, dst, cfg, 20*time.Second, map[string]string{"/dst/ambiguous": "survives"}) + + snap := waitSnap(t, r.agent, 20*time.Second, "recovered to Syncing", + func(s mirroragent.Snapshot) bool { + return s.Phase == mirroragent.PhaseSyncing && s.Watermark >= putResp.Header.Revision + }) + assert.EqualValues(t, 0, snap.ForcedResyncCount) + f, _ := readFence(t, dst, cfg) + assert.Equal(t, putResp.Header.Revision, f.Watermark, + "the adopted commit's checkpoint is the authoritative watermark") + select { + case runErr := <-r.done: + t.Fatalf("Run returned after an ambiguous commit: %v", runErr) + default: + } + }) + + t.Run("DrainRoleFlip", func(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + // Progress notifications on the quiesced source carry hdrRev == + // watermark and are skipped without a Txn, so the armed injection + // deterministically hits the role-flip Txn. (The interval must stay + // short — the ticker is also what wakes consume to check the drain + // gate.) + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 5) + amb := &ambiguousTxnClient{Client: dst} + r := startAgent(t, cfg, src, amb) + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + // The role-flip Txn commits but its response is lost: the retry must + // recognize the stored Primary fence as its own write and complete + // the drain — the pre-fix behavior wedged the cutover as a permanent + // fence violation with the target already flipped. + amb.arm.Store(true) + r.agent.RequestDrain() + require.NoError(t, r.waitErr(t, 30*time.Second), "a completed drain returns nil") + + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.PhaseDrained, snap.Phase) + assert.True(t, snap.CutoverReady) + require.NotNil(t, snap.Cutover) + assert.False(t, snap.Cutover.VerifiedTime.IsZero()) + f, _ := readFence(t, dst, cfg) + assert.Equal(t, mirroragent.RolePrimary, f.Role) + }) +} + +// claimRaceClient injects an OLD-epoch fence write between loadFence's read +// (which saw no key) and this generation's genesis claim Txn — the rolling- +// redeploy race where the outgoing generation lands its last checkpoint in +// the read/claim window. +type claimRaceClient struct { + mirroragent.Client + raw *clientv3.Client + key string + stale string + armed atomic.Bool +} + +func (c *claimRaceClient) Txn(ctx context.Context) clientv3.Txn { + if c.armed.CompareAndSwap(true, false) { + if _, err := c.raw.Put(ctx, c.key, c.stale); err != nil { + panic("claimRaceClient: injecting stale fence: " + err.Error()) + } + } + return c.Client.Txn(ctx) +} + +// TestGenesisClaimRaceTakesOver: a genesis fence claim that loses the race +// against an older generation's write must adopt the raced mod revision and +// retry the takeover — not fail the agent with a permanent FenceError. +func TestGenesisClaimRaceTakesOver(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.Epoch = 2 + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 5) + + stale, err := mirroragent.FenceValue{ + LinkUID: cfg.LinkUID, + Epoch: 1, + Role: mirroragent.RoleMirror, + Watermark: 1, + }.Encode() + require.NoError(t, err) + + race := &claimRaceClient{Client: dst, raw: dst, key: checkpointKey(cfg), stale: stale} + race.armed.Store(true) + r := startAgent(t, cfg, src, race) + + waitTargetData(t, dst, cfg, 20*time.Second, want) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing after claim-race takeover", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + assert.EqualValues(t, 0, snap.ForcedResyncCount) + f, _ := readFence(t, dst, cfg) + assert.Equal(t, int64(2), f.Epoch, "the new generation must have taken the fence over") + select { + case runErr := <-r.done: + t.Fatalf("Run returned after a claim race: %v", runErr) + default: + } +} + +// replayLoopWatchClient drives the exact shape of the resync livelock the +// detector exists for: every genesis watch delivers ONE fabricated in-scope +// event (so the replay buffer is never empty) and then dies; every tail +// re-watch reports already-compacted. Heal() restores real watches. +type replayLoopWatchClient struct { + mirroragent.Client + raw *clientv3.Client + srcPrefix string + healed atomic.Bool + + mu sync.Mutex + calls int +} + +func (c *replayLoopWatchClient) Watch( + ctx context.Context, key string, opts ...clientv3.OpOption, +) clientv3.WatchChan { + if c.healed.Load() { + return c.Client.Watch(ctx, key, opts...) + } + c.mu.Lock() + c.calls++ + n := c.calls + c.mu.Unlock() + ch := make(chan clientv3.WatchResponse, 1) + if n%2 == 0 { + // Tail re-watch: retention outran the watch. + ch <- clientv3.WatchResponse{Canceled: true, CompactRevision: 3} + close(ch) + return ch + } + // Genesis watch: one fabricated in-scope event at an existing revision, + // then channel death. The replay buffer keeps the event and genesis + // applies it — the exact sequence that must NOT reset the livelock + // detector mid-resync. + head, err := c.raw.Get(context.Background(), c.srcPrefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) + if err != nil { + close(ch) + return ch + } + ch <- clientv3.WatchResponse{ + Header: *head.Header, + Events: []*clientv3.Event{{ + Type: clientv3.EventTypePut, + Kv: &mvccpb.KeyValue{ + Key: []byte(c.srcPrefix + "replay-marker"), + Value: []byte("replayed"), + ModRevision: head.Header.Revision, + }, + }}, + } + close(ch) + return ch +} + +// TestResyncLoopLatchWithReplay: the livelock detector must latch even when +// every forced resync's replay buffer applies events — a churning source is +// the CANONICAL livelock trigger (retention < scan+apply time), and replay +// applies run inside the resync being counted, so they must never count as +// steady state. +func TestResyncLoopLatchWithReplay(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ResyncLoopThreshold = 2 + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 8) + loop := &replayLoopWatchClient{Client: src, raw: src, srcPrefix: cfg.SourcePrefix} + r := startAgent(t, cfg, loop, dst) + + snap := waitSnap(t, r.agent, 30*time.Second, "livelock latch despite replay applies", + func(s mirroragent.Snapshot) bool { return s.ResyncLoopDetected }) + assert.GreaterOrEqual(t, snap.ForcedResyncCount, int64(2)) + + // Heal: real watches again. A write applied during a LIVE tail clears + // the latch; writes swallowed by the healing resync's own scan do not, + // so keep writing until the agent has settled into a live tail. (This + // test pins the latch behavior; byte-exactness is covered elsewhere.) + loop.healed.Store(true) + deadline := time.Now().Add(30 * time.Second) + for i := 0; r.agent.Snapshot().ResyncLoopDetected; i++ { + require.True(t, time.Now().Before(deadline), + "latch never cleared after healing despite live writes") + _, err := src.Put(t.Context(), cfg.SourcePrefix+fmt.Sprintf("steady-%d", i), "ok") + require.NoError(t, err) + time.Sleep(150 * time.Millisecond) + } +} + +// TestDrainCompletesWithoutProgressTrust: on a source below the +// progress-notify trust floor (3.4.x < 3.4.25 / 3.5.x < 3.5.8) the drain +// target must be derived from the highest in-scope mod revision, not the +// cluster revision — out-of-prefix writes would otherwise park the drain in +// PhaseSyncing forever with no error (the watermark cannot advance past the +// last in-prefix event without trusted progress notifications). +func TestDrainCompletesWithoutProgressTrust(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 6) + stubSrc := &versionStubClient{Client: src, version: "3.5.7"} + r := startAgent(t, cfg, stubSrc, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + // Out-of-prefix writes push the cluster revision past every in-prefix + // event — the quiesced-prefix drain must still terminate. + ctx := t.Context() + var clusterRev int64 + for i := range 5 { + resp, err := src.Put(ctx, fmt.Sprintf("/elsewhere/%d", i), "x") + require.NoError(t, err) + clusterRev = resp.Header.Revision + } + + r.agent.RequestDrain() + require.NoError(t, r.waitErr(t, 30*time.Second), "the drain must terminate below the trust floor") + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.PhaseDrained, snap.Phase) + assert.True(t, snap.CutoverReady) + require.NotNil(t, snap.Cutover) + assert.Less(t, snap.Cutover.DrainTargetRevision, clusterRev, + "the drain target must be the in-scope high-water mark, not the cluster revision") + f, _ := readFence(t, dst, cfg) + assert.Equal(t, mirroragent.RolePrimary, f.Role) +} + +// TestProgressNotifyNotTrustedBelowFloor: on a 3.5.7 source the watermark +// must NOT advance on an idle prefix (progress notifications are unreliable +// below 3.4.25/3.5.8 and may report revisions ahead of delivered events — +// trusting them checkpoints past undelivered data), while applies still +// advance it. +func TestProgressNotifyNotTrustedBelowFloor(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.ProgressInterval = 100 * time.Millisecond + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + stubSrc := &versionStubClient{Client: src, version: "3.5.7"} + r := startAgent(t, cfg, stubSrc, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + base := waitSnap(t, r.agent, 10*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + ctx := t.Context() + var outRev int64 + for i := range 5 { + resp, err := src.Put(ctx, fmt.Sprintf("/elsewhere/%d", i), "x") + require.NoError(t, err) + outRev = resp.Header.Revision + } + // Several progress intervals: an untrusted notification must not move + // the watermark past the last applied in-prefix revision. + time.Sleep(6 * cfg.ProgressInterval) + mid := r.agent.Snapshot() + assert.Less(t, mid.Watermark, outRev, + "the watermark must not advance from untrusted progress notifications") + assert.Equal(t, base.Watermark, mid.Watermark) + + // An applied in-prefix event still advances it. + putResp, err := src.Put(ctx, "/src/applied", "yes") + require.NoError(t, err) + want["/dst/applied"] = "yes" + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 10*time.Second, "watermark advances on applies", + func(s mirroragent.Snapshot) bool { return s.Watermark >= putResp.Header.Revision }) +} + +// flakyEndpointClient reports two endpoints, the first of which is +// blackholed for the maintenance Status probe. +type flakyEndpointClient struct { + mirroragent.Client + bad string +} + +func (c *flakyEndpointClient) Endpoints() []string { + return append([]string{c.bad}, c.Client.Endpoints()...) +} + +func (c *flakyEndpointClient) Status( + ctx context.Context, endpoint string, +) (*clientv3.StatusResponse, error) { + if endpoint == c.bad { + return nil, status.Error(codes.Unavailable, "blackholed endpoint") + } + return c.Client.Status(ctx, endpoint) +} + +// TestProbeRotatesEndpoints: Status dials the named endpoint directly +// (bypassing the balancer), so the connect probe must rotate through the +// endpoint list — one dead member/NAT mapping must not wedge the agent in +// Connecting while healthy endpoints exist. +func TestProbeRotatesEndpoints(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + flaky := &flakyEndpointClient{Client: src, bad: "http://127.0.0.1:1"} + r := startAgent(t, cfg, flaky, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing despite a blackholed first endpoint", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, want) +} + +// TestPruneRefusesForeignFence: a prune pass that encounters ANOTHER link's +// reserved fence key inside this link's destination prefix (overlapping +// destination prefixes — e.g. a second EtcdMirror at a nested prefix) must +// stop loudly with a PrefixConflictError instead of deleting the sibling's +// fence and data as orphans. +func TestPruneRefusesForeignFence(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwriteAndPrune + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + + // A sibling mirror lives at the nested prefix /dst/sub/ with its own + // fence and data. + foreignFence, err := mirroragent.FenceValue{ + LinkUID: "other-link", + Epoch: 1, + Role: mirroragent.RoleMirror, + Watermark: 7, + }.Encode() + require.NoError(t, err) + ctx := t.Context() + foreignKey := "/dst/sub/" + mirroragent.DefaultCheckpointKeySuffix + _, err = dst.Put(ctx, foreignKey, foreignFence) + require.NoError(t, err) + _, err = dst.Put(ctx, "/dst/sub/data", "sibling-owned") + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + runErr := r.waitErr(t, 30*time.Second) + var pc *mirroragent.PrefixConflictError + require.ErrorAs(t, runErr, &pc, "got: %v", runErr) + assert.Equal(t, "other-link", pc.OwnerLinkUID) + assert.NotContains(t, pc.Key, "sub", "the foreign key must be surfaced redacted") + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Equal(t, mirroragent.PhaseFailed, r.agent.Snapshot().Phase) + + // Nothing of the sibling's was destroyed. + got, err := dst.Get(ctx, foreignKey) + require.NoError(t, err) + require.Len(t, got.Kvs, 1, "the sibling's fence key must survive") + assert.Equal(t, foreignFence, string(got.Kvs[0].Value)) + data, err := dst.Get(ctx, "/dst/sub/data") + require.NoError(t, err) + require.Len(t, data.Kvs, 1, "the sibling's data must survive") +} + +// TestShrinkOnTargetTxnLimit: the target is a foreign cluster whose +// --max-txn-ops the operator cannot inspect. A multi-revision flush set the +// target rejects must be re-committed at revision granularity (one shrink +// attempt) instead of failing the agent permanently — only an irreducible +// single revision is a true permanent oversize. +func TestShrinkOnTargetTxnLimit(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, func(c *embed.Config) { c.MaxTxnOps = 8 }) + cfg := baseCfg("/src/", "/dst/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwrite + cfg.MaxTxnOps = 64 // engine batches far past the target's limit of 8 + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 20) + + r := startAgent(t, cfg, src, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing after shrink", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + assert.EqualValues(t, 0, snap.ForcedResyncCount) + assert.Empty(t, snap.LastError, "a successful shrink must clear the recorded error") + + // Live tail keeps working under the same limit. + _, err := src.Put(t.Context(), "/src/after", "ok") + require.NoError(t, err) + want["/dst/after"] = "ok" + waitTargetData(t, dst, cfg, 10*time.Second, want) +} + +// txnFailingClient fails every Txn with a transient error while armed. +type txnFailingClient struct { + mirroragent.Client + failing atomic.Bool +} + +func (c *txnFailingClient) Txn(ctx context.Context) clientv3.Txn { + return &failableTxn{inner: c.Client.Txn(ctx), c: c} +} + +type failableTxn struct { + inner clientv3.Txn + c *txnFailingClient +} + +func (t *failableTxn) If(cs ...clientv3.Cmp) clientv3.Txn { t.inner = t.inner.If(cs...); return t } +func (t *failableTxn) Then(ops ...clientv3.Op) clientv3.Txn { t.inner = t.inner.Then(ops...); return t } +func (t *failableTxn) Else(ops ...clientv3.Op) clientv3.Txn { t.inner = t.inner.Else(ops...); return t } +func (t *failableTxn) Commit() (*clientv3.TxnResponse, error) { + if t.c.failing.Load() { + return nil, status.Error(codes.Unavailable, "injected target stall") + } + return t.inner.Commit() +} + +// watchCountingClient counts Watch calls. +type watchCountingClient struct { + mirroragent.Client + watches atomic.Int64 +} + +func (c *watchCountingClient) Watch( + ctx context.Context, key string, opts ...clientv3.OpOption, +) clientv3.WatchChan { + c.watches.Add(1) + return c.Client.Watch(ctx, key, opts...) +} + +// TestSustainedTargetBackoffCancelsWatch: while the target stalls, clientv3 +// buffers undelivered source watch responses without bound — after a few +// backoff rounds the engine must cancel the source watch (bounding memory) +// and, once the target heals, resume from the checkpoint watermark on a +// fresh watch with nothing lost. +func TestSustainedTargetBackoffCancelsWatch(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + countingSrc := &watchCountingClient{Client: src} + failingDst := &txnFailingClient{Client: dst} + r := startAgent(t, cfg, countingSrc, failingDst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + waitSnap(t, r.agent, 10*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + before := countingSrc.watches.Load() + + // Stall the target and keep the source churning into the stalled apply. + failingDst.failing.Store(true) + ctx := t.Context() + _, err := src.Put(ctx, "/src/stall-trigger", "x") + require.NoError(t, err) + want["/dst/stall-trigger"] = "x" + waitSnap(t, r.agent, 20*time.Second, "Degraded during the stall", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseDegraded }) + // Ride out several backoff rounds (50ms initial, 500ms cap) so the + // watch-cancel threshold (3 rounds) is comfortably crossed. + time.Sleep(1 * time.Second) + + failingDst.failing.Store(false) + for i := range 3 { + k := fmt.Sprintf("post-stall-%d", i) + _, perr := src.Put(ctx, cfg.SourcePrefix+k, "y") + require.NoError(t, perr) + want[cfg.TargetPrefix+k] = "y" + } + waitTargetData(t, dst, cfg, 30*time.Second, want) + waitSnap(t, r.agent, 20*time.Second, "Syncing after the stall", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + assert.Greater(t, countingSrc.watches.Load(), before, + "the stall must have cancelled the source watch and re-watched from the checkpoint") + assert.EqualValues(t, 0, r.agent.Snapshot().ForcedResyncCount) +} diff --git a/pkg/mirroragent/integration_test.go b/pkg/mirroragent/integration_test.go new file mode 100644 index 00000000..dd491258 --- /dev/null +++ b/pkg/mirroragent/integration_test.go @@ -0,0 +1,531 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Integration tests for the mirror engine against two in-process embedded +// etcd servers (source + target). Each test exercises one contract from the +// Design-3 spec; intervals are shrunk via engine config knobs to keep the +// suite fast. +package mirroragent_test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.etcd.io/etcd-operator/pkg/mirroragent" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/server/v3/embed" +) + +// TestColdStart covers scenario 1: a populated source prefix is fully +// scanned onto the target, the checkpoint/fence key carries the scan-base +// watermark, and the reserved key is excluded from the data invariants. +func TestColdStart(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 50) + r0 := sourceRevision(t, src, cfg.SourcePrefix) + + r := startAgent(t, cfg, src, dst) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + assert.EqualValues(t, 50, snap.InitialSyncKeyCount) + assert.EqualValues(t, 50, snap.InitialSyncTotalKeyCount) + assert.EqualValues(t, 0, snap.ForcedResyncCount) + assert.Equal(t, r0, snap.Watermark) + + // Data invariant: exactly the 50 mirrored keys, reserved key excluded. + waitTargetData(t, dst, cfg, 10*time.Second, want) + raw, err := dst.Get(t.Context(), cfg.TargetPrefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) + require.NoError(t, err) + assert.EqualValues(t, 51, raw.Count, + "raw range must hold the 50 data keys plus the reserved checkpoint key") + + f, _ := readFence(t, dst, cfg) + assert.Equal(t, cfg.LinkUID, f.LinkUID) + assert.Equal(t, cfg.Epoch, f.Epoch) + assert.Equal(t, mirroragent.RoleMirror, f.Role) + assert.Equal(t, r0, f.Watermark, "checkpoint watermark must be the scan base revision") + assert.False(t, f.Scanning) +} + +// TestLiveTail covers scenario 2: puts, deletes, and a multi-key source Txn +// arriving during the tail land exactly, and one source revision lands as +// ONE target Txn — every key of the source Txn plus the checkpoint write +// shares a single target mod revision (revision-aligned batching; a partial +// revision is unobservable at any point because the Txn is atomic). +func TestLiveTail(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + ctx := t.Context() + _, err := src.Put(ctx, "/src/k1", "v1") + require.NoError(t, err) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{"/dst/k1": "v1"}) + + _, err = src.Delete(ctx, "/src/k1") + require.NoError(t, err) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{}) + + // One multi-key source Txn = one source revision. + txnResp, err := src.Txn(ctx).Then( + clientv3.OpPut("/src/t1", "a"), + clientv3.OpPut("/src/t2", "b"), + clientv3.OpPut("/src/t3", "c"), + ).Commit() + require.NoError(t, err) + srcTxnRev := txnResp.Header.Revision + + waitTargetData(t, dst, cfg, 10*time.Second, + map[string]string{"/dst/t1": "a", "/dst/t2": "b", "/dst/t3": "c"}) + + resp, err := dst.Get(ctx, "/dst/t1", clientv3.WithRange("/dst/t4")) + require.NoError(t, err) + require.Len(t, resp.Kvs, 3) + applyRev := resp.Kvs[0].ModRevision + for _, kv := range resp.Kvs { + assert.Equal(t, applyRev, kv.ModRevision, + "all keys of one source revision must land in one target Txn") + } + f, fenceModRev := readFence(t, dst, cfg) + assert.Equal(t, applyRev, fenceModRev, + "the checkpoint must be written in the SAME Txn as the applied batch") + assert.Equal(t, srcTxnRev, f.Watermark, + "checkpoint watermark must be the applied source revision") +} + +// TestMidScanCompaction covers scenario 3, the Design-3 headline: the source +// is compacted aggressively while a rate-limited scan is in flight, and the +// scan still converges with NO forced resync — the unpinned scan plus +// watch-replay-from-R0 eliminates mid-scan compaction as a failure class. +func TestMidScanCompaction(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.MaxOpsPerSecond = 150 // ~2s scan for 300 keys + cfg.PageKeyLimit = 50 + cfg.MaxTxnOps = 20 + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 300) + r0 := sourceRevision(t, src, cfg.SourcePrefix) + + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "scan started", + func(s mirroragent.Snapshot) bool { return s.InitialSyncKeyCount > 0 }) + + // Hammer the source: live writes advance the revision and compaction + // chases the head while the scan is still running. + ctx := t.Context() + for i := range 30 { + k := fmt.Sprintf("live-%03d", i) + _, err := src.Put(ctx, cfg.SourcePrefix+k, "live") + require.NoError(t, err) + want[cfg.TargetPrefix+k] = "live" + cur, err := src.Get(ctx, cfg.SourcePrefix, clientv3.WithPrefix(), clientv3.WithCountOnly()) + require.NoError(t, err) + _, _ = src.Compact(ctx, cur.Header.Revision) // errors ("already compacted") are fine + time.Sleep(50 * time.Millisecond) + } + + // Prove the compactions bit: a revision-pinned read at the scan base — + // what a mirror.Syncer-style pinned scan would issue — is now impossible. + _, err := src.Get(ctx, cfg.SourcePrefix, clientv3.WithPrefix(), clientv3.WithRev(r0)) + require.ErrorIs(t, rpctypes.Error(err), rpctypes.ErrCompacted, + "the scan base must actually be compacted for this test to mean anything") + + waitTargetData(t, dst, cfg, 30*time.Second, want) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing after compacted scan", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + assert.EqualValues(t, 0, snap.ForcedResyncCount, + "mid-scan compaction must not force a resync (class elimination)") + assert.False(t, snap.Compacted) + assert.False(t, snap.ResyncLoopDetected) +} + +// TestRestartResume covers scenario 4: a stopped engine restarted with the +// same linkUID/epoch resumes from the checkpoint in the target — zero source +// range reads (no rescan), only the watch replays the missed writes. +func TestRestartResume(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 30) + + r1 := startAgent(t, cfg, src, dst) + waitSnap(t, r1.agent, 20*time.Second, "first run Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, want) + err := r1.stop(t) + require.ErrorIs(t, err, context.Canceled) + + // Writes while the engine is down. + ctx := t.Context() + for i := range 5 { + k := fmt.Sprintf("extra-%d", i) + _, perr := src.Put(ctx, cfg.SourcePrefix+k, "late") + require.NoError(t, perr) + want[cfg.TargetPrefix+k] = "late" + } + lastRev := sourceRevision(t, src, cfg.SourcePrefix) + stopped, _ := readFence(t, dst, cfg) + + countingSrc := &countingClient{Client: src} + recordingSrc := &watchRevRecordingClient{Client: countingSrc} + r2 := startAgent(t, cfg, recordingSrc, dst) + waitTargetData(t, dst, cfg, 20*time.Second, want) + snap := waitSnap(t, r2.agent, 10*time.Second, "resumed watermark", + func(s mirroragent.Snapshot) bool { return s.Watermark >= lastRev }) + + assert.EqualValues(t, 0, countingSrc.gets.Load(), + "resume must not issue ANY source range read — no rescan") + // Pin the resume revision itself: zero Gets only rules out RANGE + // rescans; a watch opened at rev 1 would replay the whole history + // (idempotently, so no data assertion can catch it) and re-transfer the + // full prefix on every restart. + revs := recordingSrc.recorded() + require.NotEmpty(t, revs) + assert.Equal(t, stopped.Watermark+1, revs[0], + "the resumed watch must open at exactly checkpointWatermark+1") + assert.EqualValues(t, 0, snap.InitialSyncKeyCount, "resume must not re-run the initial scan") + assert.EqualValues(t, 0, snap.ForcedResyncCount) + f, _ := readFence(t, dst, cfg) + assert.GreaterOrEqual(t, f.Watermark, lastRev) +} + +// TestFenceOverlap covers scenario 5: two engines share the reserved key; a +// newer epoch takes the fence over, the stale writer's next Txn fails its +// mod-revision compare and stops with the fencing error, and the target +// state is the new writer's alone. +func TestFenceOverlap(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfgA := baseCfg("/src/", "/dst/") + + putN(t, src, cfgA.SourcePrefix, cfgA.TargetPrefix, 5) + + rA := startAgent(t, cfgA, src, dst) + waitSnap(t, rA.agent, 20*time.Second, "A Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + cfgB := cfgA + cfgB.Epoch = 2 + rB := startAgent(t, cfgB, src, dst) + waitSnap(t, rB.agent, 20*time.Second, "B Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + // Both engines watch this write; only epoch 2 may land it. + _, err := src.Put(t.Context(), "/src/fence-probe", "who-wins") + require.NoError(t, err) + + errA := rA.waitErr(t, 20*time.Second) + var fe *mirroragent.FenceError + require.ErrorAs(t, errA, &fe, "stale engine must stop with the fencing error, got: %v", errA) + assert.Contains(t, fe.Detail, "epoch 2") + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(errA)) + assert.Equal(t, mirroragent.PhaseFailed, rA.agent.Snapshot().Phase) + + resp, err := dst.Get(t.Context(), "/dst/fence-probe") + require.NoError(t, err) + require.Len(t, resp.Kvs, 1, "the new writer must have applied the probe write") + assert.Equal(t, "who-wins", string(resp.Kvs[0].Value)) + f, _ := readFence(t, dst, cfgB) + assert.Equal(t, int64(2), f.Epoch, "target fence must be the new writer's alone") + assert.Equal(t, mirroragent.PhaseSyncing, rB.agent.Snapshot().Phase) +} + +// TestRoleFlipCutoverFence covers scenario 6: flipping the fence role to +// Primary (simulated cutover) makes the running engine's next apply fail +// loudly with the cutover-fence error class, and nothing lands after it. +func TestRoleFlipCutoverFence(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 3) + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + // Simulated cutover: rewrite the fence with role=Primary. + f, _ := readFence(t, dst, cfg) + f.Role = mirroragent.RolePrimary + val, err := f.Encode() + require.NoError(t, err) + _, err = dst.Put(t.Context(), checkpointKey(cfg), val) + require.NoError(t, err) + + _, err = src.Put(t.Context(), "/src/after-cutover", "straggler") + require.NoError(t, err) + + runErr := r.waitErr(t, 20*time.Second) + var fe *mirroragent.FenceError + require.ErrorAs(t, runErr, &fe, "engine must stop with the fencing error, got: %v", runErr) + assert.Contains(t, fe.Detail, "Primary") + assert.Contains(t, fe.Detail, "cutover") + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + + resp, err := dst.Get(t.Context(), "/dst/after-cutover") + require.NoError(t, err) + assert.Empty(t, resp.Kvs, "no straggler apply may land after the role flip") +} + +// TestOversizedRevision covers scenario 7: a single source revision whose +// total bytes and op count both exceed the flush watermarks is applied as +// ONE oversized Txn with the checkpoint riding in it (held until it lands). +// The target's --max-txn-ops (embed MaxTxnOps) is bumped to make room. +func TestOversizedRevision(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, func(c *embed.Config) { c.MaxTxnOps = 64 }) + cfg := baseCfg("/src/", "/dst/") + cfg.MaxTxnOps = 6 // 5 data op slots + checkpoint slot + cfg.TxnFlushBytes = 2048 // the txn below is ~4.8KiB + cfg.MaxOpsPerSecond = 0 + + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + + big := strings.Repeat("v", 600) + ops := make([]clientv3.Op, 0, 8) + want := make(map[string]string, 8) + for i := range 8 { + k := fmt.Sprintf("big-%d", i) + ops = append(ops, clientv3.OpPut("/src/"+k, big)) + want["/dst/"+k] = big + } + txnResp, err := src.Txn(t.Context()).Then(ops...).Commit() + require.NoError(t, err) + + waitTargetData(t, dst, cfg, 20*time.Second, want) + resp, err := dst.Get(t.Context(), "/dst/big-", clientv3.WithPrefix()) + require.NoError(t, err) + require.Len(t, resp.Kvs, 8) + applyRev := resp.Kvs[0].ModRevision + for _, kv := range resp.Kvs { + assert.Equal(t, applyRev, kv.ModRevision, + "an oversized source revision must be applied as ONE Txn, never split") + } + f, fenceModRev := readFence(t, dst, cfg) + assert.Equal(t, applyRev, fenceModRev, + "the checkpoint must be held and land in the same oversized Txn") + assert.Equal(t, txnResp.Header.Revision, f.Watermark) +} + +// TestInitialSyncModes covers scenario 8: the three initialSync modes plus +// excludePrefixes. Sub-scenarios share one server pair on disjoint prefixes. +func TestInitialSyncModes(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + ctx := t.Context() + + t.Run("RequireEmptyViolation", func(t *testing.T) { + cfg := baseCfg("/m1/", "/d1/") + _, err := dst.Put(ctx, "/d1/existing", "dirty") + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + runErr := r.waitErr(t, 20*time.Second) + var ev *mirroragent.EmptyTargetViolationError + require.ErrorAs(t, runErr, &ev, "got: %v", runErr) + assert.EqualValues(t, 1, ev.KeyCount) + assert.Equal(t, mirroragent.ClassPermanent, mirroragent.Classify(runErr)) + assert.Equal(t, mirroragent.PhaseFailed, r.agent.Snapshot().Phase) + // A RequireEmpty violation must write nothing, not even the fence. + resp, err := dst.Get(ctx, checkpointKey(cfg)) + require.NoError(t, err) + assert.Empty(t, resp.Kvs) + }) + + t.Run("OverwriteKeepsOrphans", func(t *testing.T) { + cfg := baseCfg("/m2/", "/d2/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwrite + _, err := dst.Put(ctx, "/d2/stale", "old") + require.NoError(t, err) + _, err = dst.Put(ctx, "/d2/orphan", "keep-me") + require.NoError(t, err) + _, err = src.Put(ctx, "/m2/stale", "new") + require.NoError(t, err) + _, err = src.Put(ctx, "/m2/fresh", "1") + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{ + "/d2/stale": "new", // overwritten with source truth + "/d2/fresh": "1", // mirrored + "/d2/orphan": "keep-me", // Overwrite leaves orphans alone + }) + }) + + t.Run("OverwriteAndPruneRemovesOrphans", func(t *testing.T) { + cfg := baseCfg("/m3/", "/d3/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwriteAndPrune + // Simulates failback onto a stale prefix: "zombie" stands for a key + // deleted on the (new-primary) source after cutover — the prune pass + // must remove it from the old copy. + _, err := dst.Put(ctx, "/d3/stale", "old") + require.NoError(t, err) + _, err = dst.Put(ctx, "/d3/zombie", "deleted-post-cutover") + require.NoError(t, err) + _, err = src.Put(ctx, "/m3/stale", "new") + require.NoError(t, err) + _, err = src.Put(ctx, "/m3/fresh", "1") + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + snap := waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{ + "/d3/stale": "new", + "/d3/fresh": "1", + }) + require.NotNil(t, snap.LastReconcileDrift) + assert.EqualValues(t, 1, snap.LastReconcileDrift.OrphanKeys, + "the post-cutover-deleted key must be counted and pruned as an orphan") + }) + + t.Run("ExcludePrefixes", func(t *testing.T) { + cfg := baseCfg("/m4/", "/d4/") + cfg.InitialSyncMode = mirroragent.InitialSyncOverwriteAndPrune + cfg.ExcludePrefixes = []string{"/m4/skip/"} + _, err := src.Put(ctx, "/m4/keep/a", "1") + require.NoError(t, err) + _, err = src.Put(ctx, "/m4/skip/b", "2") + require.NoError(t, err) + // Pre-existing target key under the excluded image: never pruned. + _, err = dst.Put(ctx, "/d4/skip/old", "not-an-orphan") + require.NoError(t, err) + + r := startAgent(t, cfg, src, dst) + waitSnap(t, r.agent, 20*time.Second, "Syncing", + func(s mirroragent.Snapshot) bool { return s.Phase == mirroragent.PhaseSyncing }) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{ + "/d4/keep/a": "1", + "/d4/skip/old": "not-an-orphan", + }) + // A live write under the excluded prefix must not arrive either. + _, err = src.Put(ctx, "/m4/skip/c", "3") + require.NoError(t, err) + _, err = src.Put(ctx, "/m4/keep/d", "4") + require.NoError(t, err) + waitTargetData(t, dst, cfg, 10*time.Second, map[string]string{ + "/d4/keep/a": "1", + "/d4/keep/d": "4", + "/d4/skip/old": "not-an-orphan", + }) + }) +} + +// TestDrain covers scenario 9: with a quiesced source and mode Drain, the +// engine drains, verifies, reports a stable drained revision, and flips the +// fence role to Primary. +func TestDrain(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, nil) + cfg := baseCfg("/src/", "/dst/") + cfg.Mode = mirroragent.ModeDrain + + want := putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 12) + r0 := sourceRevision(t, src, cfg.SourcePrefix) + + r := startAgent(t, cfg, src, dst) + runErr := r.waitErr(t, 30*time.Second) + require.NoError(t, runErr, "a completed drain returns nil") + + snap := r.agent.Snapshot() + assert.Equal(t, mirroragent.PhaseDrained, snap.Phase) + assert.True(t, snap.CutoverReady) + require.NotNil(t, snap.Cutover) + assert.Equal(t, r0, snap.Cutover.DrainTargetRevision) + assert.GreaterOrEqual(t, snap.Cutover.DrainedRevision, snap.Cutover.DrainTargetRevision) + assert.EqualValues(t, 12, snap.Cutover.SourceKeyCount) + assert.EqualValues(t, 12, snap.Cutover.TargetKeyCount) + assert.False(t, snap.Cutover.VerifiedTime.IsZero()) + + waitTargetData(t, dst, cfg, 5*time.Second, want) + f, _ := readFence(t, dst, cfg) + assert.Equal(t, mirroragent.RolePrimary, f.Role, "drain completion must flip the fence to Primary") + assert.Equal(t, snap.Cutover.DrainedRevision, f.Watermark) + + // Stability: the reported drained revision does not move. + time.Sleep(300 * time.Millisecond) + again := r.agent.Snapshot() + assert.Equal(t, snap.Cutover.DrainedRevision, again.Cutover.DrainedRevision) +} + +// TestTargetQuotaExhausted covers scenario 10: a target with an exhausted +// backend quota yields the typed TargetQuotaExhausted classification and the +// engine parks on the flat probe interval instead of hot-retrying. +func TestTargetQuotaExhausted(t *testing.T) { + src := startEtcd(t, nil) + dst := startEtcd(t, func(c *embed.Config) { c.QuotaBackendBytes = 4 * 1024 * 1024 }) + cfg := baseCfg("/src/", "/dst/") + cfg.QuotaProbeInterval = 250 * time.Millisecond + + // Fill the target past its quota until NOSPACE trips. + big := strings.Repeat("x", 1<<20) + sawNoSpace := false + for i := range 40 { + _, err := dst.Put(t.Context(), fmt.Sprintf("/fill/%02d", i), big) + if err != nil && errors.Is(rpctypes.Error(err), rpctypes.ErrNoSpace) { + sawNoSpace = true + break + } + require.NoError(t, err) + } + require.True(t, sawNoSpace, "target never hit NOSPACE while filling") + + putN(t, src, cfg.SourcePrefix, cfg.TargetPrefix, 2) + countingDst := &countingClient{Client: dst} + r := startAgent(t, cfg, src, countingDst) + + snap := waitSnap(t, r.agent, 20*time.Second, "quota classification", + func(s mirroragent.Snapshot) bool { return s.QuotaExhausted }) + assert.Equal(t, mirroragent.ClassQuota, snap.LastErrorClass, + "NOSPACE must classify as the quota class, never throttling or transient") + assert.Equal(t, mirroragent.PhaseDegraded, snap.Phase) + + // No hot retry loop: attempts are paced by QuotaProbeInterval. + before := countingDst.txns.Load() + time.Sleep(1200 * time.Millisecond) + delta := countingDst.txns.Load() - before + assert.LessOrEqual(t, delta, int64(8), + "quota parking must probe on the flat interval, not spin (saw %d Txns in 1.2s)", delta) + + // Run must still be parked, not returned. + select { + case err := <-r.done: + t.Fatalf("Run returned during quota park: %v", err) + default: + } +} diff --git a/pkg/mirroragent/reconcile.go b/pkg/mirroragent/reconcile.go new file mode 100644 index 00000000..a1acf315 --- /dev/null +++ b/pkg/mirroragent/reconcile.go @@ -0,0 +1,320 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "strings" + + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// reconcilePass is the shared diff-and-repair pass: the OverwriteAndPrune +// genesis pass, the mandatory mark-and-sweep after any forced resync, and +// the Drain verification repair. It merges bounded pages of BOTH sides in +// key order — the source via the excluded-range-elided scan windows (so +// excluded data is never transferred, matching the genesis scan), the target +// one page at a time — so agent memory stays bounded by two pages no matter +// how divergent the target is (an orphan-heavy target is exactly the state +// this pass exists to repair). Repair puts source-truth values over missing +// or value-divergent target keys, deleteOrphans removes target keys with no +// source counterpart. The reserved checkpoint key and images of excluded +// prefixes are never touched, a sibling link's fence key aborts the pass +// (PrefixConflictError) instead of being pruned, and every repair/delete +// rides the same fenced Txn path as applies. +func (a *Agent) reconcilePass(ctx context.Context, repair, deleteOrphans bool) (Drift, error) { + drift := Drift{Repaired: repair || deleteOrphans} + dstStart, dstEnd := a.rw.destRange() + b := newBatcher(a.cfg.MaxTxnOps, a.cfg.TxnFlushBytes) + dstCursor := dstStart + var srcSeen, dstSeen int64 + pager := &sourcePager{a: a, ranges: a.rw.scanRanges()} + for { + spage, more, err := pager.next(ctx) + if err != nil { + return drift, err + } + // The window of target keys this source page is authoritative for; + // the last page's window swallows the tail of the destination range. + winEnd := dstEnd + if more && len(spage) > 0 { + winEnd = nextKey(a.rw.image(string(spage[len(spage)-1].Key))) + } + expected := make(map[string]string, len(spage)) + for _, kv := range spage { + if dstKey, ok := a.rw.rewrite(string(kv.Key)); ok { + expected[dstKey] = string(kv.Value) + srcSeen++ + } + } + // Stream the target side of the window one page at a time — never + // materialized whole. + tcursor := dstCursor + for { + tresp, terr := a.getRetry(ctx, a.dst, tcursor, clientv3.WithRange(winEnd), + clientv3.WithLimit(int64(a.cfg.PageKeyLimit)), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if terr != nil { + return drift, terr + } + var ops []kvOp + for _, kv := range tresp.Kvs { + k := string(kv.Key) + if k == a.cfg.CheckpointKey || a.rw.excludedImage(k) { + continue + } + dstSeen++ + want, ok := expected[k] + if !ok { + if cerr := a.checkForeignFence(k, kv.Value); cerr != nil { + return drift, cerr + } + drift.OrphanKeys++ + if deleteOrphans { + ops = append(ops, kvOp{key: k, isDelete: true}) + } + continue + } + delete(expected, k) + if want != string(kv.Value) { + drift.DivergentKeys++ + if repair { + ops = append(ops, kvOp{key: k, value: want}) + } + } + } + if err := a.enqueueRepairs(ctx, b, ops); err != nil { + return drift, err + } + if len(tresp.Kvs) == 0 || !tresp.More { + break + } + tcursor = nextKey(string(tresp.Kvs[len(tresp.Kvs)-1].Key)) + } + // Source keys of this window with no target counterpart. + var missing []kvOp + for k, v := range expected { + drift.MissingKeys++ + if repair { + missing = append(missing, kvOp{key: k, value: v}) + } + } + if err := a.enqueueRepairs(ctx, b, missing); err != nil { + return drift, err + } + dstCursor = winEnd + if !more { + break + } + } + if fs := b.flush(); fs != nil { + if err := a.applyRepairFlush(ctx, fs); err != nil { + return drift, err + } + } + // Per-side key counts as observed by this pass (pre-repair): the + // equality signal behind the InvariantsHeld condition. + a.recordKeyCounts(srcSeen, dstSeen) + return drift, nil +} + +// enqueueRepairs pushes repair/prune ops through the shared batcher, +// applying any flush sets that become due. +func (a *Agent) enqueueRepairs(ctx context.Context, b *batcher, ops []kvOp) error { + for _, op := range ops { + g := revGroup{rev: a.fence.Watermark, ops: []kvOp{op}} + for _, fs := range b.add(g) { + if err := a.applyRepairFlush(ctx, &fs); err != nil { + return err + } + } + } + return nil +} + +// checkForeignFence refuses to treat another EtcdMirror link's reserved +// checkpoint/fence key (recognized by the \x00-after-prefix reserved-key +// convention plus a decodable fence document) as a prunable orphan: +// overlapping destination prefixes must stop loudly instead of silently +// destroying the sibling link's fence and data. +func (a *Agent) checkForeignFence(key string, value []byte) error { + if !strings.Contains(key, "\x00") { + return nil + } + fv, err := DecodeFenceValue(value) + if err != nil || fv.LinkUID == a.cfg.LinkUID { + return nil + } + return &PrefixConflictError{ + Key: RedactKey(a.cfg.EffectiveDestPrefix(), []byte(key)), + OwnerLinkUID: fv.LinkUID, + } +} + +// sourcePager yields the source side of the reconcile merge one bounded page +// at a time, walking the excluded-range-elided scan windows in key order so +// excluded data is never transferred (server-side elision, matching the +// genesis scan). The reported more flag is false only on the final page of +// the final non-empty window; a one-page lookahead across window boundaries +// decides it. +type sourcePager struct { + a *Agent + ranges []scanRange + ri int + cursor string // resume point within ranges[ri]; "" means the range start + buf *srcPage +} + +type srcPage struct { + kvs []*mvccpb.KeyValue + more bool // resp.More within the page's own window +} + +func (p *sourcePager) next(ctx context.Context) ([]*mvccpb.KeyValue, bool, error) { + cur := p.buf + p.buf = nil + if cur == nil { + var err error + if cur, err = p.fetch(ctx); err != nil { + return nil, false, err + } + } + if cur == nil { + return nil, false, nil + } + if cur.more { + return cur.kvs, true, nil + } + // Window boundary: look one page ahead to decide whether any source key + // remains in a later window. + nxt, err := p.fetch(ctx) + if err != nil { + return nil, false, err + } + p.buf = nxt + return cur.kvs, nxt != nil, nil +} + +// fetch returns the next non-empty page across the remaining windows, or nil +// when every window is exhausted. +func (p *sourcePager) fetch(ctx context.Context) (*srcPage, error) { + for p.ri < len(p.ranges) { + kr := p.ranges[p.ri] + start := kr.start + if p.cursor != "" { + start = p.cursor + } + resp, err := p.a.getRetry(ctx, p.a.src, start, clientv3.WithRange(kr.end), + clientv3.WithLimit(int64(p.a.cfg.PageKeyLimit)), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if err != nil { + return nil, err + } + if resp.More { + p.cursor = nextKey(string(resp.Kvs[len(resp.Kvs)-1].Key)) + } else { + p.ri++ + p.cursor = "" + } + if len(resp.Kvs) > 0 { + return &srcPage{kvs: resp.Kvs, more: resp.More}, nil + } + } + return nil, nil +} + +// recordKeyCounts publishes the per-side in-scope key counts observed by a +// reconciliation, prune, or drain verification pass. +func (a *Agent) recordKeyCounts(srcN, dstN int64) { + a.update(func(s *Snapshot) { + s.SourceKeyCount = srcN + s.TargetKeyCount = dstN + }) +} + +// applyRepairFlush writes repair/prune ops under the fence without moving +// any checkpoint field: the pass is positionless, only the compare matters. +func (a *Agent) applyRepairFlush(ctx context.Context, fs *flushSet) error { + f := a.fence + f.Epoch = a.cfg.Epoch + return a.applyOps(ctx, fs, f) +} + +// verifyCounts counts in-scope keys on both sides: excluded prefixes are +// elided via the scan windows and the reserved checkpoint key is not counted +// on the target. Source counts are pinned at the checkpoint watermark (the +// drained revision during a drain) so a non-quiesced source still yields a +// coherent snapshot; if that revision has been compacted the count falls +// back to an unpinned re-read. +func (a *Agent) verifyCounts(ctx context.Context) (srcN, dstN int64, err error) { + if srcN, err = a.countSourceInScope(ctx, a.watermark()); err != nil { + if Classify(err) != ClassResync { + return 0, 0, err + } + if srcN, err = a.countSourceInScope(ctx, 0); err != nil { + return 0, 0, err + } + } + dstStart, dstEnd := a.rw.destRange() + if dstN, err = a.countRange(ctx, a.dst, dstStart, dstEnd, 0); err != nil { + return 0, 0, err + } + ck, err := a.getRetry(ctx, a.dst, a.cfg.CheckpointKey, clientv3.WithCountOnly()) + if err != nil { + return 0, 0, err + } + dstN -= ck.Count + for _, p := range a.cfg.ExcludePrefixes { + if !strings.HasPrefix(p, a.cfg.SourcePrefix) { + continue + } + s, e := keyRange(a.rw.image(p)) + n, cerr := a.countRange(ctx, a.dst, s, e, 0) + if cerr != nil { + return 0, 0, cerr + } + dstN -= n + } + return srcN, dstN, nil +} + +// countSourceInScope sums the in-scope source key count over the scan +// windows, pinned at rev when rev > 0. +func (a *Agent) countSourceInScope(ctx context.Context, rev int64) (int64, error) { + var total int64 + for _, kr := range a.rw.scanRanges() { + n, err := a.countRange(ctx, a.src, kr.start, kr.end, rev) + if err != nil { + return 0, err + } + total += n + } + return total, nil +} + +func (a *Agent) countRange(ctx context.Context, cl Client, start, end string, rev int64) (int64, error) { + opts := []clientv3.OpOption{clientv3.WithRange(end), clientv3.WithCountOnly()} + if rev > 0 { + opts = append(opts, clientv3.WithRev(rev)) + } + resp, err := a.getRetry(ctx, cl, start, opts...) + if err != nil { + return 0, err + } + return resp.Count, nil +} diff --git a/pkg/mirroragent/rewrite.go b/pkg/mirroragent/rewrite.go new file mode 100644 index 00000000..4a67c0c0 --- /dev/null +++ b/pkg/mirroragent/rewrite.go @@ -0,0 +1,184 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "strings" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// rewriter implements the mirror's single key-rewrite formula: +// +// key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) +// +// It is an anchored strip-and-reprefix — never a substring replace — and it +// is order-preserving within the source scope, which the prune pass's merge +// scan relies on. +type rewriter struct { + srcPrefix string + // dstPrefix is the effective destination prefix (target.prefix + + // destPrefix). + dstPrefix string + exclude []string + // reservedKey is never produced by any apply path, even if a source key + // happens to map onto it. + reservedKey string +} + +func newRewriter(cfg Config) *rewriter { + return &rewriter{ + srcPrefix: cfg.SourcePrefix, + dstPrefix: cfg.EffectiveDestPrefix(), + exclude: cfg.ExcludePrefixes, + reservedKey: cfg.CheckpointKey, + } +} + +// inScope reports whether a source key is mirrored: under the source prefix +// and not excluded. +func (r *rewriter) inScope(srcKey string) bool { + if !strings.HasPrefix(srcKey, r.srcPrefix) { + return false + } + return !r.excluded(srcKey) +} + +// excluded reports whether a source key falls under any exclude prefix. +func (r *rewriter) excluded(srcKey string) bool { + for _, p := range r.exclude { + if strings.HasPrefix(srcKey, p) { + return true + } + } + return false +} + +// rewrite maps an in-scope source key to its target key. ok is false when +// the key is out of scope, excluded, or would collide with the reserved +// checkpoint key. +func (r *rewriter) rewrite(srcKey string) (string, bool) { + if !r.inScope(srcKey) { + return "", false + } + dst := r.dstPrefix + strings.TrimPrefix(srcKey, r.srcPrefix) + if dst == r.reservedKey { + return "", false + } + return dst, true +} + +// image maps ANY source key under the source prefix (including excluded +// ones) to its would-be target key, for cursor/window arithmetic in the +// merge scan. Callers must ensure srcKey has the source prefix. +func (r *rewriter) image(srcKey string) string { + return r.dstPrefix + strings.TrimPrefix(srcKey, r.srcPrefix) +} + +// excludedImage reports whether a TARGET key falls under the image of an +// exclude prefix: such keys are never treated as orphans by prune passes. +func (r *rewriter) excludedImage(dstKey string) bool { + for _, p := range r.exclude { + if !strings.HasPrefix(p, r.srcPrefix) { + continue // excluded range is outside the mirrored scope + } + if strings.HasPrefix(dstKey, r.image(p)) { + return true + } + } + return false +} + +// sourceRange returns the [start, end) watch/scan range for the source +// prefix. An empty prefix means the whole keyspace. +func (r *rewriter) sourceRange() (string, string) { + return keyRange(r.srcPrefix) +} + +// scanRange is one [Start, End) source read window. End == rangeEndInf is +// etcd's ">= Start" sentinel (unbounded). +type scanRange struct { + start, end string +} + +// rangeEndInf is etcd's unbounded range-end sentinel ("all keys >= start"). +const rangeEndInf = "\x00" + +// endAfter reports whether range end e lies strictly after key k, treating +// the sentinel as +inf. +func endAfter(e, k string) bool { + return e == rangeEndInf || e > k +} + +// scanRanges returns the source range minus every excluded range: the +// sorted, disjoint windows the genesis scan reads. Range Gets are issued +// ONLY over these windows, so excluded data is elided server-side — never +// transferred and filtered client-side. (The watch still spans the whole +// source range: watches cannot be decomposed without multiplying streams, +// and its events are filtered through rewrite.) +func (r *rewriter) scanRanges() []scanRange { + start, end := r.sourceRange() + out := []scanRange{{start: start, end: end}} + for _, p := range r.exclude { + es, ee := keyRange(p) + next := make([]scanRange, 0, len(out)+1) + for _, w := range out { + next = append(next, subtractRange(w, es, ee)...) + } + out = next + } + return out +} + +// subtractRange removes the [es, ee) slice from window w, yielding 0, 1, or +// 2 remaining windows. +func subtractRange(w scanRange, es, ee string) []scanRange { + // No overlap: the excluded range ends at/before the window starts, or + // starts at/after the window ends. + if !endAfter(ee, w.start) || !endAfter(w.end, es) { + return []scanRange{w} + } + out := make([]scanRange, 0, 2) + if es > w.start { + out = append(out, scanRange{start: w.start, end: es}) + } + if ee != rangeEndInf && endAfter(w.end, ee) { + out = append(out, scanRange{start: ee, end: w.end}) + } + return out +} + +// destRange returns the [start, end) range covering the effective +// destination prefix on the target. +func (r *rewriter) destRange() (string, string) { + return keyRange(r.dstPrefix) +} + +// keyRange converts a prefix to an etcd [start, end) range. The empty prefix +// maps to the whole keyspace ("\x00" with the >=-key range end "\x00"). +func keyRange(prefix string) (string, string) { + if prefix == "" { + return "\x00", "\x00" + } + return prefix, clientv3.GetPrefixRangeEnd(prefix) +} + +// nextKey returns the smallest key strictly greater than k, for cursor +// advancement. +func nextKey(k string) string { + return k + "\x00" +} diff --git a/pkg/mirroragent/rewrite_test.go b/pkg/mirroragent/rewrite_test.go new file mode 100644 index 00000000..e43e641c --- /dev/null +++ b/pkg/mirroragent/rewrite_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRewriteFormula pins the single anchored rewrite formula: +// key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix). +func TestRewriteFormula(t *testing.T) { + cases := []struct { + name string + cfg Config + srcKey string + want string + wantOK bool + }{ + {name: "default destPrefix strips the source prefix", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/"}, + srcKey: "/apps/foo", want: "/mirror/foo", wantOK: true}, + {name: "non-empty destPrefix is the middle term", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/", DestPrefix: "west/"}, + srcKey: "/apps/foo", want: "/mirror/west/foo", wantOK: true}, + {name: "key equal to the source prefix maps to the effective dest prefix", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/", DestPrefix: "west/"}, + srcKey: "/apps/", want: "/mirror/west/", wantOK: true}, + {name: "anchored, never a substring replace", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/"}, + srcKey: "/apps/foo/apps/bar", want: "/mirror/foo/apps/bar", wantOK: true}, + {name: "out of scope", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/"}, + srcKey: "/other/foo", wantOK: false}, + {name: "excluded prefix", + cfg: Config{SourcePrefix: "/apps/", TargetPrefix: "/mirror/", + ExcludePrefixes: []string{"/apps/skip/"}}, + srcKey: "/apps/skip/foo", wantOK: false}, + {name: "empty source prefix mirrors the whole keyspace", + cfg: Config{TargetPrefix: "/mirror/"}, + srcKey: "/anything", want: "/mirror//anything", wantOK: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rw := newRewriter(tc.cfg.withDefaults()) + got, ok := rw.rewrite(tc.srcKey) + require.Equal(t, tc.wantOK, ok) + if tc.wantOK { + assert.Equal(t, tc.want, got) + } + }) + } +} + +// TestRewriteReservedKeyCollision: a source key whose image would be the +// reserved checkpoint key is never produced by any apply path. +func TestRewriteReservedKeyCollision(t *testing.T) { + cfg := Config{SourcePrefix: "/s/", TargetPrefix: "/d/", CheckpointKey: "/d/ckpt"}.withDefaults() + rw := newRewriter(cfg) + + _, ok := rw.rewrite("/s/ckpt") + assert.False(t, ok, "the reserved key's preimage must be dropped") + got, ok := rw.rewrite("/s/ckpt2") + assert.True(t, ok, "only the exact match is reserved") + assert.Equal(t, "/d/ckpt2", got) +} + +func TestExcludedImage(t *testing.T) { + cfg := Config{ + SourcePrefix: "/s/", + TargetPrefix: "/d/", + ExcludePrefixes: []string{"/s/skip/", "/elsewhere/"}, + }.withDefaults() + rw := newRewriter(cfg) + + assert.True(t, rw.excludedImage("/d/skip/x"), "images of excluded ranges are never orphans") + assert.False(t, rw.excludedImage("/d/keep/x")) + assert.False(t, rw.excludedImage("/d/elsewhere/x"), + "excludes outside the mirrored scope have no image") +} + +func TestKeyRange(t *testing.T) { + s, e := keyRange("/p/") + assert.Equal(t, "/p/", s) + assert.Equal(t, "/p0", e, "prefix range end increments the last byte") + + s, e = keyRange("") + assert.Equal(t, "\x00", s, "empty prefix means the whole keyspace") + assert.Equal(t, "\x00", e, "with the >=-key sentinel range end") +} + +// TestScanRanges pins the range decomposition: the genesis scan reads ONLY +// the source range minus the excluded ranges — excluded data is elided +// server-side, not filtered client-side. +func TestScanRanges(t *testing.T) { + cases := []struct { + name string + src string + exclude []string + want []scanRange + }{ + {name: "no excludes", src: "/s/", + want: []scanRange{{"/s/", "/s0"}}}, + {name: "one middle exclude", src: "/s/", exclude: []string{"/s/b/"}, + want: []scanRange{{"/s/", "/s/b/"}, {"/s/b0", "/s0"}}}, + {name: "two excludes stay sorted", src: "/s/", exclude: []string{"/s/b/", "/s/d/"}, + want: []scanRange{{"/s/", "/s/b/"}, {"/s/b0", "/s/d/"}, {"/s/d0", "/s0"}}}, + {name: "exclude at the start", src: "/s/", exclude: []string{"/s/"}, + want: []scanRange{}}, + {name: "exclude covering the whole source prefix", src: "/s/sub/", exclude: []string{"/s/"}, + want: []scanRange{}}, + {name: "exclude outside the source range", src: "/s/", exclude: []string{"/t/"}, + want: []scanRange{{"/s/", "/s0"}}}, + {name: "whole keyspace with one exclude", src: "", exclude: []string{"/b/"}, + want: []scanRange{{"\x00", "/b/"}, {"/b0", "\x00"}}}, + {name: "nested excludes merge", src: "/s/", exclude: []string{"/s/b/", "/s/b/c/"}, + want: []scanRange{{"/s/", "/s/b/"}, {"/s/b0", "/s0"}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rw := newRewriter(Config{ + SourcePrefix: tc.src, + TargetPrefix: "/d/", + ExcludePrefixes: tc.exclude, + }.withDefaults()) + got := rw.scanRanges() + if len(tc.want) == 0 { + assert.Empty(t, got) + return + } + require.Equal(t, tc.want, got) + }) + } +} + +func TestEndAfter(t *testing.T) { + assert.True(t, endAfter(rangeEndInf, "/z/"), "the sentinel end is +inf") + assert.True(t, endAfter("/b/", "/a/")) + assert.False(t, endAfter("/a/", "/a/")) + assert.False(t, endAfter("/a/", "/b/")) +} diff --git a/pkg/mirroragent/scan.go b/pkg/mirroragent/scan.go new file mode 100644 index 00000000..e82f8f61 --- /dev/null +++ b/pkg/mirroragent/scan.go @@ -0,0 +1,432 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "fmt" + "sync" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// scanRestartError aborts a genesis attempt so Run restarts the scan from a +// fresh R0 — a bounded retry, distinct from a forced resync (the checkpoint +// is not invalidated). Buffered watch events were dropped, so the restarted +// scan owes a mark-and-sweep prune (deletes from the dropped window must not +// resurrect). +type scanRestartError struct { + Cause ScanRestartCause + Err error +} + +func (e *scanRestartError) Error() string { + return fmt.Sprintf("genesis scan restart required (%s): %v", e.Cause, e.Err) +} +func (e *scanRestartError) Unwrap() error { return e.Err } + +// replayBuffer drains the watch opened before a genesis scan into a +// byte-bounded buffer, so the reflector replay base is bounded by +// Config.WatchBufferBytes instead of clientv3's unbounded internal queue. +// On overflow — or a watch reconnect landing below the source compact +// revision — it cancels the watch and records a scanRestartError. +type replayBuffer struct { + limit int64 + cancelWatch context.CancelFunc + + mu sync.Mutex + resps []clientv3.WatchResponse + bytes int64 + fail *scanRestartError + + stopc chan struct{} + donec chan struct{} +} + +func newReplayBuffer(limit int64, cancelWatch context.CancelFunc) *replayBuffer { + return &replayBuffer{ + limit: limit, + cancelWatch: cancelWatch, + stopc: make(chan struct{}), + donec: make(chan struct{}), + } +} + +// fill consumes wch until stop is called, the channel dies, or a restart +// condition hits. Run it in a goroutine; it never blocks the scan. +func (rb *replayBuffer) fill(wch clientv3.WatchChan) { + defer close(rb.donec) + for { + select { + case <-rb.stopc: + return + case wr, ok := <-wch: + if !ok { + // Channel died (transient watch failure): keep what was + // buffered; the tail re-watches from the watermark and the + // missed span comes from etcd's watch history. + return + } + if wr.CompactRevision != 0 { + rb.setFail(&scanRestartError{Cause: ScanRestartWatchCompactedMidScan, Err: wr.Err()}) + return + } + if wr.Err() != nil || wr.IsProgressNotify() { + // Errors surface via channel close; progress notifications + // carry nothing to replay (the watermark must stay the scan + // base R0 while scanning). + continue + } + var n int64 + for _, ev := range wr.Events { + n += int64(len(ev.Kv.Key) + len(ev.Kv.Value)) + } + rb.mu.Lock() + rb.bytes += n + over := rb.bytes > rb.limit + if !over { + rb.resps = append(rb.resps, wr) + } + rb.mu.Unlock() + if over { + rb.setFail(&scanRestartError{ + Cause: ScanRestartWatchBufferOverflow, + Err: fmt.Errorf("replay buffer exceeded %d bytes before the base scan completed", + rb.limit), + }) + return + } + } + } +} + +// setFail records the restart condition and cancels the watch so the client +// stops accumulating events for a doomed attempt. +func (rb *replayBuffer) setFail(e *scanRestartError) { + rb.mu.Lock() + rb.fail = e + rb.mu.Unlock() + rb.cancelWatch() +} + +// err returns the pending scanRestartError, if any. +func (rb *replayBuffer) err() error { + rb.mu.Lock() + defer rb.mu.Unlock() + if rb.fail != nil { + return rb.fail + } + return nil +} + +// stop halts filling and hands back the buffered responses for replay. +func (rb *replayBuffer) stop() []clientv3.WatchResponse { + close(rb.stopc) + <-rb.donec + rb.mu.Lock() + defer rb.mu.Unlock() + return rb.resps +} + +// genesis is the cold-start / forced-resync path: RequireEmpty gate, fence +// claim, watch-before-scan, unpinned chunked scan, optional prune pass, +// replay of the buffered watch events, then the live tail over the same +// watch channel. +func (a *Agent) genesis(ctx context.Context, st startState) error { + a.setPhase(PhaseInitialSync) + srcStart, srcEnd := a.rw.sourceRange() + + // A forced resync owes a mandatory mark-and-sweep, as does an + // OverwriteAndPrune genesis. The obligation is stamped into every + // checkpoint (FenceValue.PrunePending) until the prune completes, so a + // crash mid-genesis cannot silently drop the owed sweep. + if st.forced || a.cfg.InitialSyncMode == InitialSyncOverwriteAndPrune { + a.prunePending = true + } + + // Forced resyncs never re-check RequireEmpty: the fence proves the + // destination data is this link's own. A cluster-identity mismatch + // re-arms it. + checkEmpty := (!st.haveCheckpoint && !st.forced) || st.rearmEmpty + if a.cfg.InitialSyncMode == InitialSyncRequireEmpty && checkEmpty { + if err := a.requireEmpty(ctx); err != nil { + return err + } + } + + var r0 int64 + cursor := srcStart + subrev := int64(0) + if st.haveCheckpoint && st.scanning && st.scanCursor != "" { + // Resume an interrupted scan from its cursor at the recorded base. + r0 = st.watermark + cursor = nextKey(st.scanCursor) + subrev = st.subRevision + } else { + // Observe the scan base BEFORE scanning: the watch starts at r0+1 + // and buffered events replay over the scanned base. The windows' + // counts are the InitialSync denominator, for free (excluded + // prefixes are elided from the windows, so they are never counted). + rev, total, err := a.countScanRanges(ctx) + if err != nil { + return err + } + r0 = rev + a.update(func(s *Snapshot) { + s.InitialSyncTotalKeyCount = total + s.InitialSyncKeyCount = 0 + s.InitialSyncStartTime = time.Now() + s.InitialSyncCompletionTime = time.Time{} + s.LeaseBackedKeyCount = 0 + s.SourceRevision = r0 + }) + } + + // Claim (or refresh) the fence before any data write. + if err := a.applyFenced(ctx, nil, a.newFence(r0, true, st.scanCursor, subrev), "", 0); err != nil { + return err + } + + // Open the watch BEFORE the scan (reflector pattern): events during the + // scan buffer (byte-bounded) and replay over the scanned base + // afterwards, so mid-scan compaction cannot invalidate anything the + // scan needs — the scan itself reads unpinned, at the current revision. + wctx, wcancel := context.WithCancel(clientv3.WithRequireLeader(ctx)) + defer wcancel() + wch := a.src.Watch(wctx, srcStart, clientv3.WithRange(srcEnd), + clientv3.WithRev(r0+1), clientv3.WithProgressNotify()) + rb := newReplayBuffer(a.cfg.WatchBufferBytes, wcancel) + go rb.fill(wch) + // Publish the cancel so a sustained target stall during the scan or the + // replay can stop the source stream (the tail then re-watches from the + // watermark). rb.fill tolerates the resulting channel death by design. + a.watchCancel = wcancel + + if err := a.scan(ctx, cursor, r0, subrev, rb); err != nil { + return err + } + + // The mandatory mark-and-sweep: the OverwriteAndPrune genesis pass and + // every forced resync, cleared only once the pass completed. + if a.prunePending { + drift, err := a.reconcilePass(ctx, true, true) + if err != nil { + return err + } + a.prunePending = false + a.update(func(s *Snapshot) { + s.LastReconcileTime = time.Now() + d := drift + s.LastReconcileDrift = &d + }) + } + + // Scan complete: first revision-complete checkpoint at the scan base. + if err := a.applyFenced(ctx, nil, a.newFence(r0, false, "", 0), "", 0); err != nil { + return err + } + a.update(func(s *Snapshot) { + s.InitialSyncCompletionTime = time.Now() + if r0 > s.Watermark { + s.Watermark = r0 + } + s.LastProgressTime = time.Now() + }) + + // Hand the watch over from the buffer to the live tail: stop filling, + // replay what was buffered over the scanned base, then consume the + // still-open channel directly. A restart condition recorded at any + // point during scan/prune aborts the attempt instead. + buffered := rb.stop() + if err := rb.err(); err != nil { + return err + } + for i := range buffered { + // live=false: replay applies run INSIDE the (possibly forced-resync) + // genesis and must not reset the resync-loop detector. + if err := a.handleResponse(ctx, &buffered[i], false); err != nil { + return err + } + } + return a.tail(ctx, wch, wcancel, r0) +} + +// countScanRanges counts the in-scope source keys window by window and +// returns the revision R0 observed by the first (linearizable) read plus +// the total count. No Get ever spans an excluded range. +func (a *Agent) countScanRanges(ctx context.Context) (r0, total int64, err error) { + ranges := a.rw.scanRanges() + if len(ranges) == 0 { + // Everything is excluded: one point read still pins R0. + start, _ := a.rw.sourceRange() + resp, gerr := a.getRetry(ctx, a.src, start, clientv3.WithCountOnly()) + if gerr != nil { + return 0, 0, gerr + } + return resp.Header.Revision, 0, nil + } + for i, kr := range ranges { + resp, gerr := a.getRetry(ctx, a.src, kr.start, + clientv3.WithRange(kr.end), clientv3.WithCountOnly()) + if gerr != nil { + return 0, 0, gerr + } + if i == 0 { + r0 = resp.Header.Revision + } + total += resp.Count + } + return r0, total, nil +} + +// scan pulls byte-bounded pages at the current revision (never pinned) over +// the decomposed in-scope windows and applies them as fenced Txns carrying +// the in-scan checkpoint. rb is polled between pages so an overflowed or +// compacted replay buffer aborts the attempt promptly. +func (a *Agent) scan(ctx context.Context, cursor string, r0, subrev int64, rb *replayBuffer) error { + b := newBatcher(a.cfg.MaxTxnOps, a.cfg.TxnFlushBytes) + limit := a.cfg.PageKeyLimit + for _, kr := range a.rw.scanRanges() { + if !endAfter(kr.end, cursor) { + continue // window fully below the resume cursor + } + next := kr.start + if cursor > next { + next = cursor + } + var err error + if limit, subrev, err = a.scanWindow(ctx, next, kr.end, r0, limit, subrev, b, rb); err != nil { + return err + } + } + if fs := b.flush(); fs != nil { + subrev++ + return a.applyScanFlush(ctx, fs, subrev) + } + return rb.err() +} + +// scanWindow pages one [cursor, end) window through the shared batcher. +func (a *Agent) scanWindow( + ctx context.Context, cursor, end string, r0 int64, + limit int, subrev int64, b *batcher, rb *replayBuffer, +) (int, int64, error) { + for { + if err := rb.err(); err != nil { + return limit, subrev, err + } + resp, err := a.getRetry(ctx, a.src, cursor, clientv3.WithRange(end), + clientv3.WithLimit(int64(limit)), + clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if err != nil { + return limit, subrev, err + } + if len(resp.Kvs) == 0 { + return limit, subrev, nil + } + var pageBytes, leased int64 + for _, kv := range resp.Kvs { + pageBytes += int64(len(kv.Key) + len(kv.Value)) + dstKey, ok := a.rw.rewrite(string(kv.Key)) + if !ok { + continue + } + if kv.Lease != 0 { + leased++ + } + // Synthetic single-key groups: a snapshot has no per-key + // revision boundaries to preserve. + g := revGroup{rev: r0, ops: []kvOp{{ + key: dstKey, value: string(kv.Value), srcKey: string(kv.Key), + }}} + for _, fs := range b.add(g) { + subrev++ + if err := a.applyScanFlush(ctx, &fs, subrev); err != nil { + return limit, subrev, err + } + } + } + if leased > 0 { + a.update(func(s *Snapshot) { s.LeaseBackedKeyCount += leased }) + } + limit = adaptLimit(limit, a.cfg.PageKeyLimit, pageBytes, a.cfg.PageBytes) + if !resp.More { + return limit, subrev, nil + } + cursor = nextKey(string(resp.Kvs[len(resp.Kvs)-1].Key)) + } +} + +// applyScanFlush applies one scan flush set with the in-scan checkpoint +// (Scanning=true, cursor advanced, page ordinal in SubRevision) riding the +// same Txn, so a restarted agent resumes the scan instead of starting over. +func (a *Agent) applyScanFlush(ctx context.Context, fs *flushSet, subrev int64) error { + f := a.newFence(fs.watermark, true, fs.lastSrcKey, subrev) + if err := a.applyOps(ctx, fs, f); err != nil { + return err + } + a.update(func(s *Snapshot) { + s.InitialSyncKeyCount += int64(len(fs.ops)) + s.LastProgressTime = time.Now() + }) + return nil +} + +// requireEmpty enforces InitialSyncRequireEmpty over the effective +// destination prefix; the reserved checkpoint key is excluded by exact +// match. +func (a *Agent) requireEmpty(ctx context.Context) error { + start, end := a.rw.destRange() + resp, err := a.getRetry(ctx, a.dst, start, clientv3.WithRange(end), clientv3.WithCountOnly()) + if err != nil { + return err + } + n := resp.Count + if n > 0 { + ck, cerr := a.getRetry(ctx, a.dst, a.cfg.CheckpointKey, clientv3.WithCountOnly()) + if cerr != nil { + return cerr + } + n -= ck.Count + } + if n > 0 { + return &EmptyTargetViolationError{RangeStart: start, RangeEnd: end, KeyCount: n} + } + return nil +} + +// adaptLimit derives the next page's key limit from the observed bytes/key, +// enforcing the PageBytes bound etcd Range lacks natively. +func adaptLimit(cur, maxLimit int, gotBytes, maxBytes int64) int { + if gotBytes <= 0 { + return cur + } + switch { + case gotBytes > maxBytes && cur > 1: + cur /= 2 + if cur < 1 { + cur = 1 + } + case gotBytes*2 < maxBytes && cur < maxLimit: + cur *= 2 + if cur > maxLimit { + cur = maxLimit + } + } + return cur +} diff --git a/pkg/mirroragent/snapshot.go b/pkg/mirroragent/snapshot.go new file mode 100644 index 00000000..dbd56a28 --- /dev/null +++ b/pkg/mirroragent/snapshot.go @@ -0,0 +1,177 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import "time" + +// Phase mirrors EtcdMirrorPhase in api/v1alpha1 (the library adds Drained, +// which the controller maps to the CutoverReady condition). +type Phase string + +const ( + PhaseConnecting Phase = "Connecting" + PhaseInitialSync Phase = "InitialSync" + PhaseSyncing Phase = "Syncing" + PhaseDegraded Phase = "Degraded" + PhaseFailed Phase = "Failed" + // PhaseDrained means the drain completed, verification passed, and the + // fence role is Primary. Run has returned; the agent will never write + // again. + PhaseDrained Phase = "Drained" +) + +// ScanRestartCause says why a genesis scan attempt was aborted and restarted +// from a fresh R0. Both causes surface as one operator-facing event +// (InitialSyncCompactionRaced) with the cause named in the message, so +// "buffer too small for churn" is never conflated with "compaction won a +// race the design eliminates". Repeated restarts count toward the +// resync-loop detector. +type ScanRestartCause string + +const ( + // ScanRestartWatchBufferOverflow: the replay buffer exceeded + // Config.WatchBufferBytes before the base scan completed — a + // memory-bound retry, NOT a compaction race. + ScanRestartWatchBufferOverflow ScanRestartCause = "WatchBufferOverflow" + // ScanRestartWatchCompactedMidScan: a watch reconnect landed below the + // source compact revision while the scan was still running — the rare + // genuine race. + ScanRestartWatchCompactedMidScan ScanRestartCause = "WatchCompactedMidScan" +) + +// Drift is the outcome of one reconciliation pass. +type Drift struct { + // MissingKeys were present on the source but absent on the target + // (repaired when the pass repairs). + MissingKeys int64 + // DivergentKeys were present on both sides with different values + // (repaired to source truth when the pass repairs). Distinct from + // MissingKeys so operators can tell "a resync dropped keys" from "a + // blind window went stale". + DivergentKeys int64 + // OrphanKeys were present on the target with no source counterpart + // (deleted when the pass deletes orphans). + OrphanKeys int64 + // Repaired is true when the pass wrote fixes rather than only reporting. + Repaired bool +} + +// CutoverStatus tracks a Drain-mode cutover; see EtcdMirrorCutoverStatus. +type CutoverStatus struct { + // DrainTargetRevision is the source revision observed when the drain + // started — the revision the watermark must reach. + DrainTargetRevision int64 + // DrainedRevision is the watermark at which the drain completed. + DrainedRevision int64 + // VerifiedTime is when the post-drain verification pass succeeded. + VerifiedTime time.Time + // SourceKeyCount / TargetKeyCount are the per-side key counts from the + // verification pass (source read pinned at DrainedRevision with an + // unpinned fallback if compacted; excluded prefixes and the reserved + // checkpoint key are not counted). + SourceKeyCount int64 + TargetKeyCount int64 + // LeasedKeyCount is the lease-backed key count frozen at drain + // completion, for the runbook's purge/re-lease step. + LeasedKeyCount int64 +} + +// Snapshot is a point-in-time copy of the agent's state, safe to retain. +// Later rungs (the agent binary's /statusz, the controller's status sync) +// poll this instead of scraping internals. +type Snapshot struct { + Phase Phase + + SourceVersion string + TargetVersion string + // SourceClusterID / TargetClusterID as probed at connect (0 = not yet + // probed). Both are bound into the checkpoint. + SourceClusterID uint64 + TargetClusterID uint64 + + // Watermark is the checkpoint watermark: the source revision through + // which the target is caught up, advanced by applies AND by watch + // progress notifications on idle prefixes. The fenced checkpoint key in + // the target etcd is the authoritative copy; this mirrors it. + Watermark int64 + // SourceRevision is the source cluster's revision as of the last watch + // header. Cluster-global: it advances on out-of-prefix writes, so + // SourceRevision-Watermark overstates lag for prefix-scoped mirrors. + SourceRevision int64 + // LastProgressTime is when the watermark last advanced. This — not + // apply activity — is the liveness signal: an idle prefix on a live + // watch keeps progressing via notifications. + LastProgressTime time.Time + + InitialSyncKeyCount int64 + InitialSyncTotalKeyCount int64 + InitialSyncStartTime time.Time + InitialSyncCompletionTime time.Time + + // LeaseBackedKeyCount is the number of mirrored keys whose source copy + // is lease-backed (kv.Lease != 0). Mirrored copies are NOT lease-backed + // — leases are stripped — so a nonzero count means the cutover + // runbook's purge/re-lease step applies. + LeaseBackedKeyCount int64 + + // ForcedResyncCount is monotonic, never reset. LastResyncReason is the + // most recent trigger. ResyncLoopDetected latches when + // ResyncLoopThreshold consecutive resyncs completed without reaching + // steady state (the livelock signature of retention < scan+drain time); + // it clears only when steady state is reached. + ForcedResyncCount int64 + LastResyncReason ResyncReason + ResyncLoopDetected bool + + // ScanRestartCount is monotonic: genesis scan attempts aborted and + // restarted from a fresh R0 (see ScanRestartCause). Distinct from + // ForcedResyncCount — a restart is a bounded retry within InitialSync, + // not a checkpoint invalidation — but restarts count toward the same + // resync-loop detector. + ScanRestartCount int64 + LastScanRestartCause ScanRestartCause + + // SourceKeyCount / TargetKeyCount are the per-side in-scope key counts + // observed by the most recent reconciliation, prune, or drain + // verification pass (excluded prefixes and the reserved checkpoint key + // not counted). Populated by every pass that runs regardless of config — + // forced-resync sweeps, the OverwriteAndPrune genesis pass, drain + // verification — but NOT refreshed outside those passes: a healthy + // mirror that never resyncs only gets counts from an enabled periodic + // pass. This is the equality signal the controller's InvariantsHeld + // condition reads. + SourceKeyCount int64 + TargetKeyCount int64 + + // Condition-shaped flags. + Throttled bool + QuotaExhausted bool + Compacted bool + + LastReconcileTime time.Time + LastReconcileDrift *Drift + + // LastError / LastErrorClass describe the most recent classified + // failure ("" when the last attempt succeeded). + LastError string + LastErrorClass Class + + // Cutover is populated once a drain starts; CutoverReady flips when the + // fence role is Primary. + CutoverReady bool + Cutover *CutoverStatus +} diff --git a/pkg/mirroragent/tail.go b/pkg/mirroragent/tail.go new file mode 100644 index 00000000..a167d2a7 --- /dev/null +++ b/pkg/mirroragent/tail.go @@ -0,0 +1,289 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "context" + "errors" + "fmt" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +// tail is the live watch-replay loop. wch may carry a watch opened before a +// genesis scan (whose buffered events replay over the scanned base), with +// wcancel its cancel func; when wch is nil, a watch opens at fromRev+1. +// Transient watch failures re-watch from the checkpoint watermark; +// compaction of the resume revision propagates as a forced resync. The +// current watch's cancel is published via Agent.watchCancel so a stalled +// apply can stop the source stream (bounded memory during target stalls). +func (a *Agent) tail(ctx context.Context, wch clientv3.WatchChan, wcancel context.CancelFunc, fromRev int64) error { + a.setPhase(PhaseSyncing) + srcStart, srcEnd := a.rw.sourceRange() + for { + if wch == nil { + rev := a.watermark() + if rev < fromRev { + rev = fromRev + } + var wctx context.Context + wctx, wcancel = context.WithCancel(clientv3.WithRequireLeader(ctx)) + wch = a.src.Watch(wctx, srcStart, clientv3.WithRange(srcEnd), + clientv3.WithRev(rev+1), clientv3.WithProgressNotify()) + } + a.watchCancel = wcancel + err := a.consume(ctx, wch) + a.watchCancel = nil + if wcancel != nil { + wcancel() + } + wch, wcancel = nil, nil + if errors.Is(err, errDrained) || ctx.Err() != nil { + return err + } + switch class := Classify(err); class { + case ClassTransient, ClassThrottle: + a.recordErr(err, class) + a.setPhase(PhaseDegraded) + if serr := sleepCtx(ctx, a.bo.next(class)); serr != nil { + return serr + } + a.setPhase(PhaseSyncing) + default: + return err + } + } +} + +// consume applies watch responses until the channel closes or fails. It +// drives client-side progress requests (server-side notify intervals are +// uncontrollable on foreign clusters) and checks the drain gate between +// responses. +func (a *Agent) consume(ctx context.Context, wch clientv3.WatchChan) error { + ticker := time.NewTicker(a.cfg.ProgressInterval) + defer ticker.Stop() + for { + if err := a.maybeDrain(ctx); err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + // The progress request MUST carry the same outgoing metadata as + // the Watch call: clientv3 keys watcher gRPC streams by ctx + // metadata, and every engine watch is opened WithRequireLeader. + // A bare ctx here would address an empty stream and no + // notification would ever arrive on an idle prefix. + rctx, cancel := context.WithTimeout(clientv3.WithRequireLeader(ctx), a.cfg.RequestTimeout) + _ = a.src.RequestProgress(rctx) + cancel() + case wr, ok := <-wch: + if !ok { + return fmt.Errorf("source watch channel closed") + } + if wr.CompactRevision != 0 { + return &ResyncError{Reason: ResyncReasonCompacted, Cause: wr.Err()} + } + if err := wr.Err(); err != nil { + return err + } + if err := a.handleResponse(ctx, &wr, true); err != nil { + return err + } + } + } +} + +// handleResponse applies one watch response: whole revisions coalesced into +// fenced Txns, flushed only at source-revision boundaries; trusted progress +// notifications advance the watermark checkpoint on idle prefixes. live is +// true only for responses consumed from a live tail channel — genesis +// replay-buffer applies pass false so they never reset the resync-loop +// detector from inside the very resync it is counting. +func (a *Agent) handleResponse(ctx context.Context, wr *clientv3.WatchResponse, live bool) error { + hdrRev := wr.Header.Revision + a.update(func(s *Snapshot) { + if hdrRev > s.SourceRevision { + s.SourceRevision = hdrRev + } + }) + if wr.IsProgressNotify() { + if !a.trustProgress || hdrRev <= a.watermark() { + return nil + } + if err := a.applyFenced(ctx, nil, a.newFence(hdrRev, false, "", 0), "", 0); err != nil { + return err + } + a.advanceWatermark(hdrRev) + return nil + } + + ops := make([]kvOp, 0, len(wr.Events)) + revs := make([]int64, 0, len(wr.Events)) + var leased int64 + for _, ev := range wr.Events { + srcKey := string(ev.Kv.Key) + dstKey, ok := a.rw.rewrite(srcKey) + if !ok { + continue + } + op := kvOp{key: dstKey, srcKey: srcKey} + if ev.Type == clientv3.EventTypeDelete { + op.isDelete = true + } else { + op.value = string(ev.Kv.Value) + if ev.Kv.Lease != 0 { + leased++ + } + } + ops = append(ops, op) + revs = append(revs, ev.Kv.ModRevision) + } + if leased > 0 { + a.update(func(s *Snapshot) { s.LeaseBackedKeyCount += leased }) + } + if len(ops) == 0 { + return nil + } + // A source revision's events never span watch responses, so flushing at + // the end of the response only ever cuts at a revision boundary. + b := newBatcher(a.cfg.MaxTxnOps, a.cfg.TxnFlushBytes) + for _, g := range groupByRevision(ops, revs) { + for _, fs := range b.add(g) { + if err := a.applyLiveFlush(ctx, &fs); err != nil { + return err + } + } + } + if fs := b.flush(); fs != nil { + if err := a.applyLiveFlush(ctx, fs); err != nil { + return err + } + } + if live { + a.steadyState() + } + return nil +} + +// applyLiveFlush applies one revision-complete flush set; the checkpoint +// watermark advances to the set's last complete revision in the same Txn. +func (a *Agent) applyLiveFlush(ctx context.Context, fs *flushSet) error { + if err := a.applyOps(ctx, fs, a.newFence(fs.watermark, false, "", 0)); err != nil { + return err + } + a.advanceWatermark(fs.watermark) + return nil +} + +// maybeDrain drives Drain mode: record the drain target revision once, then +// complete the cutover when the checkpoint watermark reaches it. +func (a *Agent) maybeDrain(ctx context.Context) error { + if a.cfg.Mode != ModeDrain && !a.drainReq.Load() { + return nil + } + snap := a.Snapshot() + if snap.Cutover == nil { + srcStart, srcEnd := a.rw.sourceRange() + var target int64 + if a.trustProgress { + resp, err := a.getRetry(ctx, a.src, srcStart, + clientv3.WithRange(srcEnd), clientv3.WithCountOnly()) + if err != nil { + return err + } + target = resp.Header.Revision + } else { + // Below the progress-trust floor (source < 3.4.25 / 3.5.8) the + // watermark advances ONLY on applied in-prefix events, so a + // cluster-revision drain target would never terminate on a shared + // source: out-of-prefix writes push it past the last in-prefix + // event while the drain itself quiesces in-prefix writers. Fall + // back to the highest in-scope mod revision; a tombstone above it + // (an in-flight delete) is caught and repaired by the drain + // verification pass before the role flips. + resp, err := a.getRetry(ctx, a.src, srcStart, clientv3.WithRange(srcEnd), + clientv3.WithSort(clientv3.SortByModRevision, clientv3.SortDescend), + clientv3.WithLimit(1), clientv3.WithKeysOnly()) + if err != nil { + return err + } + if len(resp.Kvs) > 0 { + target = resp.Kvs[0].ModRevision + } else { + target = a.watermark() + } + } + a.update(func(s *Snapshot) { + s.Cutover = &CutoverStatus{DrainTargetRevision: target} + if target > s.SourceRevision { + s.SourceRevision = target + } + }) + snap = a.Snapshot() + } + if snap.Watermark < snap.Cutover.DrainTargetRevision { + return nil + } + return a.completeDrain(ctx) +} + +// completeDrain verifies convergence, records the cutover block, and flips +// the fence role to Primary so any straggler mirror apply fails its compare +// loudly. Returns errDrained on success. +func (a *Agent) completeDrain(ctx context.Context) error { + srcN, dstN, err := a.verifyCounts(ctx) + if err != nil { + return err + } + a.recordKeyCounts(srcN, dstN) + if srcN != dstN { + // One repair+prune pass and a recount; a persisting mismatch is real + // divergence and must fail the drain rather than cut over. + if _, rerr := a.reconcilePass(ctx, true, true); rerr != nil { + return rerr + } + if srcN, dstN, err = a.verifyCounts(ctx); err != nil { + return err + } + a.recordKeyCounts(srcN, dstN) + if srcN != dstN { + return &DrainVerificationError{SourceKeys: srcN, TargetKeys: dstN} + } + } + wm := a.watermark() + f := a.newFence(wm, false, "", 0) + f.Role = RolePrimary + if err := a.applyFenced(ctx, nil, f, "", 0); err != nil { + return err + } + now := time.Now() + a.update(func(s *Snapshot) { + c := *s.Cutover + c.DrainedRevision = wm + c.VerifiedTime = now + c.SourceKeyCount = srcN + c.TargetKeyCount = dstN + c.LeasedKeyCount = s.LeaseBackedKeyCount + s.Cutover = &c + s.CutoverReady = true + s.Phase = PhaseDrained + }) + return errDrained +} diff --git a/pkg/mirroragent/version.go b/pkg/mirroragent/version.go new file mode 100644 index 00000000..87cea9f2 --- /dev/null +++ b/pkg/mirroragent/version.go @@ -0,0 +1,65 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "fmt" + + "github.com/coreos/go-semver/semver" +) + +// hardVersionFloor is the declared minimum etcd server version. Below it the +// agent fails permanently (UnsupportedVersion) rather than degrading in +// undefined ways. +const hardVersionFloor = "3.4.0" + +// Watch progress notifications are unreliable below 3.4.25 / 3.5.8. Below +// these floors the agent does not trust progress notifications: the +// watermark only advances on applies, so idle prefixes resync from scratch +// after any restart longer than retention, and a Drain may not terminate on +// a quiet prefix. +var progressTrustFloors = map[int64]semver.Version{ + 4: {Major: 3, Minor: 4, Patch: 25}, + 5: {Major: 3, Minor: 5, Patch: 8}, +} + +// versionInfo is the outcome of the connect-time maintenance Status() probe. +type versionInfo struct { + Version string + // TrustProgressNotify gates the watermark machinery that drives lag, + // idle-prefix checkpointing, and the Drain gate. + TrustProgressNotify bool +} + +// classifyVersion enforces the hard floor and derives progress trust. +// side is "source" or "target"; the hard floor is enforced on both, the +// progress-trust floor only matters for the source (the watched side). +func classifyVersion(side, version string) (versionInfo, error) { + v, err := semver.NewVersion(version) + if err != nil { + return versionInfo{}, fmt.Errorf("unparseable %s etcd version %q: %w", side, version, err) + } + floor := semver.New(hardVersionFloor) + if v.LessThan(*floor) { + return versionInfo{}, &UnsupportedVersionError{Side: side, Version: version} + } + info := versionInfo{Version: version, TrustProgressNotify: true} + if trustFloor, ok := progressTrustFloors[v.Minor]; ok && v.Major == 3 { + info.TrustProgressNotify = !v.LessThan(trustFloor) + } + return info, nil +} diff --git a/pkg/mirroragent/version_test.go b/pkg/mirroragent/version_test.go new file mode 100644 index 00000000..2a29a1b9 --- /dev/null +++ b/pkg/mirroragent/version_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mirroragent + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestClassifyVersion pins the version gates: the >=3.4.0 hard floor (below +// it the agent fails permanently) and the 3.4.25/3.5.8 progress-trust floor +// (below it progress notifications can report revisions ahead of delivered +// events, so trusting them would checkpoint the watermark past undelivered +// data — silent loss after a restart). +func TestClassifyVersion(t *testing.T) { + cases := []struct { + version string + wantErr bool + wantFloor bool // UnsupportedVersionError specifically + wantTrust bool + }{ + {version: "3.3.9", wantErr: true, wantFloor: true}, + {version: "3.0.0", wantErr: true, wantFloor: true}, + {version: "3.4.0", wantTrust: false}, + {version: "3.4.24", wantTrust: false}, + {version: "3.4.25", wantTrust: true}, + {version: "3.4.33", wantTrust: true}, + {version: "3.5.0", wantTrust: false}, + {version: "3.5.7", wantTrust: false}, + {version: "3.5.8", wantTrust: true}, + {version: "3.6.0", wantTrust: true}, + {version: "3.6.12", wantTrust: true}, + {version: "4.0.0", wantTrust: true}, + {version: "garbage", wantErr: true}, + {version: "", wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.version, func(t *testing.T) { + info, err := classifyVersion("source", tc.version) + if tc.wantErr { + require.Error(t, err) + var uv *UnsupportedVersionError + if tc.wantFloor { + require.ErrorAs(t, err, &uv) + assert.Equal(t, "source", uv.Side) + assert.Equal(t, ClassPermanent, Classify(err)) + } + return + } + require.NoError(t, err) + assert.Equal(t, tc.version, info.Version) + assert.Equal(t, tc.wantTrust, info.TrustProgressNotify, + "progress-trust gate for %s", tc.version) + }) + } +} From df7b7e9ce53c8b4875a89d52032e125902e40e4e Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Tue, 7 Jul 2026 19:41:52 -0400 Subject: [PATCH 4/5] test(etcdmirror): deflake the resync-loop latch clear phase A single post-heal put can race the recovery attempt's restart backoff and land below the scan's R0: the scan applies it, the live tail has nothing to deliver, and no live apply ever clears the latch. Write a fresh key per poll tick so one is guaranteed to arrive via the tail. Signed-off-by: Xavier Lange --- pkg/mirroragent/integration_delta_test.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/mirroragent/integration_delta_test.go b/pkg/mirroragent/integration_delta_test.go index a7eb10b0..00b9283b 100644 --- a/pkg/mirroragent/integration_delta_test.go +++ b/pkg/mirroragent/integration_delta_test.go @@ -412,13 +412,21 @@ func TestResyncLoopLatch(t *testing.T) { waitTargetData(t, dst, cfg, 30*time.Second, want) // Steady state = a successfully applied TAIL response; only that clears - // the latch (scan convergence alone does not). - _, err = src.Put(t.Context(), "/src/steady", "ok") - require.NoError(t, err) - want["/dst/steady"] = "ok" - waitTargetData(t, dst, cfg, 20*time.Second, want) + // the latch (scan convergence alone does not). A single post-heal put can + // race the recovery attempt's restart backoff and land below the scan's + // R0 — applied by the scan, leaving the live tail with nothing to + // deliver — so write a fresh key per poll tick until one arrives live. + steady := 0 waitSnap(t, r.agent, 20*time.Second, "latch clears at steady state", - func(s mirroragent.Snapshot) bool { return !s.ResyncLoopDetected && !s.Compacted }) + func(s mirroragent.Snapshot) bool { + if !s.ResyncLoopDetected && !s.Compacted { + return true + } + steady++ + _, perr := src.Put(t.Context(), fmt.Sprintf("/src/steady-%d", steady), "ok") + require.NoError(t, perr) + return false + }) } // rangeRecordingClient records every Get's [start, end) window, to prove From 03c5fce0e4903898ecd442986da878da9d2e58ce Mon Sep 17 00:00:00 2001 From: Xavier Lange Date: Wed, 8 Jul 2026 05:09:26 -0400 Subject: [PATCH 5/5] test(etcdmirror): poll the fence for PrunePending after mismatch re-arm Data convergence and the checkpoint write that clears PrunePending are separate Txns; a one-shot fence read can land between them under load. Signed-off-by: Xavier Lange (cherry picked from commit e9d2e9f2e7327a355e09f0eadafb2ce03e8b834f) --- pkg/mirroragent/integration_delta_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/mirroragent/integration_delta_test.go b/pkg/mirroragent/integration_delta_test.go index 00b9283b..f32bd6cc 100644 --- a/pkg/mirroragent/integration_delta_test.go +++ b/pkg/mirroragent/integration_delta_test.go @@ -329,7 +329,15 @@ func TestClusterIDMismatchRearm(t *testing.T) { waitTargetData(t, dst, cfg, 30*time.Second, map[string]string{"/dst/fresh": "new-world"}) snap := r2.agent.Snapshot() assert.Equal(t, mirroragent.ResyncReasonClusterIDMismatch, snap.LastResyncReason) + // Data convergence and the checkpoint write that clears PrunePending + // are separate Txns; under load the fence read can land between them, + // so poll the fence rather than reading it once. + deadline := time.Now().Add(10 * time.Second) f, _ := readFence(t, dst, cfg) + for f.PrunePending && time.Now().Before(deadline) { + time.Sleep(100 * time.Millisecond) + f, _ = readFence(t, dst, cfg) + } assert.False(t, f.PrunePending) assert.Equal(t, snap.SourceClusterID, f.SourceClusterID, "the checkpoint must re-bind to the new source cluster identity")