diff --git a/api/v1alpha1/etcdmirror_types.go b/api/v1alpha1/etcdmirror_types.go new file mode 100644 index 00000000..ff4b4d7a --- /dev/null +++ b/api/v1alpha1/etcdmirror_types.go @@ -0,0 +1,934 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EtcdMirrorMode selects the mirror's operating mode. +type EtcdMirrorMode string + +const ( + // EtcdMirrorModeSync is normal continuous replication. + EtcdMirrorModeSync EtcdMirrorMode = "Sync" + // EtcdMirrorModeDrain prepares for cutover: the agent records the source + // revision observed when Drain is requested (status.cutover.drainTargetRevision), + // keeps replicating until the checkpoint watermark reaches it, runs a + // verification pass (per-side key counts, lease-backed key count), then + // sets the CutoverReady condition and flips the fence key's role to + // Primary so any straggler apply fails its mod-revision compare loudly. + // Runbook: quiesce source writers -> set mode=Drain -> + // `kubectl wait --for=condition=CutoverReady etcdmirror/` -> + // purge/re-lease lease-backed keys -> repoint clients -> delete the CR. + EtcdMirrorModeDrain EtcdMirrorMode = "Drain" +) + +// EtcdMirrorInitialSyncMode governs how the agent treats pre-existing keys +// under the effective destination prefix at genesis (first-ever sync, or a +// checkpoint invalidated by a cluster-identity mismatch). +type EtcdMirrorInitialSyncMode string + +const ( + // EtcdMirrorInitialSyncRequireEmpty refuses to start if the destination + // prefix already holds any key (Phase -> Failed, condition + // EmptyTargetViolation). The reserved checkpoint key is excluded by exact + // match. + // + // RE-ARM CONTRACT: a source OR target cluster-ID mismatch invalidates + // the checkpoint, forces genesis, and RE-ARMS this check. An ordinary + // forced resync (Compacted) does NOT re-check RequireEmpty: the decoded, + // ownership-validated fence proves the destination data is this link's + // own. + EtcdMirrorInitialSyncRequireEmpty EtcdMirrorInitialSyncMode = "RequireEmpty" + // EtcdMirrorInitialSyncOverwrite scans and writes over whatever is there. + // Keys present on the target but absent on the source are left alone. + EtcdMirrorInitialSyncOverwrite EtcdMirrorInitialSyncMode = "Overwrite" + // EtcdMirrorInitialSyncOverwriteAndPrune is Overwrite plus one mandatory + // orphan-prune pass after the scan: target keys under the destination + // prefix with no source counterpart are deleted. This makes reversal onto + // a previously-populated prefix (failback) a first-class correct + // operation instead of silently resurrecting deleted keys. + EtcdMirrorInitialSyncOverwriteAndPrune EtcdMirrorInitialSyncMode = "OverwriteAndPrune" +) + +// EtcdMirrorSpec defines the desired state of an EtcdMirror. +// +// Range-defining and rewrite fields are immutable (CEL transition rules +// below): source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes +// and checkpoint.key. Changing what range is mirrored, or where it lands, +// mid-life silently diverges: a restarted agent resumes from its checkpoint +// without a scan, so removing an exclusion never backfills pre-existing keys +// and adding one strands already-mirrored keys as permanent orphans — all +// with every condition green. Endpoints stay mutable — rotating an NLB DNS +// name or adding a member to the same cluster is routine; pointing at a +// different cluster is caught at runtime by the checkpoint's dual-cluster-ID +// binding, not by spec validation. +// +// The transition rules compare VALUES, presence-normalized: for these +// fields the empty value and an absent field are semantically identical +// (prefix "" = whole keyspace, destPrefix "" = strip the source prefix), and +// Go typed clients drop explicit "" through omitempty — presence-based rules +// would reject every typed-client update of a CR created with an explicit +// empty string. +// +// +kubebuilder:validation:XValidation:rule="(has(self.source.prefix) ? self.source.prefix : \"\") == (has(oldSelf.source.prefix) ? oldSelf.source.prefix : \"\")",message="source.prefix is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.target.prefix) ? self.target.prefix : \"\") == (has(oldSelf.target.prefix) ? oldSelf.target.prefix : \"\")",message="target.prefix is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.sync) && has(self.sync.destPrefix) ? self.sync.destPrefix : \"\") == (has(oldSelf.sync) && has(oldSelf.sync.destPrefix) ? oldSelf.sync.destPrefix : \"\")",message="sync.destPrefix is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.sync) && has(self.sync.excludePrefixes) ? self.sync.excludePrefixes : []) == (has(oldSelf.sync) && has(oldSelf.sync.excludePrefixes) ? oldSelf.sync.excludePrefixes : [])",message="sync.excludePrefixes is immutable" +// +kubebuilder:validation:XValidation:rule="(has(self.checkpoint) && has(self.checkpoint.key) ? self.checkpoint.key : \"\") == (has(oldSelf.checkpoint) && has(oldSelf.checkpoint.key) ? oldSelf.checkpoint.key : \"\")",message="checkpoint.key is immutable" +// +kubebuilder:validation:XValidation:rule="!has(self.checkpoint) || !has(self.checkpoint.key) || self.checkpoint.key == \"\" || self.checkpoint.key.startsWith((has(self.target.prefix) ? self.target.prefix : \"\") + (has(self.sync) && has(self.sync.destPrefix) ? self.sync.destPrefix : \"\"))",message="checkpoint.key must live under the effective destination prefix (target.prefix + sync.destPrefix)" +type EtcdMirrorSpec struct { + // Mode selects continuous replication (Sync, the default) or a cutover + // drain (Drain). See EtcdMirrorModeDrain for the cutover contract. + // +optional + // +kubebuilder:validation:Enum=Sync;Drain + // +kubebuilder:default=Sync + Mode EtcdMirrorMode `json:"mode,omitempty"` + + // Source is the etcd cluster keys are read from. EtcdMirror never writes + // back to Source; the agent's source-side client is only ever used for + // Get/Watch, never Put/Delete/Txn. + // + // VERSION FLOOR: source etcd must be >= 3.4 (probed via maintenance + // Status() at connect; below the floor the mirror goes Failed with reason + // UnsupportedVersion). >= 3.4.25 / 3.5.8 is the recommended floor: below + // it, watch progress notifications are unreliable and the agent cannot + // trust the watermark machinery that drives lag, the checkpoint, and the + // Drain gate. + Source EtcdMirrorEndpoint `json:"source"` + + // Target is the etcd cluster keys are written into. + // + // SECURITY PREREQUISITE: Target's credential (etcd RBAC user and/or the + // client certificate's role) MUST be range-scoped to the effective + // destination prefix — never a cluster-admin-equivalent credential. The + // agent's client-side rewrite logic is defense against bugs in its own + // code, NOT a security boundary. The grant must also cover the reserved + // checkpoint key (see Checkpoint). Configure via `etcdctl role + // grant-permission --prefix=true readwrite ` before + // pointing an EtcdMirror at the cluster. + // + // The target must run with auto-compaction enabled: forced-resync churn + // and prune passes march an uncompacted target toward its storage quota + // (2GiB by default), which surfaces as TargetQuotaExhausted. + Target EtcdMirrorEndpoint `json:"target"` + + // InitialSync governs genesis behavior: how pre-existing destination keys + // are treated (Mode) and optionally where replication starts + // (StartRevision). + // +optional + InitialSync *EtcdMirrorInitialSyncSpec `json:"initialSync,omitempty"` + + // Sync tunes runtime sync behavior (batching, paging, rate limiting, + // prefix rewrite, timeouts, backoff). + // +optional + Sync EtcdMirrorSyncSpec `json:"sync,omitempty"` + + // Checkpoint configures the reserved checkpoint/fence key on the target. + // +optional + Checkpoint *EtcdMirrorCheckpointSpec `json:"checkpoint,omitempty"` + + // Reconciliation optionally enables a periodic full diff-and-repair pass + // layered on top of the continuous watch-based mirror. Independent of + // this setting, one reconciliation-with-delete pass always runs after any + // forced resync (mark-and-sweep), and as the OverwriteAndPrune genesis + // pass and the Drain verification pass. + // +optional + Reconciliation *EtcdMirrorReconciliationSpec `json:"reconciliation,omitempty"` + + // PodTemplate carries scheduling/affinity/labels/annotations for the + // agent pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. + // +optional + PodTemplate *PodTemplate `json:"podTemplate,omitempty"` + + // Resources are the agent container's compute resources. The agent's + // memory model is bounded by Sync.PageBytes (single in-flight scan page, + // no unbounded read-ahead); size limits accordingly. + // +optional + Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + + // Paused, when true, scales the agent Deployment to zero without deleting + // the CR or its checkpoint. The checkpoint lives in the target etcd, so + // resume picks up from the last fenced watermark. NOTE: pausing longer + // than the source's compaction retention guarantees a full forced resync + // on resume — there is no free lunch past the retention window. + // +optional + Paused bool `json:"paused,omitempty"` +} + +// EtcdMirrorInitialSyncSpec governs the genesis scan. +// +// +kubebuilder:validation:XValidation:rule="!(has(self.startRevision) && self.startRevision > 0 && (!has(self.mode) || self.mode == 'RequireEmpty'))",message="initialSync.startRevision requires initialSync.mode Overwrite or OverwriteAndPrune (a seeded target is not empty)" +type EtcdMirrorInitialSyncSpec struct { + // Mode governs pre-existing destination keys at genesis. Defaults to + // RequireEmpty (refuse a non-empty destination prefix). + // +optional + // +kubebuilder:validation:Enum=RequireEmpty;Overwrite;OverwriteAndPrune + // +kubebuilder:default=RequireEmpty + Mode EtcdMirrorInitialSyncMode `json:"mode,omitempty"` + + // StartRevision, when > 0, skips the genesis scan entirely and starts + // watching from StartRevision+1. For fidelity-preserving seeds: restore + // the target from a source snapshot (`etcdutl snapshot restore + // --bump-revision --mark-compacted`), then mirror only the delta. + // Requires Mode Overwrite or OverwriteAndPrune (CEL-enforced). + // +optional + // +kubebuilder:validation:Minimum=0 + StartRevision int64 `json:"startRevision,omitempty"` +} + +// EtcdMirrorEndpoint describes how to reach, authenticate to, and scope one +// side (source or target) of a mirror. Both sides need an identical shape +// (address resolution + prefix + TLS + auth), so one type serves both roles, +// the same way BackupDestination is reused verbatim between EtcdBackup and +// EtcdRestore rather than forked into near-duplicate per-role types. +// +// Exactly one of EndpointList or ServiceRef must be set. An empty +// endpointList ([]) is treated as unset, per Kubernetes list conventions — +// so `endpointList: []` alongside a serviceRef is accepted. +// +// Endpoint scheme and the TLS block must agree (CEL-enforced both ways): +// http:// endpoints with a tls block would silently drop TLS at dial time; +// https:// endpoints without one would dial with undeclared system-roots +// TLS. The agent derives the dial scheme from the presence of the tls block, +// so the declared contract is true by construction. +// +// +kubebuilder:validation:XValidation:rule="(has(self.endpointList) && size(self.endpointList) > 0) != has(self.serviceRef)",message="exactly one of endpointList or serviceRef must be set" +// +kubebuilder:validation:XValidation:rule="!(has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, e.startsWith('http://')))",message="http:// endpoints conflict with a tls block: use https:// endpoints or remove tls" +// +kubebuilder:validation:XValidation:rule="!(!has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, e.startsWith('https://')))",message="https:// endpoints require a tls block (an empty tls block selects server-auth TLS against system trust roots)" +type EtcdMirrorEndpoint struct { + // EndpointList is a raw set of etcd client-URL host:port (or + // scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". + // This is the ONLY supported mechanism for a cluster external to this + // Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB); + // there is deliberately no tunnel/port-forward mode — terminate any + // tunnel upstream and hand EtcdMirror the resulting stable endpoint(s). + // + // Prefer listing per-member endpoints over a single load-balancer VIP: a + // TCP-health-checked VIP cannot see etcd quorum, and the client's own + // balancer handles per-member failover. + // + // IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN) + // to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a + // hostname present as a DNS SAN on the certificate, or ensure the + // certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch; + // do not reach for InsecureSkipVerify. + // +optional + EndpointList []string `json:"endpointList,omitempty"` + + // ServiceRef points at a Kubernetes Service in this cluster whose DNS + // name resolves the etcd client endpoint(s), for the same-cluster case. + // Namespace defaults to the EtcdMirror's own namespace when empty. + // +optional + ServiceRef *EtcdMirrorServiceRef `json:"serviceRef,omitempty"` + + // Prefix is the etcd key prefix on THIS side. On Source, only keys under + // this prefix are synced; empty means the whole keyspace. On Target, this + // is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's + // rewrite formula). Immutable after creation. + // +optional + Prefix string `json:"prefix,omitempty"` + + // TLS configures the agent's client TLS for THIS side. Nil means the + // agent dials this side in cleartext (and https:// endpoints are + // CEL-rejected). An empty block means server-auth TLS verified against + // the system trust roots. + // +optional + TLS *EtcdMirrorTLS `json:"tls,omitempty"` + + // Auth configures etcd username/password (RBAC) auth for THIS side. + // +optional + Auth *EtcdMirrorAuth `json:"auth,omitempty"` +} + +// EtcdMirrorServiceRef points at a Service and the client port on it to dial. +type EtcdMirrorServiceRef struct { + // Name is the Service name. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Namespace defaults to the EtcdMirror's own namespace when empty. + // +optional + Namespace string `json:"namespace,omitempty"` + + // Port is the Service port name or number exposing etcd's client API. + // Defaults to "client" when empty, matching PodMonitorSpec.Port's + // convention elsewhere in this API group. + // +optional + Port string `json:"port,omitempty"` +} + +// EtcdMirrorTLS configures the mirror agent's client TLS for one side of a +// mirror. Plain secretRef, not a reuse of EtcdClusterTLS/TLSSurface — the +// agent is always a client to clusters it does not own, so issuer-selection +// machinery doesn't apply. +// +// ROTATION CONTRACT: the agent re-reads TLS material from the mounted Secret +// on every handshake (transport.TLSInfo file paths, not a one-shot +// tls.Config); certificate rotation requires no pod restart. +// +// +kubebuilder:validation:XValidation:rule="!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk",message="insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk to also be true" +// +kubebuilder:validation:XValidation:rule="!has(self.secretRef) || (has(self.secretRef.name) && size(self.secretRef.name) > 0)",message="secretRef.name must be non-empty when secretRef is set" +type EtcdMirrorTLS struct { + // SecretRef names a Secret (in the EtcdMirror's namespace) holding this + // side's TLS material in the standard kubernetes.io/tls-compatible shape: + // - ca.crt: PEM CA bundle used to verify the peer's server certificate + // (unless CABundleRef overrides it, or InsecureSkipVerify is true). + // - tls.crt / tls.key: PEM client certificate + key, for mTLS. + // Optional — omit both for server-auth-only TLS. + // Nil means no client identity and verification against the system trust + // roots (the etcdctl default). Note a source running with + // --client-cert-auth (the RKE1 default) rejects certless clients at the + // handshake regardless of etcd RBAC auth; server-auth-only + Auth is not + // viable against such a source. + // +optional + SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"` + + // CABundleRef optionally sources the trust anchors from a separate + // Secret or ConfigMap key, decoupling trust from the identity Secret + // (Gateway API caCertificateRefs precedent). Takes precedence over + // SecretRef's ca.crt. + // +optional + CABundleRef *EtcdMirrorCABundleRef `json:"caBundleRef,omitempty"` + + // InsecureSkipVerify disables server certificate verification. Strongly + // discouraged, especially for a source reached over the public internet. + // Requires InsecureSkipVerifyAcknowledgeRisk to also be set true + // (CEL-enforced). The controller additionally emits a standing Warning + // event whenever this is true. + // +optional + // +kubebuilder:default=false + InsecureSkipVerify bool `json:"insecureSkipVerify,omitempty"` + + // InsecureSkipVerifyAcknowledgeRisk must independently be set true + // whenever InsecureSkipVerify is true (CEL-enforced companion field). Its + // only purpose is to require a deliberate, separate, reviewable line in + // the manifest diff before disabling TLS verification. + // +optional + // +kubebuilder:default=false + InsecureSkipVerifyAcknowledgeRisk bool `json:"insecureSkipVerifyAcknowledgeRisk,omitempty"` + + // ServerName overrides the TLS ServerName (SNI) used for verification, + // for cases where the dialed address doesn't match a SAN on the + // certificate (e.g. dialing an NLB IP directly). Applies to EVERY + // endpoint in the list, so mixing endpoints with different certificates + // behind one ServerName will fail verification on the mismatched ones. + // +optional + ServerName string `json:"serverName,omitempty"` +} + +// EtcdMirrorCABundleRef points at one key of a Secret or ConfigMap holding a +// PEM CA bundle. +type EtcdMirrorCABundleRef struct { + // Kind is Secret or ConfigMap. Defaults to ConfigMap. + // +optional + // +kubebuilder:validation:Enum=Secret;ConfigMap + // +kubebuilder:default=ConfigMap + Kind string `json:"kind,omitempty"` + + // Name of the Secret or ConfigMap, in the EtcdMirror's namespace. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Key within the object. Defaults to "ca.crt". + // +optional + Key string `json:"key,omitempty"` +} + +// EtcdMirrorAuth configures etcd RBAC username/password auth for one side. +// +// +kubebuilder:validation:XValidation:rule="has(self.secretRef.name) && size(self.secretRef.name) > 0",message="secretRef.name is required" +type EtcdMirrorAuth struct { + // SecretRef names a Secret holding "username" and "password" keys. If + // this whole Auth block is nil, the agent does not call etcd's + // Authenticate() at all; the pinned v3 client transparently re-auths on + // token expiry. PRECEDENCE: when both a client certificate and Auth are + // supplied, etcd uses the token identity, not the certificate CN — the + // Auth user must hold the range-scoped role. + SecretRef corev1.LocalObjectReference `json:"secretRef"` +} + +// EtcdMirrorSyncSpec tunes the mirror's runtime sync behavior. +// +// KEY REWRITE — one formula, no other composition: +// +// key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) +// +// (anchored strip-and-reprefix; never a substring replace). +// +// BATCHING INVARIANT: target Txns flush ONLY at source-revision boundaries — +// a source revision's events are never split across Txns, whole revisions +// are coalesced up to the MaxTxnOps/TxnFlushBytes watermarks, and one op +// slot in MaxTxnOps is always reserved for the checkpoint write that rides +// in the same Txn. A single source revision larger than MaxTxnOps is applied +// as one oversized Txn (provision the target's --max-txn-ops accordingly) +// with the checkpoint held until it lands. +// +// RETENTION PREREQUISITE: the source's compaction retention window must +// exceed the worst-case initial scan + throttled drain time, approximately +// sourceKeyCount / min(effective scan rate, MaxOpsPerSecond). If it does +// not, genesis (and every forced resync) loses the race with compaction and +// the mirror livelocks (surfaced via the resync-loop detector). +type EtcdMirrorSyncSpec struct { + // DestPrefix is the middle term of the rewrite formula above. Default "" + // means the source prefix is stripped and key remainders land directly + // under target.prefix. Immutable after creation. + // +optional + DestPrefix string `json:"destPrefix,omitempty"` + + // ExcludePrefixes lists source key prefixes (full source-side keys, e.g. + // "/registry/events/") skipped entirely: not scanned, not watched, not + // counted, not pruned. Use to drop high-churn low-value ranges and cut + // WAN cost, or to skip lease-backed ranges that don't survive mirroring. + // Nested/duplicate entries are normalized by the agent (an entry covered + // by another is dropped). Immutable after creation (it defines the + // mirrored range): removing an exclusion would require a backfill scan + // the checkpoint-resume path never runs, and adding one would strand + // already-mirrored keys as permanent orphans — change it via + // delete-and-recreate with an appropriate initialSync.mode instead. + // +optional + // +kubebuilder:validation:MaxItems=64 + ExcludePrefixes []string `json:"excludePrefixes,omitempty"` + + // MaxTxnOps bounds how many operations the agent batches into a single + // target Txn, including the reserved checkpoint-write slot. Must not + // exceed the target's --max-txn-ops (etcd default 128). Defaults to 128. + // +optional + // +kubebuilder:validation:Minimum=2 + MaxTxnOps int32 `json:"maxTxnOps,omitempty"` + + // TxnFlushBytes is the byte watermark at which a batch is flushed (at the + // next source-revision boundary). Keep well under etcd's request size + // limits: a Txn over ~1.5MiB is rejected by the server and one over 2MiB + // by the client send cap — both classified permanent errors, not + // throttling. Defaults to 1Mi. + // +optional + TxnFlushBytes *resource.Quantity `json:"txnFlushBytes,omitempty"` + + // PageKeyLimit bounds keys per source scan page during InitialSync and + // reconciliation. The scan is pull-based, one page in flight — no + // read-ahead — so this and PageBytes bound agent memory. Defaults to 512. + // +optional + // +kubebuilder:validation:Minimum=1 + PageKeyLimit int32 `json:"pageKeyLimit,omitempty"` + + // PageBytes bounds bytes per source scan page. Defaults to 1Mi. + // +optional + PageBytes *resource.Quantity `json:"pageBytes,omitempty"` + + // WatchBufferBytes bounds the memory used to buffer watch events + // observed from R0 while the genesis scan runs (the reflector replay + // buffer). On overflow the agent cancels the source watch and restarts + // the scan from a fresh R0 (see the InitialSyncCompactionRaced event) — + // a bounded retry instead of unbounded growth when source churn outruns + // scan+apply throughput. Defaults to 16Mi (must stay in lockstep with + // pkg/mirroragent's DefaultWatchBufferBytes). + // +optional + WatchBufferBytes *resource.Quantity `json:"watchBufferBytes,omitempty"` + + // MaxOpsPerSecond rate-limits the agent's target write rate (a token + // bucket over puts+deletes/sec), applied to both the genesis scan and + // watch-driven applies. Zero (default) means unlimited. Mind the + // retention prerequisite above when throttling. + // +optional + // +kubebuilder:validation:Minimum=0 + MaxOpsPerSecond int32 `json:"maxOpsPerSecond,omitempty"` + + // RequestTimeout is the per-RPC context deadline applied to every unary + // call on both sides (watches excluded — they are long-lived by design + // and covered by progress-notification liveness instead). Without it a + // blackholed call through an NLB never errors and backoff never engages. + // Defaults to 30s. + // +optional + RequestTimeout *metav1.Duration `json:"requestTimeout,omitempty"` + + // DialTimeout bounds establishing the initial client connection to each + // side. Defaults to 10s. + // +optional + DialTimeout *metav1.Duration `json:"dialTimeout,omitempty"` + + // ReconnectBackoff bounds the retry/backoff loop wrapping connection-class + // errors. Throttling-class errors (target rate rejection) use a more + // conservative curve derived from the same bounds; quota exhaustion + // (TargetQuotaExhausted) and permanent errors are never retried through + // this loop. Defaults to exponential backoff from 1s to 30s. + // +optional + ReconnectBackoff *EtcdMirrorBackoffSpec `json:"reconnectBackoff,omitempty"` +} + +type EtcdMirrorBackoffSpec struct { + // +optional + InitialDelay *metav1.Duration `json:"initialDelay,omitempty"` + // +optional + MaxDelay *metav1.Duration `json:"maxDelay,omitempty"` +} + +// EtcdMirrorCheckpointSpec configures the reserved checkpoint/fence key the +// agent maintains IN THE TARGET etcd. The checkpoint (the source-revision +// watermark plus {linkUID, epoch, role}) is written in the SAME Txn as every +// applied batch and fenced with a mod-revision compare on EVERY write path +// (applies, reconciliation repairs, prune deletes), so two agents can never +// interleave writes and a straggler apply after cutover fails loudly. The +// key is excluded by exact match from scans, counts, prune passes, and the +// RequireEmpty check; the target RBAC grant must cover it; CR deletion +// removes it via a delete-one-key finalizer. +type EtcdMirrorCheckpointSpec struct { + // Key overrides the reserved checkpoint key. Defaults to the effective + // destination prefix + "\x00etcdmirror-checkpoint" — the \x00 byte after + // the prefix cannot collide with any real key under it. MUST live under + // the effective destination prefix (target.prefix + sync.destPrefix, + // CEL-enforced): the range-scoped target credential covers it, and the + // exact-match exclusion from scans/counts/prune only works inside the + // mirrored range. Immutable after creation. + // +optional + Key string `json:"key,omitempty"` +} + +// EtcdMirrorReconciliationSpec configures the periodic full reconciliation +// pass. The same engine also runs unconditionally (regardless of Enabled or +// DeleteOrphans) as the post-forced-resync mark-and-sweep, the +// OverwriteAndPrune genesis pass, and the Drain verification pass. +type EtcdMirrorReconciliationSpec struct { + // Enabled toggles the PERIODIC pass. Defaults to false: it is a full + // diff of the prefix contents on both sides (O(keyspace)), so it is + // opt-in. + // +optional + Enabled bool `json:"enabled,omitempty"` + + // Interval between periodic passes. Defaults to 1h when Enabled. + // +optional + Interval *metav1.Duration `json:"interval,omitempty"` + + // DeleteOrphans, when true, allows the PERIODIC pass to delete target + // keys under the destination prefix that have no corresponding source + // key. Defaults to false. (Forced-resync sweeps and OverwriteAndPrune + // always delete orphans; this knob only governs the periodic pass.) + // +optional + DeleteOrphans bool `json:"deleteOrphans,omitempty"` +} + +// EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. +// Unlike BackupPhase/RestorePhase, most phases here are NOT terminal — a +// healthy mirror spends its life in Syncing; there is no "Completed" state. +type EtcdMirrorPhase string + +const ( + // EtcdMirrorPhasePending means the EtcdMirror has been accepted but the + // agent workload has not been created yet. + EtcdMirrorPhasePending EtcdMirrorPhase = "Pending" + // EtcdMirrorPhaseConnecting means the agent pod is running, establishing + // client connections to both sides and probing versions/cluster IDs. + EtcdMirrorPhaseConnecting EtcdMirrorPhase = "Connecting" + // EtcdMirrorPhaseInitialSync means the agent is running the genesis scan: + // an UNPINNED chunked scan with the watch already open from the revision + // observed before the scan started, buffered events replayed over the + // scanned base (reflector pattern). Because pages read at the current + // revision, mid-scan compaction cannot fail the scan. Also entered during + // a forced resync (then with condition Compacted=True/Reason=ForcedResync). + EtcdMirrorPhaseInitialSync EtcdMirrorPhase = "InitialSync" + // EtcdMirrorPhaseSyncing is the steady state: watching and applying live + // changes, watermark advancing via progress notifications. + EtcdMirrorPhaseSyncing EtcdMirrorPhase = "Syncing" + // EtcdMirrorPhaseDegraded means the agent is in a retry/backoff loop + // (connection or throttling class) and is expected to self-heal. Forced + // resyncs are NOT Degraded; they report as InitialSync + Compacted=True. + EtcdMirrorPhaseDegraded EtcdMirrorPhase = "Degraded" + // EtcdMirrorPhasePaused means spec.paused is true; the agent Deployment + // is scaled to zero. The checkpoint is retained in the target. + EtcdMirrorPhasePaused EtcdMirrorPhase = "Paused" + // EtcdMirrorPhaseFailed means a terminal, non-recoverable error requiring + // operator intervention: EmptyTargetViolation at genesis, source below + // the 3.4 version floor (UnsupportedVersion), a permanent-class write + // error (oversized revision vs target limits), malformed cert material, + // or unresolvable spec misconfiguration. + EtcdMirrorPhaseFailed EtcdMirrorPhase = "Failed" +) + +// Condition types reported on EtcdMirror status. +// +// PAGING ALGEBRA (for alert authors): page on Available=False sustained for +// your tolerance window UNLESS Compacted=True AND progress fields are +// advancing (a forced resync healing itself); TargetQuotaExhausted and +// ResyncLoopDetected page immediately — neither self-heals. +const ( + // EtcdMirrorConditionAvailable is True only when the agent pod is + // running, in the Syncing phase, AND the checkpoint watermark is + // advancing (via applies or watch progress notifications) within the + // staleness threshold. Watermark-derived, not apply-derived: an idle + // prefix on a live watch stays Available; a wedged loop that stops + // confirming progress does not. + EtcdMirrorConditionAvailable = "Available" + // EtcdMirrorConditionSourceReachable is True when the agent's last + // attempt to reach Source succeeded. Split from TargetReachable because + // in the primary use case (source over the public internet) source + // reachability is the most likely persistent failure mode. + EtcdMirrorConditionSourceReachable = "SourceReachable" + // EtcdMirrorConditionTargetReachable is the target-side analogue. + EtcdMirrorConditionTargetReachable = "TargetReachable" + // EtcdMirrorConditionTargetThrottled is True while the agent is backing + // off from throttling-class errors on Target (rate rejection / + // ErrTooManyRequests). Distinct from TargetReachable=False ("up but + // rejecting my write rate" is actionable differently) and from + // TargetQuotaExhausted (backoff cannot heal a full quota). + EtcdMirrorConditionTargetThrottled = "TargetThrottled" + // EtcdMirrorConditionTargetQuotaExhausted is True when a target write + // failed with etcd's NOSPACE (rpctypes.ErrNoSpace). Permanent until an + // operator compacts/defrags/disarms the target; the agent stops writing + // rather than burning backoff against a full quota. Detected from the + // typed write-path error, never AlarmList (which needs root). + EtcdMirrorConditionTargetQuotaExhausted = "TargetQuotaExhausted" + // EtcdMirrorConditionInitialSyncComplete is True once the genesis scan + // has completed against the currently-checkpointed cluster identities. + // Durable across forced resyncs (Compacted covers those); reset to False + // when the checkpoint is invalidated by a mismatch of EITHER bound + // cluster ID (source or target), which also re-arms the RequireEmpty + // check. + EtcdMirrorConditionInitialSyncComplete = "InitialSyncComplete" + // EtcdMirrorConditionCompacted is True (Reason ForcedResync) while the + // agent heals from source compaction outrunning the watch (restart or + // pause longer than retention; a WatchResponse with CompactRevision != + // 0). Mid-scan compaction is NOT in this class — the unpinned scan is + // immune by construction. Every forced resync ends with a mandatory + // mark-and-sweep prune. Reverts to False when steady-state resumes. + EtcdMirrorConditionCompacted = "Compacted" + // EtcdMirrorConditionResyncLoopDetected is True when N consecutive forced + // resyncs completed without reaching steady state — the livelock + // signature of source retention < scan+drain time. Does not self-heal: + // raise retention, raise MaxOpsPerSecond, or shrink the prefix. + EtcdMirrorConditionResyncLoopDetected = "ResyncLoopDetected" + // EtcdMirrorConditionReplicationLagExceeded is True when the checkpoint + // watermark has stayed more than a threshold behind the source's + // current revision for a sustained duration. Both terms come from the + // same watch/progress machinery (never from comparing the two live + // status fields, which snapshot at different instants). Threshold and + // duration are agent-internal constants in v1. + EtcdMirrorConditionReplicationLagExceeded = "ReplicationLagExceeded" + // EtcdMirrorConditionDriftDetected is True when the last reconciliation + // pass found orphaned/missing keys. Carries counts in Message. Sticky + // until the next pass reports clean. + EtcdMirrorConditionDriftDetected = "DriftDetected" + // EtcdMirrorConditionEmptyTargetViolation is True (terminal, Phase -> + // Failed) when initialSync.mode is RequireEmpty and the destination + // prefix was non-empty at genesis. The condition Message embeds the + // exact `etcdctl del` command for the offending range. The reserved + // checkpoint key is excluded by exact match. + EtcdMirrorConditionEmptyTargetViolation = "EmptyTargetViolation" + // EtcdMirrorConditionCutoverReady is True when spec.mode is Drain, the + // watermark has reached status.cutover.drainTargetRevision, and the + // verification pass succeeded. From then the fence key's role is + // Primary and any straggler mirror apply fails its compare. Gate + // promotion on `kubectl wait --for=condition=CutoverReady`. + EtcdMirrorConditionCutoverReady = "CutoverReady" + // EtcdMirrorConditionInvariantsHeld is the composed verification verdict: + // True when ReplicationLagExceeded is False, the per-side key counts + // (status.sourceKeyCount/targetKeyCount, reserved key excluded) are + // equal, DriftDetected is False, and the pass that produced the counts + // is fresh. Freshness: with the periodic reconciliation pass enabled, + // within 2x spec.reconciliation.interval; with it disabled the counts + // only refresh on mandatory passes (forced-resync sweeps, + // OverwriteAndPrune genesis, drain verification), so the condition is + // Unknown/stale-reasoned once the last such pass ages out — enable + // reconciliation to make this a continuous signal. It means + // "verification invariants hold", never "safe to cut over" — cutover is + // gated on CutoverReady, which additionally requires spec.mode Drain and + // a reached drainTargetRevision. + EtcdMirrorConditionInvariantsHeld = "InvariantsHeld" + // EtcdMirrorConditionLearnerEndpoint is True when the maintenance + // Status() probe reports IsLearner=true for a configured endpoint. + // Non-blocking (learners self-heal and the balancer routes around them), + // but a learner pick can serve stale reads mid-catch-up. + EtcdMirrorConditionLearnerEndpoint = "LearnerEndpoint" + // EtcdMirrorConditionPrefixConflict is controller-set: another + // EtcdMirror targets an overlapping effective destination range on the + // same target cluster. Declared now so the name is API contract before + // any setter exists. Independent of the controller check, the agent + // itself refuses (permanently, Phase=Failed) to prune a reserved fence + // key owned by a different link, so an undetected overlap stops loudly + // instead of destroying the sibling mirror's fence and data. + EtcdMirrorConditionPrefixConflict = "PrefixConflict" + // EtcdMirrorConditionDirectionConflict is controller-set: this + // mirror's bound source/target cluster IDs are the inverse of another + // EtcdMirror's — two CRs forming a two-way loop, caught by cluster-ID + // binding even when a respelled endpoint string would fool a spec + // comparison. + EtcdMirrorConditionDirectionConflict = "DirectionConflict" +) + +// Condition reasons and Event reasons that are part of the API contract. +const ( + // EtcdMirrorReasonForcedResync is the Compacted condition's reason while + // a forced resync is in flight. + EtcdMirrorReasonForcedResync = "ForcedResync" + // EtcdMirrorReasonUnsupportedVersion is the Failed-phase reason when the + // source is below the 3.4 floor. + EtcdMirrorReasonUnsupportedVersion = "UnsupportedVersion" + + // EtcdMirrorReasonCompacted / EtcdMirrorReasonClusterIDMismatch name WHY + // a forced resync was required (mirroring pkg/mirroragent's ResyncReason + // values). Surfaces: the Compacted condition's MESSAGE and the + // forced-resync events' messages (and, later, the forced-resync metric's + // reason label). The Compacted condition's Reason is always ForcedResync + // while a resync is in flight — alert on that, not on these — and + // forcedResyncCount is a plain counter with no per-reason breakdown. The + // ClusterIDMismatch message additionally names the mismatched side. + EtcdMirrorReasonCompacted = "Compacted" + EtcdMirrorReasonClusterIDMismatch = "ClusterIDMismatch" + // EtcdMirrorReasonCheckpointInvalid is a Failed-phase reason: the stored + // checkpoint was corrupt or of an unknown wire version. PERMANENT — the + // agent fails closed and never auto-resyncs; the operator must inspect + // and delete the reserved key to recover. + EtcdMirrorReasonCheckpointInvalid = "CheckpointInvalid" + + // EtcdMirrorEventForcedResyncStarted / Completed bracket every forced + // resync (Warning/Normal respectively). + EtcdMirrorEventForcedResyncStarted = "ForcedResyncStarted" + EtcdMirrorEventForcedResyncCompleted = "ForcedResyncCompleted" + // EtcdMirrorEventCheckpointInvalidated is emitted when the checkpoint is + // discarded because a bound cluster ID (source or target) no longer + // matches, forcing genesis and re-arming the RequireEmpty check. + EtcdMirrorEventCheckpointInvalidated = "CheckpointInvalidated" + // EtcdMirrorEventInitialSyncCompactionRaced marks an InitialSync attempt + // aborted and restarted from a fresh R0. Two causes, named in the event + // message: WatchBufferOverflow (the replay buffer exceeded + // sync.watchBufferBytes before the base scan completed — a memory-bound + // retry, NOT a compaction race) and WatchCompactedMidScan (a watch + // reconnect landed below the source compact revision — the rare genuine + // race). One event name so operators can alert on scan restarts; the + // cause string prevents conflating "buffer too small for churn" with + // "compaction won a race the design eliminates". Repeated occurrences + // count toward ResyncLoopDetected. + EtcdMirrorEventInitialSyncCompactionRaced = "InitialSyncCompactionRaced" +) + +// EtcdMirrorStatus defines the observed state of an EtcdMirror. Progress +// fields are synced periodically (not per-op), so they are a coarse, +// point-in-time mirror of the agent's authoritative checkpoint in the target +// etcd, useful for kubectl/dashboards, not an audit log. +// +// CUTOVER GATE CONTRACT: never compute "caught up" by comparing +// SourceRevision to LastAppliedRevision — they snapshot at different +// instants, and SourceRevision advances on out-of-prefix writes (revisions +// are cluster-global). The manual gate is: quiesce source writers, read the +// source's current revision R yourself (`etcdctl endpoint status`), then +// poll until LastAppliedRevision >= R. The in-CR gate is spec.mode=Drain + +// the CutoverReady condition. Relax target RBAC only after the mirror is +// paused or deleted. +type EtcdMirrorStatus struct { + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // +optional + Phase EtcdMirrorPhase `json:"phase,omitempty"` + + // LastAppliedRevision is the checkpoint watermark: the source revision + // through which the target is caught up, advanced by applies AND by + // watch progress notifications on idle prefixes. The fenced checkpoint + // key in the target etcd is the authoritative copy; this mirrors it. + // +optional + LastAppliedRevision int64 `json:"lastAppliedRevision,omitempty"` + + // SourceRevision is the source cluster's revision as of the last status + // sync. Cluster-global: it advances on writes outside the mirrored + // prefix, so SourceRevision - LastAppliedRevision OVERSTATES lag for + // prefix-scoped mirrors. See the cutover gate contract above. + // +optional + SourceRevision int64 `json:"sourceRevision,omitempty"` + + // SourceClusterID and TargetClusterID are the cluster IDs both bound + // into the checkpoint. Either changing across reconciles is the visible + // symptom of an endpoint now pointing at a different cluster than the + // checkpoint was taken against (CheckpointInvalidated event, forced + // genesis, RequireEmpty re-armed). + // +optional + SourceClusterID string `json:"sourceClusterID,omitempty"` + // +optional + TargetClusterID string `json:"targetClusterID,omitempty"` + + // SourceVersion and TargetVersion are the etcd server versions from the + // maintenance Status() probe at connect. + // +optional + SourceVersion string `json:"sourceVersion,omitempty"` + // +optional + TargetVersion string `json:"targetVersion,omitempty"` + + // InitialSyncKeyCount is the number of keys applied so far by the + // current/last genesis scan. Live-updating during InitialSync (each + // status sync), so InitialSyncKeyCount/InitialSyncTotalKeyCount is a + // progress fraction. + // +optional + InitialSyncKeyCount int64 `json:"initialSyncKeyCount,omitempty"` + // InitialSyncTotalKeyCount is the denominator: the source-side key count + // under the prefix observed at scan start (first page's RangeResponse + // count). + // +optional + InitialSyncTotalKeyCount int64 `json:"initialSyncTotalKeyCount,omitempty"` + // +optional + InitialSyncStartTime *metav1.Time `json:"initialSyncStartTime,omitempty"` + // +optional + InitialSyncCompletionTime *metav1.Time `json:"initialSyncCompletionTime,omitempty"` + + // LeaseBackedKeyCount is the number of mirrored keys whose source copy is + // lease-backed (kv.Lease != 0). Mirrored copies are NOT lease-backed — + // leases are stripped (see Fidelity Caveats in docs/etcdmirror.md) — so a + // nonzero count means the cutover runbook's purge/re-lease step applies. + // +optional + LeaseBackedKeyCount int64 `json:"leaseBackedKeyCount,omitempty"` + + // ForcedResyncCount counts forced resyncs (compaction outran the watch, + // checkpoint invalidated by a cluster-ID mismatch, or checkpoint + // corrupt/unknown-version). Monotonic, never reset. + // +optional + ForcedResyncCount int32 `json:"forcedResyncCount,omitempty"` + + // LastReconciliationTime and LastReconciliationDrift record the most + // recent reconciliation pass (periodic or mandatory). + // +optional + LastReconciliationTime *metav1.Time `json:"lastReconciliationTime,omitempty"` + // +optional + LastReconciliationDrift *EtcdMirrorDriftInfo `json:"lastReconciliationDrift,omitempty"` + + // SourceKeyCount and TargetKeyCount are the per-side key counts from the + // most recent diff/verification pass (reserved checkpoint key and + // excluded prefixes not counted; drain-verification source reads pinned + // at the drained revision with a compacted-fallback re-read). Populated + // by every pass that runs regardless of spec.reconciliation.enabled — + // the mandatory mark-and-sweep after any forced resync, the + // OverwriteAndPrune genesis pass, and the drain verification — plus the + // periodic pass when it is enabled. NOT refreshed on every status sync: + // a healthy RequireEmpty mirror that never forces a resync only gets + // counts from an enabled periodic pass. This is the equality signal + // InvariantsHeld reads; status.cutover's copies remain the frozen + // drain-time snapshot. + // +optional + SourceKeyCount int64 `json:"sourceKeyCount,omitempty"` + // +optional + TargetKeyCount int64 `json:"targetKeyCount,omitempty"` + + // LastStatusSyncTime is when status was last refreshed from the agent, so + // staleness of everything above is directly observable. + // +optional + LastStatusSyncTime *metav1.Time `json:"lastStatusSyncTime,omitempty"` + + // LastProgressTime is when the watermark last advanced (apply or watch + // progress notification). Distinct from LastStatusSyncTime: status can + // keep syncing from a wedged loop; this field is what Available and + // ReplicationLagExceeded are derived from. + // +optional + LastProgressTime *metav1.Time `json:"lastProgressTime,omitempty"` + + // Cutover is populated while spec.mode is Drain. + // +optional + Cutover *EtcdMirrorCutoverStatus `json:"cutover,omitempty"` + + // AgentPod is the name of the current agent pod (the sole pod of the + // size-1 Deployment), for convenient kubectl logs/exec. + // +optional + AgentPod string `json:"agentPod,omitempty"` + + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// EtcdMirrorCutoverStatus tracks a Drain-mode cutover. +type EtcdMirrorCutoverStatus struct { + // DrainTargetRevision is the source revision observed when Drain was + // requested — the revision the watermark must reach. + // +optional + DrainTargetRevision int64 `json:"drainTargetRevision,omitempty"` + // DrainedRevision is the watermark at which the drain completed. + // +optional + DrainedRevision int64 `json:"drainedRevision,omitempty"` + // VerifiedTime is when the post-drain verification pass succeeded. + // +optional + VerifiedTime *metav1.Time `json:"verifiedTime,omitempty"` + // SourceKeyCount and TargetKeyCount are the per-side key counts from the + // verification pass (source read pinned at the drained revision; + // reserved checkpoint key excluded). + // +optional + SourceKeyCount int64 `json:"sourceKeyCount,omitempty"` + // +optional + TargetKeyCount int64 `json:"targetKeyCount,omitempty"` + // LeasedKeyCount is LeaseBackedKeyCount frozen at drain completion, for + // the runbook's purge/re-lease step. + // +optional + LeasedKeyCount int64 `json:"leasedKeyCount,omitempty"` +} + +type EtcdMirrorDriftInfo struct { + // MissingKeys were present on the source but absent on the target. + MissingKeys int64 `json:"missingKeys,omitempty"` + // DivergentKeys were present on both sides with different values — + // distinct from MissingKeys so "a resync dropped keys" is never + // conflated with "a blind window went stale". + DivergentKeys int64 `json:"divergentKeys,omitempty"` + // OrphanKeys were present on the target with no source counterpart. + OrphanKeys int64 `json:"orphanKeys,omitempty"` + // Repaired is true when the pass wrote fixes rather than only reporting. + Repaired bool `json:"repaired,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Available",type=string,JSONPath=`.status.conditions[?(@.type=="Available")].status` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Revision",type=integer,JSONPath=`.status.lastAppliedRevision` +// +kubebuilder:printcolumn:name="Source-Rev",type=integer,JSONPath=`.status.sourceRevision` +// +kubebuilder:printcolumn:name="Last-Progress",type=date,JSONPath=`.status.lastProgressTime` +// +kubebuilder:printcolumn:name="Source",type=string,priority=1,JSONPath=`.spec.source.prefix` +// +kubebuilder:printcolumn:name="Target",type=string,priority=1,JSONPath=`.spec.target.prefix` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// EtcdMirror is the Schema for the etcdmirrors API. It describes a +// continuous, one-way key-range sync from a source etcd cluster to a target +// etcd cluster, run as a single supervised stateless pod (a size-1 +// Deployment; progress lives in a fenced checkpoint key in the target etcd, +// not on a volume). +// +// It is a byte-copy of keys and values, not a replica: revisions, versions, +// and create/mod ordering are target-assigned, and leases are stripped. See +// Fidelity Caveats in docs/etcdmirror.md before depending on anything but +// key/value content. +// +// Two-way sync: never — etcd revisions are cluster-local and there is no +// per-key provenance channel, so bidirectional sync is structurally +// inexpressible. Reversal for cutover/failback: yes — delete the CR and +// create a new one with swapped endpoints, initialSync.mode +// OverwriteAndPrune, only after the forward mirror reported CutoverReady. +// See docs/etcdmirror.md. +type EtcdMirror struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec EtcdMirrorSpec `json:"spec,omitempty"` + Status EtcdMirrorStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// EtcdMirrorList contains a list of EtcdMirror. +type EtcdMirrorList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EtcdMirror `json:"items"` +} diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go index 9a178715..54ef002f 100644 --- a/api/v1alpha1/groupversion_info.go +++ b/api/v1alpha1/groupversion_info.go @@ -40,6 +40,8 @@ func addKnownTypes(s *runtime.Scheme) error { s.AddKnownTypes(GroupVersion, &EtcdCluster{}, &EtcdClusterList{}, + &EtcdMirror{}, + &EtcdMirrorList{}, ) metav1.AddToGroupVersion(s, GroupVersion) return nil diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9db6ca12..0f6a17af 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" netx "net" ) @@ -200,6 +200,425 @@ func (in *EtcdClusterStatus) DeepCopy() *EtcdClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirror) DeepCopyInto(out *EtcdMirror) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirror. +func (in *EtcdMirror) DeepCopy() *EtcdMirror { + if in == nil { + return nil + } + out := new(EtcdMirror) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdMirror) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorAuth) DeepCopyInto(out *EtcdMirrorAuth) { + *out = *in + out.SecretRef = in.SecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorAuth. +func (in *EtcdMirrorAuth) DeepCopy() *EtcdMirrorAuth { + if in == nil { + return nil + } + out := new(EtcdMirrorAuth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorBackoffSpec) DeepCopyInto(out *EtcdMirrorBackoffSpec) { + *out = *in + if in.InitialDelay != nil { + in, out := &in.InitialDelay, &out.InitialDelay + *out = new(metav1.Duration) + **out = **in + } + if in.MaxDelay != nil { + in, out := &in.MaxDelay, &out.MaxDelay + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorBackoffSpec. +func (in *EtcdMirrorBackoffSpec) DeepCopy() *EtcdMirrorBackoffSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorBackoffSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorCABundleRef) DeepCopyInto(out *EtcdMirrorCABundleRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorCABundleRef. +func (in *EtcdMirrorCABundleRef) DeepCopy() *EtcdMirrorCABundleRef { + if in == nil { + return nil + } + out := new(EtcdMirrorCABundleRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorCheckpointSpec) DeepCopyInto(out *EtcdMirrorCheckpointSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorCheckpointSpec. +func (in *EtcdMirrorCheckpointSpec) DeepCopy() *EtcdMirrorCheckpointSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorCheckpointSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorCutoverStatus) DeepCopyInto(out *EtcdMirrorCutoverStatus) { + *out = *in + if in.VerifiedTime != nil { + in, out := &in.VerifiedTime, &out.VerifiedTime + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorCutoverStatus. +func (in *EtcdMirrorCutoverStatus) DeepCopy() *EtcdMirrorCutoverStatus { + if in == nil { + return nil + } + out := new(EtcdMirrorCutoverStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorDriftInfo) DeepCopyInto(out *EtcdMirrorDriftInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorDriftInfo. +func (in *EtcdMirrorDriftInfo) DeepCopy() *EtcdMirrorDriftInfo { + if in == nil { + return nil + } + out := new(EtcdMirrorDriftInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorEndpoint) DeepCopyInto(out *EtcdMirrorEndpoint) { + *out = *in + if in.EndpointList != nil { + in, out := &in.EndpointList, &out.EndpointList + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ServiceRef != nil { + in, out := &in.ServiceRef, &out.ServiceRef + *out = new(EtcdMirrorServiceRef) + **out = **in + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(EtcdMirrorTLS) + (*in).DeepCopyInto(*out) + } + if in.Auth != nil { + in, out := &in.Auth, &out.Auth + *out = new(EtcdMirrorAuth) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorEndpoint. +func (in *EtcdMirrorEndpoint) DeepCopy() *EtcdMirrorEndpoint { + if in == nil { + return nil + } + out := new(EtcdMirrorEndpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorInitialSyncSpec) DeepCopyInto(out *EtcdMirrorInitialSyncSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorInitialSyncSpec. +func (in *EtcdMirrorInitialSyncSpec) DeepCopy() *EtcdMirrorInitialSyncSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorInitialSyncSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorList) DeepCopyInto(out *EtcdMirrorList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EtcdMirror, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorList. +func (in *EtcdMirrorList) DeepCopy() *EtcdMirrorList { + if in == nil { + return nil + } + out := new(EtcdMirrorList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdMirrorList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorReconciliationSpec) DeepCopyInto(out *EtcdMirrorReconciliationSpec) { + *out = *in + if in.Interval != nil { + in, out := &in.Interval, &out.Interval + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorReconciliationSpec. +func (in *EtcdMirrorReconciliationSpec) DeepCopy() *EtcdMirrorReconciliationSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorReconciliationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorServiceRef) DeepCopyInto(out *EtcdMirrorServiceRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorServiceRef. +func (in *EtcdMirrorServiceRef) DeepCopy() *EtcdMirrorServiceRef { + if in == nil { + return nil + } + out := new(EtcdMirrorServiceRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorSpec) DeepCopyInto(out *EtcdMirrorSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + in.Target.DeepCopyInto(&out.Target) + if in.InitialSync != nil { + in, out := &in.InitialSync, &out.InitialSync + *out = new(EtcdMirrorInitialSyncSpec) + **out = **in + } + in.Sync.DeepCopyInto(&out.Sync) + if in.Checkpoint != nil { + in, out := &in.Checkpoint, &out.Checkpoint + *out = new(EtcdMirrorCheckpointSpec) + **out = **in + } + if in.Reconciliation != nil { + in, out := &in.Reconciliation, &out.Reconciliation + *out = new(EtcdMirrorReconciliationSpec) + (*in).DeepCopyInto(*out) + } + if in.PodTemplate != nil { + in, out := &in.PodTemplate, &out.PodTemplate + *out = new(PodTemplate) + (*in).DeepCopyInto(*out) + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorSpec. +func (in *EtcdMirrorSpec) DeepCopy() *EtcdMirrorSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorStatus) DeepCopyInto(out *EtcdMirrorStatus) { + *out = *in + if in.InitialSyncStartTime != nil { + in, out := &in.InitialSyncStartTime, &out.InitialSyncStartTime + *out = (*in).DeepCopy() + } + if in.InitialSyncCompletionTime != nil { + in, out := &in.InitialSyncCompletionTime, &out.InitialSyncCompletionTime + *out = (*in).DeepCopy() + } + if in.LastReconciliationTime != nil { + in, out := &in.LastReconciliationTime, &out.LastReconciliationTime + *out = (*in).DeepCopy() + } + if in.LastReconciliationDrift != nil { + in, out := &in.LastReconciliationDrift, &out.LastReconciliationDrift + *out = new(EtcdMirrorDriftInfo) + **out = **in + } + if in.LastStatusSyncTime != nil { + in, out := &in.LastStatusSyncTime, &out.LastStatusSyncTime + *out = (*in).DeepCopy() + } + if in.LastProgressTime != nil { + in, out := &in.LastProgressTime, &out.LastProgressTime + *out = (*in).DeepCopy() + } + if in.Cutover != nil { + in, out := &in.Cutover, &out.Cutover + *out = new(EtcdMirrorCutoverStatus) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorStatus. +func (in *EtcdMirrorStatus) DeepCopy() *EtcdMirrorStatus { + if in == nil { + return nil + } + out := new(EtcdMirrorStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorSyncSpec) DeepCopyInto(out *EtcdMirrorSyncSpec) { + *out = *in + if in.ExcludePrefixes != nil { + in, out := &in.ExcludePrefixes, &out.ExcludePrefixes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TxnFlushBytes != nil { + in, out := &in.TxnFlushBytes, &out.TxnFlushBytes + x := (*in).DeepCopy() + *out = &x + } + if in.PageBytes != nil { + in, out := &in.PageBytes, &out.PageBytes + x := (*in).DeepCopy() + *out = &x + } + if in.WatchBufferBytes != nil { + in, out := &in.WatchBufferBytes, &out.WatchBufferBytes + x := (*in).DeepCopy() + *out = &x + } + if in.RequestTimeout != nil { + in, out := &in.RequestTimeout, &out.RequestTimeout + *out = new(metav1.Duration) + **out = **in + } + if in.DialTimeout != nil { + in, out := &in.DialTimeout, &out.DialTimeout + *out = new(metav1.Duration) + **out = **in + } + if in.ReconnectBackoff != nil { + in, out := &in.ReconnectBackoff, &out.ReconnectBackoff + *out = new(EtcdMirrorBackoffSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorSyncSpec. +func (in *EtcdMirrorSyncSpec) DeepCopy() *EtcdMirrorSyncSpec { + if in == nil { + return nil + } + out := new(EtcdMirrorSyncSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdMirrorTLS) DeepCopyInto(out *EtcdMirrorTLS) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.CABundleRef != nil { + in, out := &in.CABundleRef, &out.CABundleRef + *out = new(EtcdMirrorCABundleRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdMirrorTLS. +func (in *EtcdMirrorTLS) DeepCopy() *EtcdMirrorTLS { + if in == nil { + return nil + } + out := new(EtcdMirrorTLS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemberStatus) DeepCopyInto(out *MemberStatus) { *out = *in diff --git a/config/crd/bases/operator.etcd.io_etcdmirrors.yaml b/config/crd/bases/operator.etcd.io_etcdmirrors.yaml new file mode 100644 index 00000000..8e497c1e --- /dev/null +++ b/config/crd/bases/operator.etcd.io_etcdmirrors.yaml @@ -0,0 +1,2053 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: etcdmirrors.operator.etcd.io +spec: + group: operator.etcd.io + names: + kind: EtcdMirror + listKind: EtcdMirrorList + plural: etcdmirrors + singular: etcdmirror + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Available")].status + name: Available + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.lastAppliedRevision + name: Revision + type: integer + - jsonPath: .status.sourceRevision + name: Source-Rev + type: integer + - jsonPath: .status.lastProgressTime + name: Last-Progress + type: date + - jsonPath: .spec.source.prefix + name: Source + priority: 1 + type: string + - jsonPath: .spec.target.prefix + name: Target + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + EtcdMirror is the Schema for the etcdmirrors API. It describes a + continuous, one-way key-range sync from a source etcd cluster to a target + etcd cluster, run as a single supervised stateless pod (a size-1 + Deployment; progress lives in a fenced checkpoint key in the target etcd, + not on a volume). + + It is a byte-copy of keys and values, not a replica: revisions, versions, + and create/mod ordering are target-assigned, and leases are stripped. See + Fidelity Caveats in docs/etcdmirror.md before depending on anything but + key/value content. + + Two-way sync: never — etcd revisions are cluster-local and there is no + per-key provenance channel, so bidirectional sync is structurally + inexpressible. Reversal for cutover/failback: yes — delete the CR and + create a new one with swapped endpoints, initialSync.mode + OverwriteAndPrune, only after the forward mirror reported CutoverReady. + See docs/etcdmirror.md. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EtcdMirrorSpec defines the desired state of an EtcdMirror. + + Range-defining and rewrite fields are immutable (CEL transition rules + below): source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes + and checkpoint.key. Changing what range is mirrored, or where it lands, + mid-life silently diverges: a restarted agent resumes from its checkpoint + without a scan, so removing an exclusion never backfills pre-existing keys + and adding one strands already-mirrored keys as permanent orphans — all + with every condition green. Endpoints stay mutable — rotating an NLB DNS + name or adding a member to the same cluster is routine; pointing at a + different cluster is caught at runtime by the checkpoint's dual-cluster-ID + binding, not by spec validation. + + The transition rules compare VALUES, presence-normalized: for these + fields the empty value and an absent field are semantically identical + (prefix "" = whole keyspace, destPrefix "" = strip the source prefix), and + Go typed clients drop explicit "" through omitempty — presence-based rules + would reject every typed-client update of a CR created with an explicit + empty string. + properties: + checkpoint: + description: Checkpoint configures the reserved checkpoint/fence key + on the target. + properties: + key: + description: |- + Key overrides the reserved checkpoint key. Defaults to the effective + destination prefix + "\x00etcdmirror-checkpoint" — the \x00 byte after + the prefix cannot collide with any real key under it. MUST live under + the effective destination prefix (target.prefix + sync.destPrefix, + CEL-enforced): the range-scoped target credential covers it, and the + exact-match exclusion from scans/counts/prune only works inside the + mirrored range. Immutable after creation. + type: string + type: object + initialSync: + description: |- + InitialSync governs genesis behavior: how pre-existing destination keys + are treated (Mode) and optionally where replication starts + (StartRevision). + properties: + mode: + default: RequireEmpty + description: |- + Mode governs pre-existing destination keys at genesis. Defaults to + RequireEmpty (refuse a non-empty destination prefix). + enum: + - RequireEmpty + - Overwrite + - OverwriteAndPrune + type: string + startRevision: + description: |- + StartRevision, when > 0, skips the genesis scan entirely and starts + watching from StartRevision+1. For fidelity-preserving seeds: restore + the target from a source snapshot (`etcdutl snapshot restore + --bump-revision --mark-compacted`), then mirror only the delta. + Requires Mode Overwrite or OverwriteAndPrune (CEL-enforced). + format: int64 + minimum: 0 + type: integer + type: object + x-kubernetes-validations: + - message: initialSync.startRevision requires initialSync.mode Overwrite + or OverwriteAndPrune (a seeded target is not empty) + rule: '!(has(self.startRevision) && self.startRevision > 0 && (!has(self.mode) + || self.mode == ''RequireEmpty''))' + mode: + default: Sync + description: |- + Mode selects continuous replication (Sync, the default) or a cutover + drain (Drain). See EtcdMirrorModeDrain for the cutover contract. + enum: + - Sync + - Drain + type: string + paused: + description: |- + Paused, when true, scales the agent Deployment to zero without deleting + the CR or its checkpoint. The checkpoint lives in the target etcd, so + resume picks up from the last fenced watermark. NOTE: pausing longer + than the source's compaction retention guarantees a full forced resync + on resume — there is no free lunch past the retention window. + type: boolean + podTemplate: + description: |- + PodTemplate carries scheduling/affinity/labels/annotations for the + agent pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. + properties: + metadata: + description: Metadata is the metadata to add to the pod. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + spec: + properties: + affinity: + description: Affinity is a group of affinity scheduling rules. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in the + range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + type: object + reconciliation: + description: |- + Reconciliation optionally enables a periodic full diff-and-repair pass + layered on top of the continuous watch-based mirror. Independent of + this setting, one reconciliation-with-delete pass always runs after any + forced resync (mark-and-sweep), and as the OverwriteAndPrune genesis + pass and the Drain verification pass. + properties: + deleteOrphans: + description: |- + DeleteOrphans, when true, allows the PERIODIC pass to delete target + keys under the destination prefix that have no corresponding source + key. Defaults to false. (Forced-resync sweeps and OverwriteAndPrune + always delete orphans; this knob only governs the periodic pass.) + type: boolean + enabled: + description: |- + Enabled toggles the PERIODIC pass. Defaults to false: it is a full + diff of the prefix contents on both sides (O(keyspace)), so it is + opt-in. + type: boolean + interval: + description: Interval between periodic passes. Defaults to 1h + when Enabled. + type: string + type: object + resources: + description: |- + Resources are the agent container's compute resources. The agent's + memory model is bounded by Sync.PageBytes (single in-flight scan page, + no unbounded read-ahead); size limits accordingly. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + source: + description: |- + Source is the etcd cluster keys are read from. EtcdMirror never writes + back to Source; the agent's source-side client is only ever used for + Get/Watch, never Put/Delete/Txn. + + VERSION FLOOR: source etcd must be >= 3.4 (probed via maintenance + Status() at connect; below the floor the mirror goes Failed with reason + UnsupportedVersion). >= 3.4.25 / 3.5.8 is the recommended floor: below + it, watch progress notifications are unreliable and the agent cannot + trust the watermark machinery that drives lag, the checkpoint, and the + Drain gate. + properties: + auth: + description: Auth configures etcd username/password (RBAC) auth + for THIS side. + properties: + secretRef: + description: |- + SecretRef names a Secret holding "username" and "password" keys. If + this whole Auth block is nil, the agent does not call etcd's + Authenticate() at all; the pinned v3 client transparently re-auths on + token expiry. PRECEDENCE: when both a client certificate and Auth are + supplied, etcd uses the token identity, not the certificate CN — the + Auth user must hold the range-scoped role. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + x-kubernetes-validations: + - message: secretRef.name is required + rule: has(self.secretRef.name) && size(self.secretRef.name) + > 0 + endpointList: + description: |- + EndpointList is a raw set of etcd client-URL host:port (or + scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". + This is the ONLY supported mechanism for a cluster external to this + Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB); + there is deliberately no tunnel/port-forward mode — terminate any + tunnel upstream and hand EtcdMirror the resulting stable endpoint(s). + + Prefer listing per-member endpoints over a single load-balancer VIP: a + TCP-health-checked VIP cannot see etcd quorum, and the client's own + balancer handles per-member failover. + + IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN) + to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a + hostname present as a DNS SAN on the certificate, or ensure the + certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch; + do not reach for InsecureSkipVerify. + items: + type: string + type: array + prefix: + description: |- + Prefix is the etcd key prefix on THIS side. On Source, only keys under + this prefix are synced; empty means the whole keyspace. On Target, this + is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's + rewrite formula). Immutable after creation. + type: string + serviceRef: + description: |- + ServiceRef points at a Kubernetes Service in this cluster whose DNS + name resolves the etcd client endpoint(s), for the same-cluster case. + Namespace defaults to the EtcdMirror's own namespace when empty. + properties: + name: + description: Name is the Service name. + minLength: 1 + type: string + namespace: + description: Namespace defaults to the EtcdMirror's own namespace + when empty. + type: string + port: + description: |- + Port is the Service port name or number exposing etcd's client API. + Defaults to "client" when empty, matching PodMonitorSpec.Port's + convention elsewhere in this API group. + type: string + required: + - name + type: object + tls: + description: |- + TLS configures the agent's client TLS for THIS side. Nil means the + agent dials this side in cleartext (and https:// endpoints are + CEL-rejected). An empty block means server-auth TLS verified against + the system trust roots. + properties: + caBundleRef: + description: |- + CABundleRef optionally sources the trust anchors from a separate + Secret or ConfigMap key, decoupling trust from the identity Secret + (Gateway API caCertificateRefs precedent). Takes precedence over + SecretRef's ca.crt. + properties: + key: + description: Key within the object. Defaults to "ca.crt". + type: string + kind: + default: ConfigMap + description: Kind is Secret or ConfigMap. Defaults to + ConfigMap. + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the Secret or ConfigMap, in the EtcdMirror's + namespace. + minLength: 1 + type: string + required: + - name + type: object + insecureSkipVerify: + default: false + description: |- + InsecureSkipVerify disables server certificate verification. Strongly + discouraged, especially for a source reached over the public internet. + Requires InsecureSkipVerifyAcknowledgeRisk to also be set true + (CEL-enforced). The controller additionally emits a standing Warning + event whenever this is true. + type: boolean + insecureSkipVerifyAcknowledgeRisk: + default: false + description: |- + InsecureSkipVerifyAcknowledgeRisk must independently be set true + whenever InsecureSkipVerify is true (CEL-enforced companion field). Its + only purpose is to require a deliberate, separate, reviewable line in + the manifest diff before disabling TLS verification. + type: boolean + secretRef: + description: |- + SecretRef names a Secret (in the EtcdMirror's namespace) holding this + side's TLS material in the standard kubernetes.io/tls-compatible shape: + - ca.crt: PEM CA bundle used to verify the peer's server certificate + (unless CABundleRef overrides it, or InsecureSkipVerify is true). + - tls.crt / tls.key: PEM client certificate + key, for mTLS. + Optional — omit both for server-auth-only TLS. + Nil means no client identity and verification against the system trust + roots (the etcdctl default). Note a source running with + --client-cert-auth (the RKE1 default) rejects certless clients at the + handshake regardless of etcd RBAC auth; server-auth-only + Auth is not + viable against such a source. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + serverName: + description: |- + ServerName overrides the TLS ServerName (SNI) used for verification, + for cases where the dialed address doesn't match a SAN on the + certificate (e.g. dialing an NLB IP directly). Applies to EVERY + endpoint in the list, so mixing endpoints with different certificates + behind one ServerName will fail verification on the mismatched ones. + type: string + type: object + x-kubernetes-validations: + - message: insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk + to also be true + rule: '!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk' + - message: secretRef.name must be non-empty when secretRef is + set + rule: '!has(self.secretRef) || (has(self.secretRef.name) && + size(self.secretRef.name) > 0)' + type: object + x-kubernetes-validations: + - message: exactly one of endpointList or serviceRef must be set + rule: (has(self.endpointList) && size(self.endpointList) > 0) != + has(self.serviceRef) + - message: 'http:// endpoints conflict with a tls block: use https:// + endpoints or remove tls' + rule: '!(has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''http://'')))' + - message: https:// endpoints require a tls block (an empty tls block + selects server-auth TLS against system trust roots) + rule: '!(!has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''https://'')))' + sync: + description: |- + Sync tunes runtime sync behavior (batching, paging, rate limiting, + prefix rewrite, timeouts, backoff). + properties: + destPrefix: + description: |- + DestPrefix is the middle term of the rewrite formula above. Default "" + means the source prefix is stripped and key remainders land directly + under target.prefix. Immutable after creation. + type: string + dialTimeout: + description: |- + DialTimeout bounds establishing the initial client connection to each + side. Defaults to 10s. + type: string + excludePrefixes: + description: |- + ExcludePrefixes lists source key prefixes (full source-side keys, e.g. + "/registry/events/") skipped entirely: not scanned, not watched, not + counted, not pruned. Use to drop high-churn low-value ranges and cut + WAN cost, or to skip lease-backed ranges that don't survive mirroring. + Nested/duplicate entries are normalized by the agent (an entry covered + by another is dropped). Immutable after creation (it defines the + mirrored range): removing an exclusion would require a backfill scan + the checkpoint-resume path never runs, and adding one would strand + already-mirrored keys as permanent orphans — change it via + delete-and-recreate with an appropriate initialSync.mode instead. + items: + type: string + maxItems: 64 + type: array + maxOpsPerSecond: + description: |- + MaxOpsPerSecond rate-limits the agent's target write rate (a token + bucket over puts+deletes/sec), applied to both the genesis scan and + watch-driven applies. Zero (default) means unlimited. Mind the + retention prerequisite above when throttling. + format: int32 + minimum: 0 + type: integer + maxTxnOps: + description: |- + MaxTxnOps bounds how many operations the agent batches into a single + target Txn, including the reserved checkpoint-write slot. Must not + exceed the target's --max-txn-ops (etcd default 128). Defaults to 128. + format: int32 + minimum: 2 + type: integer + pageBytes: + anyOf: + - type: integer + - type: string + description: PageBytes bounds bytes per source scan page. Defaults + to 1Mi. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + pageKeyLimit: + description: |- + PageKeyLimit bounds keys per source scan page during InitialSync and + reconciliation. The scan is pull-based, one page in flight — no + read-ahead — so this and PageBytes bound agent memory. Defaults to 512. + format: int32 + minimum: 1 + type: integer + reconnectBackoff: + description: |- + ReconnectBackoff bounds the retry/backoff loop wrapping connection-class + errors. Throttling-class errors (target rate rejection) use a more + conservative curve derived from the same bounds; quota exhaustion + (TargetQuotaExhausted) and permanent errors are never retried through + this loop. Defaults to exponential backoff from 1s to 30s. + properties: + initialDelay: + type: string + maxDelay: + type: string + type: object + requestTimeout: + description: |- + RequestTimeout is the per-RPC context deadline applied to every unary + call on both sides (watches excluded — they are long-lived by design + and covered by progress-notification liveness instead). Without it a + blackholed call through an NLB never errors and backoff never engages. + Defaults to 30s. + type: string + txnFlushBytes: + anyOf: + - type: integer + - type: string + description: |- + TxnFlushBytes is the byte watermark at which a batch is flushed (at the + next source-revision boundary). Keep well under etcd's request size + limits: a Txn over ~1.5MiB is rejected by the server and one over 2MiB + by the client send cap — both classified permanent errors, not + throttling. Defaults to 1Mi. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + watchBufferBytes: + anyOf: + - type: integer + - type: string + description: |- + WatchBufferBytes bounds the memory used to buffer watch events + observed from R0 while the genesis scan runs (the reflector replay + buffer). On overflow the agent cancels the source watch and restarts + the scan from a fresh R0 (see the InitialSyncCompactionRaced event) — + a bounded retry instead of unbounded growth when source churn outruns + scan+apply throughput. Defaults to 16Mi (must stay in lockstep with + pkg/mirroragent's DefaultWatchBufferBytes). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + target: + description: |- + Target is the etcd cluster keys are written into. + + SECURITY PREREQUISITE: Target's credential (etcd RBAC user and/or the + client certificate's role) MUST be range-scoped to the effective + destination prefix — never a cluster-admin-equivalent credential. The + agent's client-side rewrite logic is defense against bugs in its own + code, NOT a security boundary. The grant must also cover the reserved + checkpoint key (see Checkpoint). Configure via `etcdctl role + grant-permission --prefix=true readwrite ` before + pointing an EtcdMirror at the cluster. + + The target must run with auto-compaction enabled: forced-resync churn + and prune passes march an uncompacted target toward its storage quota + (2GiB by default), which surfaces as TargetQuotaExhausted. + properties: + auth: + description: Auth configures etcd username/password (RBAC) auth + for THIS side. + properties: + secretRef: + description: |- + SecretRef names a Secret holding "username" and "password" keys. If + this whole Auth block is nil, the agent does not call etcd's + Authenticate() at all; the pinned v3 client transparently re-auths on + token expiry. PRECEDENCE: when both a client certificate and Auth are + supplied, etcd uses the token identity, not the certificate CN — the + Auth user must hold the range-scoped role. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + x-kubernetes-validations: + - message: secretRef.name is required + rule: has(self.secretRef.name) && size(self.secretRef.name) + > 0 + endpointList: + description: |- + EndpointList is a raw set of etcd client-URL host:port (or + scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379". + This is the ONLY supported mechanism for a cluster external to this + Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB); + there is deliberately no tunnel/port-forward mode — terminate any + tunnel upstream and hand EtcdMirror the resulting stable endpoint(s). + + Prefer listing per-member endpoints over a single load-balancer VIP: a + TCP-health-checked VIP cannot see etcd quorum, and the client's own + balancer handles per-member failover. + + IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN) + to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a + hostname present as a DNS SAN on the certificate, or ensure the + certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch; + do not reach for InsecureSkipVerify. + items: + type: string + type: array + prefix: + description: |- + Prefix is the etcd key prefix on THIS side. On Source, only keys under + this prefix are synced; empty means the whole keyspace. On Target, this + is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's + rewrite formula). Immutable after creation. + type: string + serviceRef: + description: |- + ServiceRef points at a Kubernetes Service in this cluster whose DNS + name resolves the etcd client endpoint(s), for the same-cluster case. + Namespace defaults to the EtcdMirror's own namespace when empty. + properties: + name: + description: Name is the Service name. + minLength: 1 + type: string + namespace: + description: Namespace defaults to the EtcdMirror's own namespace + when empty. + type: string + port: + description: |- + Port is the Service port name or number exposing etcd's client API. + Defaults to "client" when empty, matching PodMonitorSpec.Port's + convention elsewhere in this API group. + type: string + required: + - name + type: object + tls: + description: |- + TLS configures the agent's client TLS for THIS side. Nil means the + agent dials this side in cleartext (and https:// endpoints are + CEL-rejected). An empty block means server-auth TLS verified against + the system trust roots. + properties: + caBundleRef: + description: |- + CABundleRef optionally sources the trust anchors from a separate + Secret or ConfigMap key, decoupling trust from the identity Secret + (Gateway API caCertificateRefs precedent). Takes precedence over + SecretRef's ca.crt. + properties: + key: + description: Key within the object. Defaults to "ca.crt". + type: string + kind: + default: ConfigMap + description: Kind is Secret or ConfigMap. Defaults to + ConfigMap. + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the Secret or ConfigMap, in the EtcdMirror's + namespace. + minLength: 1 + type: string + required: + - name + type: object + insecureSkipVerify: + default: false + description: |- + InsecureSkipVerify disables server certificate verification. Strongly + discouraged, especially for a source reached over the public internet. + Requires InsecureSkipVerifyAcknowledgeRisk to also be set true + (CEL-enforced). The controller additionally emits a standing Warning + event whenever this is true. + type: boolean + insecureSkipVerifyAcknowledgeRisk: + default: false + description: |- + InsecureSkipVerifyAcknowledgeRisk must independently be set true + whenever InsecureSkipVerify is true (CEL-enforced companion field). Its + only purpose is to require a deliberate, separate, reviewable line in + the manifest diff before disabling TLS verification. + type: boolean + secretRef: + description: |- + SecretRef names a Secret (in the EtcdMirror's namespace) holding this + side's TLS material in the standard kubernetes.io/tls-compatible shape: + - ca.crt: PEM CA bundle used to verify the peer's server certificate + (unless CABundleRef overrides it, or InsecureSkipVerify is true). + - tls.crt / tls.key: PEM client certificate + key, for mTLS. + Optional — omit both for server-auth-only TLS. + Nil means no client identity and verification against the system trust + roots (the etcdctl default). Note a source running with + --client-cert-auth (the RKE1 default) rejects certless clients at the + handshake regardless of etcd RBAC auth; server-auth-only + Auth is not + viable against such a source. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + serverName: + description: |- + ServerName overrides the TLS ServerName (SNI) used for verification, + for cases where the dialed address doesn't match a SAN on the + certificate (e.g. dialing an NLB IP directly). Applies to EVERY + endpoint in the list, so mixing endpoints with different certificates + behind one ServerName will fail verification on the mismatched ones. + type: string + type: object + x-kubernetes-validations: + - message: insecureSkipVerify requires insecureSkipVerifyAcknowledgeRisk + to also be true + rule: '!self.insecureSkipVerify || self.insecureSkipVerifyAcknowledgeRisk' + - message: secretRef.name must be non-empty when secretRef is + set + rule: '!has(self.secretRef) || (has(self.secretRef.name) && + size(self.secretRef.name) > 0)' + type: object + x-kubernetes-validations: + - message: exactly one of endpointList or serviceRef must be set + rule: (has(self.endpointList) && size(self.endpointList) > 0) != + has(self.serviceRef) + - message: 'http:// endpoints conflict with a tls block: use https:// + endpoints or remove tls' + rule: '!(has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''http://'')))' + - message: https:// endpoints require a tls block (an empty tls block + selects server-auth TLS against system trust roots) + rule: '!(!has(self.tls) && has(self.endpointList) && self.endpointList.exists(e, + e.startsWith(''https://'')))' + required: + - source + - target + type: object + x-kubernetes-validations: + - message: source.prefix is immutable + rule: '(has(self.source.prefix) ? self.source.prefix : "") == (has(oldSelf.source.prefix) + ? oldSelf.source.prefix : "")' + - message: target.prefix is immutable + rule: '(has(self.target.prefix) ? self.target.prefix : "") == (has(oldSelf.target.prefix) + ? oldSelf.target.prefix : "")' + - message: sync.destPrefix is immutable + rule: '(has(self.sync) && has(self.sync.destPrefix) ? self.sync.destPrefix + : "") == (has(oldSelf.sync) && has(oldSelf.sync.destPrefix) ? oldSelf.sync.destPrefix + : "")' + - message: sync.excludePrefixes is immutable + rule: '(has(self.sync) && has(self.sync.excludePrefixes) ? self.sync.excludePrefixes + : []) == (has(oldSelf.sync) && has(oldSelf.sync.excludePrefixes) ? + oldSelf.sync.excludePrefixes : [])' + - message: checkpoint.key is immutable + rule: '(has(self.checkpoint) && has(self.checkpoint.key) ? self.checkpoint.key + : "") == (has(oldSelf.checkpoint) && has(oldSelf.checkpoint.key) ? + oldSelf.checkpoint.key : "")' + - message: checkpoint.key must live under the effective destination prefix + (target.prefix + sync.destPrefix) + rule: '!has(self.checkpoint) || !has(self.checkpoint.key) || self.checkpoint.key + == "" || self.checkpoint.key.startsWith((has(self.target.prefix) ? + self.target.prefix : "") + (has(self.sync) && has(self.sync.destPrefix) + ? self.sync.destPrefix : ""))' + status: + description: |- + EtcdMirrorStatus defines the observed state of an EtcdMirror. Progress + fields are synced periodically (not per-op), so they are a coarse, + point-in-time mirror of the agent's authoritative checkpoint in the target + etcd, useful for kubectl/dashboards, not an audit log. + + CUTOVER GATE CONTRACT: never compute "caught up" by comparing + SourceRevision to LastAppliedRevision — they snapshot at different + instants, and SourceRevision advances on out-of-prefix writes (revisions + are cluster-global). The manual gate is: quiesce source writers, read the + source's current revision R yourself (`etcdctl endpoint status`), then + poll until LastAppliedRevision >= R. The in-CR gate is spec.mode=Drain + + the CutoverReady condition. Relax target RBAC only after the mirror is + paused or deleted. + properties: + agentPod: + description: |- + AgentPod is the name of the current agent pod (the sole pod of the + size-1 Deployment), for convenient kubectl logs/exec. + type: string + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + cutover: + description: Cutover is populated while spec.mode is Drain. + properties: + drainTargetRevision: + description: |- + DrainTargetRevision is the source revision observed when Drain was + requested — the revision the watermark must reach. + format: int64 + type: integer + drainedRevision: + description: DrainedRevision is the watermark at which the drain + completed. + format: int64 + type: integer + leasedKeyCount: + description: |- + LeasedKeyCount is LeaseBackedKeyCount frozen at drain completion, for + the runbook's purge/re-lease step. + format: int64 + type: integer + sourceKeyCount: + description: |- + SourceKeyCount and TargetKeyCount are the per-side key counts from the + verification pass (source read pinned at the drained revision; + reserved checkpoint key excluded). + format: int64 + type: integer + targetKeyCount: + format: int64 + type: integer + verifiedTime: + description: VerifiedTime is when the post-drain verification + pass succeeded. + format: date-time + type: string + type: object + forcedResyncCount: + description: |- + ForcedResyncCount counts forced resyncs (compaction outran the watch, + checkpoint invalidated by a cluster-ID mismatch, or checkpoint + corrupt/unknown-version). Monotonic, never reset. + format: int32 + type: integer + initialSyncCompletionTime: + format: date-time + type: string + initialSyncKeyCount: + description: |- + InitialSyncKeyCount is the number of keys applied so far by the + current/last genesis scan. Live-updating during InitialSync (each + status sync), so InitialSyncKeyCount/InitialSyncTotalKeyCount is a + progress fraction. + format: int64 + type: integer + initialSyncStartTime: + format: date-time + type: string + initialSyncTotalKeyCount: + description: |- + InitialSyncTotalKeyCount is the denominator: the source-side key count + under the prefix observed at scan start (first page's RangeResponse + count). + format: int64 + type: integer + lastAppliedRevision: + description: |- + LastAppliedRevision is the checkpoint watermark: the source revision + through which the target is caught up, advanced by applies AND by + watch progress notifications on idle prefixes. The fenced checkpoint + key in the target etcd is the authoritative copy; this mirrors it. + format: int64 + type: integer + lastProgressTime: + description: |- + LastProgressTime is when the watermark last advanced (apply or watch + progress notification). Distinct from LastStatusSyncTime: status can + keep syncing from a wedged loop; this field is what Available and + ReplicationLagExceeded are derived from. + format: date-time + type: string + lastReconciliationDrift: + properties: + divergentKeys: + description: |- + DivergentKeys were present on both sides with different values — + distinct from MissingKeys so "a resync dropped keys" is never + conflated with "a blind window went stale". + format: int64 + type: integer + missingKeys: + description: MissingKeys were present on the source but absent + on the target. + format: int64 + type: integer + orphanKeys: + description: OrphanKeys were present on the target with no source + counterpart. + format: int64 + type: integer + repaired: + description: Repaired is true when the pass wrote fixes rather + than only reporting. + type: boolean + type: object + lastReconciliationTime: + description: |- + LastReconciliationTime and LastReconciliationDrift record the most + recent reconciliation pass (periodic or mandatory). + format: date-time + type: string + lastStatusSyncTime: + description: |- + LastStatusSyncTime is when status was last refreshed from the agent, so + staleness of everything above is directly observable. + format: date-time + type: string + leaseBackedKeyCount: + description: |- + LeaseBackedKeyCount is the number of mirrored keys whose source copy is + lease-backed (kv.Lease != 0). Mirrored copies are NOT lease-backed — + leases are stripped (see Fidelity Caveats in docs/etcdmirror.md) — so a + nonzero count means the cutover runbook's purge/re-lease step applies. + format: int64 + type: integer + observedGeneration: + format: int64 + type: integer + phase: + description: |- + EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. + Unlike BackupPhase/RestorePhase, most phases here are NOT terminal — a + healthy mirror spends its life in Syncing; there is no "Completed" state. + type: string + sourceClusterID: + description: |- + SourceClusterID and TargetClusterID are the cluster IDs both bound + into the checkpoint. Either changing across reconciles is the visible + symptom of an endpoint now pointing at a different cluster than the + checkpoint was taken against (CheckpointInvalidated event, forced + genesis, RequireEmpty re-armed). + type: string + sourceKeyCount: + description: |- + SourceKeyCount and TargetKeyCount are the per-side key counts from the + most recent diff/verification pass (reserved checkpoint key and + excluded prefixes not counted; drain-verification source reads pinned + at the drained revision with a compacted-fallback re-read). Populated + by every pass that runs regardless of spec.reconciliation.enabled — + the mandatory mark-and-sweep after any forced resync, the + OverwriteAndPrune genesis pass, and the drain verification — plus the + periodic pass when it is enabled. NOT refreshed on every status sync: + a healthy RequireEmpty mirror that never forces a resync only gets + counts from an enabled periodic pass. This is the equality signal + InvariantsHeld reads; status.cutover's copies remain the frozen + drain-time snapshot. + format: int64 + type: integer + sourceRevision: + description: |- + SourceRevision is the source cluster's revision as of the last status + sync. Cluster-global: it advances on writes outside the mirrored + prefix, so SourceRevision - LastAppliedRevision OVERSTATES lag for + prefix-scoped mirrors. See the cutover gate contract above. + format: int64 + type: integer + sourceVersion: + description: |- + SourceVersion and TargetVersion are the etcd server versions from the + maintenance Status() probe at connect. + type: string + targetClusterID: + type: string + targetKeyCount: + format: int64 + type: integer + targetVersion: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/samples/operator_v1alpha1_etcdmirror.yaml b/config/samples/operator_v1alpha1_etcdmirror.yaml new file mode 100644 index 00000000..3a3558ec --- /dev/null +++ b/config/samples/operator_v1alpha1_etcdmirror.yaml @@ -0,0 +1,88 @@ +apiVersion: operator.etcd.io/v1alpha1 +kind: EtcdMirror +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdmirror-sample +spec: + # Sync = continuous replication; flip to Drain at cutover time and wait for + # the CutoverReady condition. + mode: Sync + + # External RKE1/AWS source reached over a public NLB, over TLS with a + # CA-verified server certificate. https:// endpoints require a tls block + # (an empty tls block would mean system trust roots). + source: + endpointList: + - "https://etcd-rke1.example.com:2379" + prefix: "/registry/" + tls: + secretRef: + name: etcdmirror-sample-source-tls + auth: + secretRef: + name: etcdmirror-sample-source-auth + + # In-cluster EtcdCluster reached via its client Service. + target: + serviceRef: + name: etcdcluster-sample-client + port: client + prefix: "/mirrored/" + tls: + secretRef: + name: etcdmirror-sample-target-tls + + initialSync: + # RequireEmpty (default): refuse a non-empty destination prefix at + # genesis. Overwrite / OverwriteAndPrune for re-pointing at a + # previously-populated prefix (e.g. failback). + mode: RequireEmpty + + sync: + # Key rewrite — ONE formula: + # key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) + # + # source key destPrefix resulting target key + # /registry/pods/x "" /mirrored/pods/x + # /registry/pods/x "registry/" /mirrored/registry/pods/x + # /registry/pods/x "backup/" /mirrored/backup/pods/x + # + # Default "" strips the source prefix. Immutable after creation. Leave + # unset for the default — an explicit destPrefix: "" is equivalent but + # pointless. + # destPrefix: "registry/" + # High-churn, lease-backed, or low-value ranges to skip entirely. + # Immutable after creation (it defines the mirrored range): change via + # delete-and-recreate with an appropriate initialSync.mode. + excludePrefixes: + - "/registry/events/" + maxTxnOps: 128 # includes one reserved slot for the checkpoint write + txnFlushBytes: 1Mi + pageKeyLimit: 512 + pageBytes: 1Mi + # Replay-buffer bound during the genesis scan; raise for high-churn + # sources (overflow restarts the scan from a fresh revision, surfaced + # as the InitialSyncCompactionRaced event). + # watchBufferBytes: 16Mi + maxOpsPerSecond: 500 # keep source retention > scan+drain time at this rate + requestTimeout: 30s + dialTimeout: 10s + reconnectBackoff: + initialDelay: 1s + maxDelay: 30s + + # The checkpoint lives in the TARGET etcd, fenced and written with each + # batch. Default key: + "\x00etcdmirror-checkpoint". + # An override MUST stay under the effective destination prefix (the + # range-scoped target credential covers it; CEL-enforced) — pick a + # different suffix under it if the default collides with your own + # reserved-key scheme. + # checkpoint: + # key: "/mirrored/\x00my-own-checkpoint-name" + + reconciliation: + enabled: true + interval: 1h + deleteOrphans: false diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md index 6a32be4e..b27eafb2 100644 --- a/docs/api-references/docs.md +++ b/docs/api-references/docs.md @@ -11,6 +11,8 @@ Package v1alpha1 contains API Schema definitions for the operator v1alpha1 API g ### Resource Types - [EtcdCluster](#etcdcluster) - [EtcdClusterList](#etcdclusterlist) +- [EtcdMirror](#etcdmirror) +- [EtcdMirrorList](#etcdmirrorlist) @@ -115,6 +117,444 @@ _Appears in:_ +#### EtcdMirror + + + +EtcdMirror is the Schema for the etcdmirrors API. It describes a +continuous, one-way key-range sync from a source etcd cluster to a target +etcd cluster, run as a single supervised stateless pod (a size-1 +Deployment; progress lives in a fenced checkpoint key in the target etcd, +not on a volume). + +It is a byte-copy of keys and values, not a replica: revisions, versions, +and create/mod ordering are target-assigned, and leases are stripped. See +Fidelity Caveats in docs/etcdmirror.md before depending on anything but +key/value content. + +Two-way sync: never — etcd revisions are cluster-local and there is no +per-key provenance channel, so bidirectional sync is structurally +inexpressible. Reversal for cutover/failback: yes — delete the CR and +create a new one with swapped endpoints, initialSync.mode +OverwriteAndPrune, only after the forward mirror reported CutoverReady. +See docs/etcdmirror.md. + + + +_Appears in:_ +- [EtcdMirrorList](#etcdmirrorlist) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `operator.etcd.io/v1alpha1` | | | +| `kind` _string_ | `EtcdMirror` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[EtcdMirrorSpec](#etcdmirrorspec)_ | | | | + + +#### EtcdMirrorAuth + + + +EtcdMirrorAuth configures etcd RBAC username/password auth for one side. + + + +_Appears in:_ +- [EtcdMirrorEndpoint](#etcdmirrorendpoint) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `secretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#localobjectreference-v1-core)_ | SecretRef names a Secret holding "username" and "password" keys. If
this whole Auth block is nil, the agent does not call etcd's
Authenticate() at all; the pinned v3 client transparently re-auths on
token expiry. PRECEDENCE: when both a client certificate and Auth are
supplied, etcd uses the token identity, not the certificate CN — the
Auth user must hold the range-scoped role. | | | + + +#### EtcdMirrorBackoffSpec + + + + + + + +_Appears in:_ +- [EtcdMirrorSyncSpec](#etcdmirrorsyncspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `initialDelay` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | | | Optional: \{\}
| +| `maxDelay` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | | | Optional: \{\}
| + + +#### EtcdMirrorCABundleRef + + + +EtcdMirrorCABundleRef points at one key of a Secret or ConfigMap holding a +PEM CA bundle. + + + +_Appears in:_ +- [EtcdMirrorTLS](#etcdmirrortls) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `kind` _string_ | Kind is Secret or ConfigMap. Defaults to ConfigMap. | ConfigMap | Enum: [Secret ConfigMap]
Optional: \{\}
| +| `name` _string_ | Name of the Secret or ConfigMap, in the EtcdMirror's namespace. | | MinLength: 1
| +| `key` _string_ | Key within the object. Defaults to "ca.crt". | | Optional: \{\}
| + + +#### EtcdMirrorCheckpointSpec + + + +EtcdMirrorCheckpointSpec configures the reserved checkpoint/fence key the +agent maintains IN THE TARGET etcd. The checkpoint (the source-revision +watermark plus {linkUID, epoch, role}) is written in the SAME Txn as every +applied batch and fenced with a mod-revision compare on EVERY write path +(applies, reconciliation repairs, prune deletes), so two agents can never +interleave writes and a straggler apply after cutover fails loudly. The +key is excluded by exact match from scans, counts, prune passes, and the +RequireEmpty check; the target RBAC grant must cover it; CR deletion +removes it via a delete-one-key finalizer. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `key` _string_ | Key overrides the reserved checkpoint key. Defaults to the effective
destination prefix + "\x00etcdmirror-checkpoint" — the \x00 byte after
the prefix cannot collide with any real key under it. MUST live under
the effective destination prefix (target.prefix + sync.destPrefix,
CEL-enforced): the range-scoped target credential covers it, and the
exact-match exclusion from scans/counts/prune only works inside the
mirrored range. Immutable after creation. | | Optional: \{\}
| + + +#### EtcdMirrorCutoverStatus + + + +EtcdMirrorCutoverStatus tracks a Drain-mode cutover. + + + +_Appears in:_ +- [EtcdMirrorStatus](#etcdmirrorstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `drainTargetRevision` _integer_ | DrainTargetRevision is the source revision observed when Drain was
requested — the revision the watermark must reach. | | Optional: \{\}
| +| `drainedRevision` _integer_ | DrainedRevision is the watermark at which the drain completed. | | Optional: \{\}
| +| `verifiedTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#time-v1-meta)_ | VerifiedTime is when the post-drain verification pass succeeded. | | Optional: \{\}
| +| `sourceKeyCount` _integer_ | SourceKeyCount and TargetKeyCount are the per-side key counts from the
verification pass (source read pinned at the drained revision;
reserved checkpoint key excluded). | | Optional: \{\}
| +| `targetKeyCount` _integer_ | | | Optional: \{\}
| +| `leasedKeyCount` _integer_ | LeasedKeyCount is LeaseBackedKeyCount frozen at drain completion, for
the runbook's purge/re-lease step. | | Optional: \{\}
| + + +#### EtcdMirrorDriftInfo + + + + + + + +_Appears in:_ +- [EtcdMirrorStatus](#etcdmirrorstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `missingKeys` _integer_ | MissingKeys were present on the source but absent on the target. | | | +| `divergentKeys` _integer_ | DivergentKeys were present on both sides with different values —
distinct from MissingKeys so "a resync dropped keys" is never
conflated with "a blind window went stale". | | | +| `orphanKeys` _integer_ | OrphanKeys were present on the target with no source counterpart. | | | +| `repaired` _boolean_ | Repaired is true when the pass wrote fixes rather than only reporting. | | | + + +#### EtcdMirrorEndpoint + + + +EtcdMirrorEndpoint describes how to reach, authenticate to, and scope one +side (source or target) of a mirror. Both sides need an identical shape +(address resolution + prefix + TLS + auth), so one type serves both roles, +the same way BackupDestination is reused verbatim between EtcdBackup and +EtcdRestore rather than forked into near-duplicate per-role types. + +Exactly one of EndpointList or ServiceRef must be set. An empty +endpointList ([]) is treated as unset, per Kubernetes list conventions — +so `endpointList: []` alongside a serviceRef is accepted. + +Endpoint scheme and the TLS block must agree (CEL-enforced both ways): +http:// endpoints with a tls block would silently drop TLS at dial time; +https:// endpoints without one would dial with undeclared system-roots +TLS. The agent derives the dial scheme from the presence of the tls block, +so the declared contract is true by construction. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `endpointList` _string array_ | EndpointList is a raw set of etcd client-URL host:port (or
scheme://host:port) strings, e.g. "https://etcd-rke1.example.com:2379".
This is the ONLY supported mechanism for a cluster external to this
Kubernetes cluster (e.g. an RKE1/AWS source reached over a public NLB);
there is deliberately no tunnel/port-forward mode — terminate any
tunnel upstream and hand EtcdMirror the resulting stable endpoint(s).
Prefer listing per-member endpoints over a single load-balancer VIP: a
TCP-health-checked VIP cannot see etcd quorum, and the client's own
balancer handles per-member failover.
IP-LITERAL ENDPOINTS: Go's TLS stack requires an IP SAN (not a DNS SAN)
to verify a bare-IP endpoint. Either set EtcdMirrorTLS.ServerName to a
hostname present as a DNS SAN on the certificate, or ensure the
certificate carries a matching IP SAN. Fix the SAN/ServerName mismatch;
do not reach for InsecureSkipVerify. | | Optional: \{\}
| +| `serviceRef` _[EtcdMirrorServiceRef](#etcdmirrorserviceref)_ | ServiceRef points at a Kubernetes Service in this cluster whose DNS
name resolves the etcd client endpoint(s), for the same-cluster case.
Namespace defaults to the EtcdMirror's own namespace when empty. | | Optional: \{\}
| +| `prefix` _string_ | Prefix is the etcd key prefix on THIS side. On Source, only keys under
this prefix are synced; empty means the whole keyspace. On Target, this
is the prefix under which mirrored keys land (see EtcdMirrorSyncSpec's
rewrite formula). Immutable after creation. | | Optional: \{\}
| +| `tls` _[EtcdMirrorTLS](#etcdmirrortls)_ | TLS configures the agent's client TLS for THIS side. Nil means the
agent dials this side in cleartext (and https:// endpoints are
CEL-rejected). An empty block means server-auth TLS verified against
the system trust roots. | | Optional: \{\}
| +| `auth` _[EtcdMirrorAuth](#etcdmirrorauth)_ | Auth configures etcd username/password (RBAC) auth for THIS side. | | Optional: \{\}
| + + +#### EtcdMirrorInitialSyncMode + +_Underlying type:_ _string_ + +EtcdMirrorInitialSyncMode governs how the agent treats pre-existing keys +under the effective destination prefix at genesis (first-ever sync, or a +checkpoint invalidated by a cluster-identity mismatch). + + + +_Appears in:_ +- [EtcdMirrorInitialSyncSpec](#etcdmirrorinitialsyncspec) + +| Field | Description | +| --- | --- | +| `RequireEmpty` | EtcdMirrorInitialSyncRequireEmpty refuses to start if the destination
prefix already holds any key (Phase -> Failed, condition
EmptyTargetViolation). The reserved checkpoint key is excluded by exact
match.
RE-ARM CONTRACT: a source OR target cluster-ID mismatch invalidates
the checkpoint, forces genesis, and RE-ARMS this check. An ordinary
forced resync (Compacted) does NOT re-check RequireEmpty: the decoded,
ownership-validated fence proves the destination data is this link's
own.
| +| `Overwrite` | EtcdMirrorInitialSyncOverwrite scans and writes over whatever is there.
Keys present on the target but absent on the source are left alone.
| +| `OverwriteAndPrune` | EtcdMirrorInitialSyncOverwriteAndPrune is Overwrite plus one mandatory
orphan-prune pass after the scan: target keys under the destination
prefix with no source counterpart are deleted. This makes reversal onto
a previously-populated prefix (failback) a first-class correct
operation instead of silently resurrecting deleted keys.
| + + +#### EtcdMirrorInitialSyncSpec + + + +EtcdMirrorInitialSyncSpec governs the genesis scan. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `mode` _[EtcdMirrorInitialSyncMode](#etcdmirrorinitialsyncmode)_ | Mode governs pre-existing destination keys at genesis. Defaults to
RequireEmpty (refuse a non-empty destination prefix). | RequireEmpty | Enum: [RequireEmpty Overwrite OverwriteAndPrune]
Optional: \{\}
| +| `startRevision` _integer_ | StartRevision, when > 0, skips the genesis scan entirely and starts
watching from StartRevision+1. For fidelity-preserving seeds: restore
the target from a source snapshot (`etcdutl snapshot restore
--bump-revision --mark-compacted`), then mirror only the delta.
Requires Mode Overwrite or OverwriteAndPrune (CEL-enforced). | | Minimum: 0
Optional: \{\}
| + + +#### EtcdMirrorList + + + +EtcdMirrorList contains a list of EtcdMirror. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `operator.etcd.io/v1alpha1` | | | +| `kind` _string_ | `EtcdMirrorList` | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[EtcdMirror](#etcdmirror) array_ | | | | + + +#### EtcdMirrorMode + +_Underlying type:_ _string_ + +EtcdMirrorMode selects the mirror's operating mode. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | +| --- | --- | +| `Sync` | EtcdMirrorModeSync is normal continuous replication.
| +| `Drain` | EtcdMirrorModeDrain prepares for cutover: the agent records the source
revision observed when Drain is requested (status.cutover.drainTargetRevision),
keeps replicating until the checkpoint watermark reaches it, runs a
verification pass (per-side key counts, lease-backed key count), then
sets the CutoverReady condition and flips the fence key's role to
Primary so any straggler apply fails its mod-revision compare loudly.
Runbook: quiesce source writers -> set mode=Drain ->
`kubectl wait --for=condition=CutoverReady etcdmirror/` ->
purge/re-lease lease-backed keys -> repoint clients -> delete the CR.
| + + +#### EtcdMirrorPhase + +_Underlying type:_ _string_ + +EtcdMirrorPhase is a high-level summary of an EtcdMirror's lifecycle. +Unlike BackupPhase/RestorePhase, most phases here are NOT terminal — a +healthy mirror spends its life in Syncing; there is no "Completed" state. + + + +_Appears in:_ +- [EtcdMirrorStatus](#etcdmirrorstatus) + +| Field | Description | +| --- | --- | +| `Pending` | EtcdMirrorPhasePending means the EtcdMirror has been accepted but the
agent workload has not been created yet.
| +| `Connecting` | EtcdMirrorPhaseConnecting means the agent pod is running, establishing
client connections to both sides and probing versions/cluster IDs.
| +| `InitialSync` | EtcdMirrorPhaseInitialSync means the agent is running the genesis scan:
an UNPINNED chunked scan with the watch already open from the revision
observed before the scan started, buffered events replayed over the
scanned base (reflector pattern). Because pages read at the current
revision, mid-scan compaction cannot fail the scan. Also entered during
a forced resync (then with condition Compacted=True/Reason=ForcedResync).
| +| `Syncing` | EtcdMirrorPhaseSyncing is the steady state: watching and applying live
changes, watermark advancing via progress notifications.
| +| `Degraded` | EtcdMirrorPhaseDegraded means the agent is in a retry/backoff loop
(connection or throttling class) and is expected to self-heal. Forced
resyncs are NOT Degraded; they report as InitialSync + Compacted=True.
| +| `Paused` | EtcdMirrorPhasePaused means spec.paused is true; the agent Deployment
is scaled to zero. The checkpoint is retained in the target.
| +| `Failed` | EtcdMirrorPhaseFailed means a terminal, non-recoverable error requiring
operator intervention: EmptyTargetViolation at genesis, source below
the 3.4 version floor (UnsupportedVersion), a permanent-class write
error (oversized revision vs target limits), malformed cert material,
or unresolvable spec misconfiguration.
| + + +#### EtcdMirrorReconciliationSpec + + + +EtcdMirrorReconciliationSpec configures the periodic full reconciliation +pass. The same engine also runs unconditionally (regardless of Enabled or +DeleteOrphans) as the post-forced-resync mark-and-sweep, the +OverwriteAndPrune genesis pass, and the Drain verification pass. + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `enabled` _boolean_ | Enabled toggles the PERIODIC pass. Defaults to false: it is a full
diff of the prefix contents on both sides (O(keyspace)), so it is
opt-in. | | Optional: \{\}
| +| `interval` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | Interval between periodic passes. Defaults to 1h when Enabled. | | Optional: \{\}
| +| `deleteOrphans` _boolean_ | DeleteOrphans, when true, allows the PERIODIC pass to delete target
keys under the destination prefix that have no corresponding source
key. Defaults to false. (Forced-resync sweeps and OverwriteAndPrune
always delete orphans; this knob only governs the periodic pass.) | | Optional: \{\}
| + + +#### EtcdMirrorServiceRef + + + +EtcdMirrorServiceRef points at a Service and the client port on it to dial. + + + +_Appears in:_ +- [EtcdMirrorEndpoint](#etcdmirrorendpoint) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name is the Service name. | | MinLength: 1
| +| `namespace` _string_ | Namespace defaults to the EtcdMirror's own namespace when empty. | | Optional: \{\}
| +| `port` _string_ | Port is the Service port name or number exposing etcd's client API.
Defaults to "client" when empty, matching PodMonitorSpec.Port's
convention elsewhere in this API group. | | Optional: \{\}
| + + +#### EtcdMirrorSpec + + + +EtcdMirrorSpec defines the desired state of an EtcdMirror. + +Range-defining and rewrite fields are immutable (CEL transition rules +below): source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes +and checkpoint.key. Changing what range is mirrored, or where it lands, +mid-life silently diverges: a restarted agent resumes from its checkpoint +without a scan, so removing an exclusion never backfills pre-existing keys +and adding one strands already-mirrored keys as permanent orphans — all +with every condition green. Endpoints stay mutable — rotating an NLB DNS +name or adding a member to the same cluster is routine; pointing at a +different cluster is caught at runtime by the checkpoint's dual-cluster-ID +binding, not by spec validation. + +The transition rules compare VALUES, presence-normalized: for these +fields the empty value and an absent field are semantically identical +(prefix "" = whole keyspace, destPrefix "" = strip the source prefix), and +Go typed clients drop explicit "" through omitempty — presence-based rules +would reject every typed-client update of a CR created with an explicit +empty string. + + + +_Appears in:_ +- [EtcdMirror](#etcdmirror) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `mode` _[EtcdMirrorMode](#etcdmirrormode)_ | Mode selects continuous replication (Sync, the default) or a cutover
drain (Drain). See EtcdMirrorModeDrain for the cutover contract. | Sync | Enum: [Sync Drain]
Optional: \{\}
| +| `source` _[EtcdMirrorEndpoint](#etcdmirrorendpoint)_ | Source is the etcd cluster keys are read from. EtcdMirror never writes
back to Source; the agent's source-side client is only ever used for
Get/Watch, never Put/Delete/Txn.
VERSION FLOOR: source etcd must be >= 3.4 (probed via maintenance
Status() at connect; below the floor the mirror goes Failed with reason
UnsupportedVersion). >= 3.4.25 / 3.5.8 is the recommended floor: below
it, watch progress notifications are unreliable and the agent cannot
trust the watermark machinery that drives lag, the checkpoint, and the
Drain gate. | | | +| `target` _[EtcdMirrorEndpoint](#etcdmirrorendpoint)_ | Target is the etcd cluster keys are written into.
SECURITY PREREQUISITE: Target's credential (etcd RBAC user and/or the
client certificate's role) MUST be range-scoped to the effective
destination prefix — never a cluster-admin-equivalent credential. The
agent's client-side rewrite logic is defense against bugs in its own
code, NOT a security boundary. The grant must also cover the reserved
checkpoint key (see Checkpoint). Configure via `etcdctl role
grant-permission --prefix=true readwrite ` before
pointing an EtcdMirror at the cluster.
The target must run with auto-compaction enabled: forced-resync churn
and prune passes march an uncompacted target toward its storage quota
(2GiB by default), which surfaces as TargetQuotaExhausted. | | | +| `initialSync` _[EtcdMirrorInitialSyncSpec](#etcdmirrorinitialsyncspec)_ | InitialSync governs genesis behavior: how pre-existing destination keys
are treated (Mode) and optionally where replication starts
(StartRevision). | | Optional: \{\}
| +| `sync` _[EtcdMirrorSyncSpec](#etcdmirrorsyncspec)_ | Sync tunes runtime sync behavior (batching, paging, rate limiting,
prefix rewrite, timeouts, backoff). | | Optional: \{\}
| +| `checkpoint` _[EtcdMirrorCheckpointSpec](#etcdmirrorcheckpointspec)_ | Checkpoint configures the reserved checkpoint/fence key on the target. | | Optional: \{\}
| +| `reconciliation` _[EtcdMirrorReconciliationSpec](#etcdmirrorreconciliationspec)_ | Reconciliation optionally enables a periodic full diff-and-repair pass
layered on top of the continuous watch-based mirror. Independent of
this setting, one reconciliation-with-delete pass always runs after any
forced resync (mark-and-sweep), and as the OverwriteAndPrune genesis
pass and the Drain verification pass. | | Optional: \{\}
| +| `podTemplate` _[PodTemplate](#podtemplate)_ | PodTemplate carries scheduling/affinity/labels/annotations for the
agent pod, reusing EtcdClusterSpec's PodTemplate shape verbatim. | | Optional: \{\}
| +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#resourcerequirements-v1-core)_ | Resources are the agent container's compute resources. The agent's
memory model is bounded by Sync.PageBytes (single in-flight scan page,
no unbounded read-ahead); size limits accordingly. | | Optional: \{\}
| +| `paused` _boolean_ | Paused, when true, scales the agent Deployment to zero without deleting
the CR or its checkpoint. The checkpoint lives in the target etcd, so
resume picks up from the last fenced watermark. NOTE: pausing longer
than the source's compaction retention guarantees a full forced resync
on resume — there is no free lunch past the retention window. | | Optional: \{\}
| + + + + +#### EtcdMirrorSyncSpec + + + +EtcdMirrorSyncSpec tunes the mirror's runtime sync behavior. + +KEY REWRITE — one formula, no other composition: + + key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix) + +(anchored strip-and-reprefix; never a substring replace). + +BATCHING INVARIANT: target Txns flush ONLY at source-revision boundaries — +a source revision's events are never split across Txns, whole revisions +are coalesced up to the MaxTxnOps/TxnFlushBytes watermarks, and one op +slot in MaxTxnOps is always reserved for the checkpoint write that rides +in the same Txn. A single source revision larger than MaxTxnOps is applied +as one oversized Txn (provision the target's --max-txn-ops accordingly) +with the checkpoint held until it lands. + +RETENTION PREREQUISITE: the source's compaction retention window must +exceed the worst-case initial scan + throttled drain time, approximately +sourceKeyCount / min(effective scan rate, MaxOpsPerSecond). If it does +not, genesis (and every forced resync) loses the race with compaction and +the mirror livelocks (surfaced via the resync-loop detector). + + + +_Appears in:_ +- [EtcdMirrorSpec](#etcdmirrorspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `destPrefix` _string_ | DestPrefix is the middle term of the rewrite formula above. Default ""
means the source prefix is stripped and key remainders land directly
under target.prefix. Immutable after creation. | | Optional: \{\}
| +| `excludePrefixes` _string array_ | ExcludePrefixes lists source key prefixes (full source-side keys, e.g.
"/registry/events/") skipped entirely: not scanned, not watched, not
counted, not pruned. Use to drop high-churn low-value ranges and cut
WAN cost, or to skip lease-backed ranges that don't survive mirroring.
Nested/duplicate entries are normalized by the agent (an entry covered
by another is dropped). Immutable after creation (it defines the
mirrored range): removing an exclusion would require a backfill scan
the checkpoint-resume path never runs, and adding one would strand
already-mirrored keys as permanent orphans — change it via
delete-and-recreate with an appropriate initialSync.mode instead. | | MaxItems: 64
Optional: \{\}
| +| `maxTxnOps` _integer_ | MaxTxnOps bounds how many operations the agent batches into a single
target Txn, including the reserved checkpoint-write slot. Must not
exceed the target's --max-txn-ops (etcd default 128). Defaults to 128. | | Minimum: 2
Optional: \{\}
| +| `txnFlushBytes` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#quantity-resource-api)_ | TxnFlushBytes is the byte watermark at which a batch is flushed (at the
next source-revision boundary). Keep well under etcd's request size
limits: a Txn over ~1.5MiB is rejected by the server and one over 2MiB
by the client send cap — both classified permanent errors, not
throttling. Defaults to 1Mi. | | Optional: \{\}
| +| `pageKeyLimit` _integer_ | PageKeyLimit bounds keys per source scan page during InitialSync and
reconciliation. The scan is pull-based, one page in flight — no
read-ahead — so this and PageBytes bound agent memory. Defaults to 512. | | Minimum: 1
Optional: \{\}
| +| `pageBytes` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#quantity-resource-api)_ | PageBytes bounds bytes per source scan page. Defaults to 1Mi. | | Optional: \{\}
| +| `watchBufferBytes` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#quantity-resource-api)_ | WatchBufferBytes bounds the memory used to buffer watch events
observed from R0 while the genesis scan runs (the reflector replay
buffer). On overflow the agent cancels the source watch and restarts
the scan from a fresh R0 (see the InitialSyncCompactionRaced event) —
a bounded retry instead of unbounded growth when source churn outruns
scan+apply throughput. Defaults to 16Mi (must stay in lockstep with
pkg/mirroragent's DefaultWatchBufferBytes). | | Optional: \{\}
| +| `maxOpsPerSecond` _integer_ | MaxOpsPerSecond rate-limits the agent's target write rate (a token
bucket over puts+deletes/sec), applied to both the genesis scan and
watch-driven applies. Zero (default) means unlimited. Mind the
retention prerequisite above when throttling. | | Minimum: 0
Optional: \{\}
| +| `requestTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | RequestTimeout is the per-RPC context deadline applied to every unary
call on both sides (watches excluded — they are long-lived by design
and covered by progress-notification liveness instead). Without it a
blackholed call through an NLB never errors and backoff never engages.
Defaults to 30s. | | Optional: \{\}
| +| `dialTimeout` _[Duration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#duration-v1-meta)_ | DialTimeout bounds establishing the initial client connection to each
side. Defaults to 10s. | | Optional: \{\}
| +| `reconnectBackoff` _[EtcdMirrorBackoffSpec](#etcdmirrorbackoffspec)_ | ReconnectBackoff bounds the retry/backoff loop wrapping connection-class
errors. Throttling-class errors (target rate rejection) use a more
conservative curve derived from the same bounds; quota exhaustion
(TargetQuotaExhausted) and permanent errors are never retried through
this loop. Defaults to exponential backoff from 1s to 30s. | | Optional: \{\}
| + + +#### EtcdMirrorTLS + + + +EtcdMirrorTLS configures the mirror agent's client TLS for one side of a +mirror. Plain secretRef, not a reuse of EtcdClusterTLS/TLSSurface — the +agent is always a client to clusters it does not own, so issuer-selection +machinery doesn't apply. + +ROTATION CONTRACT: the agent re-reads TLS material from the mounted Secret +on every handshake (transport.TLSInfo file paths, not a one-shot +tls.Config); certificate rotation requires no pod restart. + + + +_Appears in:_ +- [EtcdMirrorEndpoint](#etcdmirrorendpoint) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `secretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#localobjectreference-v1-core)_ | SecretRef names a Secret (in the EtcdMirror's namespace) holding this
side's TLS material in the standard kubernetes.io/tls-compatible shape:
- ca.crt: PEM CA bundle used to verify the peer's server certificate
(unless CABundleRef overrides it, or InsecureSkipVerify is true).
- tls.crt / tls.key: PEM client certificate + key, for mTLS.
Optional — omit both for server-auth-only TLS.
Nil means no client identity and verification against the system trust
roots (the etcdctl default). Note a source running with
--client-cert-auth (the RKE1 default) rejects certless clients at the
handshake regardless of etcd RBAC auth; server-auth-only + Auth is not
viable against such a source. | | Optional: \{\}
| +| `caBundleRef` _[EtcdMirrorCABundleRef](#etcdmirrorcabundleref)_ | CABundleRef optionally sources the trust anchors from a separate
Secret or ConfigMap key, decoupling trust from the identity Secret
(Gateway API caCertificateRefs precedent). Takes precedence over
SecretRef's ca.crt. | | Optional: \{\}
| +| `insecureSkipVerify` _boolean_ | InsecureSkipVerify disables server certificate verification. Strongly
discouraged, especially for a source reached over the public internet.
Requires InsecureSkipVerifyAcknowledgeRisk to also be set true
(CEL-enforced). The controller additionally emits a standing Warning
event whenever this is true. | false | Optional: \{\}
| +| `insecureSkipVerifyAcknowledgeRisk` _boolean_ | InsecureSkipVerifyAcknowledgeRisk must independently be set true
whenever InsecureSkipVerify is true (CEL-enforced companion field). Its
only purpose is to require a deliberate, separate, reviewable line in
the manifest diff before disabling TLS verification. | false | Optional: \{\}
| +| `serverName` _string_ | ServerName overrides the TLS ServerName (SNI) used for verification,
for cases where the dialed address doesn't match a SAN on the
certificate (e.g. dialing an NLB IP directly). Applies to EVERY
endpoint in the list, so mixing endpoints with different certificates
behind one ServerName will fail verification on the mismatched ones. | | Optional: \{\}
| + + #### MemberStatus @@ -153,6 +593,24 @@ _Appears in:_ | `labels` _object (keys:string, values:string)_ | | | | +#### PodSpec + + + + + + + +_Appears in:_ +- [PodTemplate](#podtemplate) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#affinity-v1-core)_ | | | | +| `nodeSelector` _object (keys:string, values:string)_ | | | | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#toleration-v1-core) array_ | | | | + + #### PodTemplate @@ -163,10 +621,12 @@ _Appears in:_ _Appears in:_ - [EtcdClusterSpec](#etcdclusterspec) +- [EtcdMirrorSpec](#etcdmirrorspec) | Field | Description | Default | Validation | | --- | --- | --- | --- | | `metadata` _[PodMetadata](#podmetadata)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PodSpec](#podspec)_ | | | | #### ProviderAutoConfig diff --git a/docs/etcdmirror.md b/docs/etcdmirror.md new file mode 100644 index 00000000..2c754511 --- /dev/null +++ b/docs/etcdmirror.md @@ -0,0 +1,141 @@ +# EtcdMirror + +`EtcdMirror` continuously copies a key range from a source etcd cluster into a +target etcd cluster, one way, as a single supervised stateless pod. Progress is +checkpointed in a reserved, fenced key **in the target etcd** — written in the +same transaction as every applied batch — so the agent has no volume and can be +rescheduled freely. + +It is a **byte-copy of keys and values, not a replica**. If you need a replica, +add members to the cluster; if you need a point-in-time copy with revision +fidelity, use `etcdutl snapshot restore`. + +## What is and is not preserved + +| Property | Preserved? | Notes | +| --- | --- | --- | +| Key names | yes | rewritten by one formula: `key' = target.prefix + destPrefix + TrimPrefix(key, source.prefix)` | +| Values | yes | byte-identical | +| Per-revision atomicity | yes | batches flush only at source-revision boundaries; a source revision is never split across target Txns | +| Revisions / mod_revision / create_revision | **no** | target-assigned. Stored fence tokens, persisted watch bookmarks, and CreateRevision-ordered elections do not survive mirroring | +| Version counters | **no** | target-assigned | +| Leases / TTLs | **no** | leases are stripped; a lease-backed source key becomes a permanent target key. Masked while the mirror runs (source expiry replicates as a delete); at cutover every in-flight leased key is immortal. `status.leaseBackedKeyCount` reports exposure; the cutover runbook includes a purge/re-lease step. Consider `sync.excludePrefixes` for lease-heavy ranges | +| Cross-key Txn atomicity | best-effort | whole revisions are coalesced; a revision larger than `maxTxnOps` is applied as one oversized Txn (provision the target's `--max-txn-ops`) | + +A fidelity-preserving alternative for migrations: seed the target with +`etcdutl snapshot restore --bump-revision --mark-compacted`, then mirror only +the delta using `initialSync.startRevision`. + +## Sync engine behavior + +- **Genesis (InitialSync):** unpinned chunked scan (one byte-bounded page in + flight) with the watch already open from the revision observed before the + scan; buffered events replay over the scanned base. Mid-scan compaction on + the source therefore cannot fail the scan. +- **Steady state:** watch with progress notifications; the checkpoint watermark + advances even when the mirrored prefix is idle. +- **Forced resync** (watch outrun by compaction — e.g. the mirror was down or + paused longer than the source's compaction retention): reported as + `Phase=InitialSync` with condition `Compacted=True/ForcedResync`, bracketed by + `ForcedResyncStarted`/`ForcedResyncCompleted` events. Every forced resync ends + with a mandatory mark-and-sweep prune, so deletes that happened during the + blind window do not resurrect. +- **Retention prerequisite:** source compaction retention must exceed the + worst-case scan + throttled drain time (roughly + `keyCount / min(scanRate, maxOpsPerSecond)`), or genesis/forced resyncs + livelock — surfaced as `ResyncLoopDetected`, which does not self-heal. +- **Checkpoint fencing:** the reserved key carries `{linkUID, epoch, role}` and + every write path (applies, reconciliation repairs, prune deletes) is fenced + with a mod-revision compare, so two agents can never interleave and a + straggler apply after cutover fails loudly. On a failed compare the engine + re-reads the fence — a Txn that committed while its response was lost (WAN + timeout) is recognized as this agent's own write and adopted, never + misreported as a fence violation. +- **Destination overlap guard:** a prune pass that finds ANOTHER link's + reserved fence key inside this link's destination prefix stops with a + permanent prefix-conflict error instead of deleting the sibling mirror's + fence and data. + +## Monitoring / paging algebra + +Page on `Available=False` sustained, **unless** `Compacted=True` and progress +fields are advancing (a forced resync healing itself). `TargetQuotaExhausted` +and `ResyncLoopDetected` page immediately — neither self-heals. + +Never compute lag as `status.sourceRevision - status.lastAppliedRevision`: +revisions are cluster-global, so out-of-prefix source writes inflate the +difference, and the two fields snapshot at different instants. + +The `InitialSyncCompactionRaced` event marks a genesis-scan attempt aborted +and restarted from a fresh revision. One event name, two causes named in the +message — do not conflate them: + +- `WatchBufferOverflow`: the replay buffer exceeded `sync.watchBufferBytes` + before the base scan completed. A memory-bound retry, **not** a compaction + race — raise `watchBufferBytes` or the scan rate for high-churn sources. +- `WatchCompactedMidScan`: a watch reconnect landed below the source compact + revision — the rare genuine race. + +Repeated occurrences of either count toward `ResyncLoopDetected`. + +`InvariantsHeld=True` means the verification invariants hold (lag within +threshold, per-side key counts equal, no drift, pass fresh) — it never means +"safe to cut over"; that is `CutoverReady`, which additionally requires +`spec.mode: Drain` and a reached drain target revision. + +## Operations: error taxonomy and runbook + +| Error | gRPC code | Class | Retry policy | Condition / Reason | +| --- | --- | --- | --- | --- | +| `ErrCompacted` on watch reopen | OutOfRange | Resync | forced resync (scan + mandatory prune); never generic retry | `Compacted=True/ForcedResync`; `forcedResyncCount`++ (`Compacted`) | +| `ErrNoSpace` | ResourceExhausted | Quota | park on slow flat timer; never hot-loop; recovers without genesis once operator compacts/defrags/disarms | `TargetQuotaExhausted=True` (pages immediately) | +| client send cap ("trying to send message larger than max") | ResourceExhausted | Permanent | never retried identically; redacted key surfaced | Failed if unavoidable | +| `ErrTooManyRequests` | ResourceExhausted | Throttle | conservative distinct curve | `TargetThrottled=True` | +| `ErrRequestTooLarge` | InvalidArgument | Permanent | one shrink attempt at revision granularity, else Failed | redacted key surfaced | +| `ErrTooManyOps` | InvalidArgument | Permanent | one shrink attempt at revision granularity; else raise target `--max-txn-ops` or lower `spec.sync.maxTxnOps` | Failed | +| Unavailable / `ErrNoLeader` | Unavailable | Transient | ReconnectBackoff curve | Source/TargetReachable=False (reason NoLeader when applicable) | +| DeadlineExceeded (requestTimeout) | DeadlineExceeded | Transient | ReconnectBackoff — the blackholed-NLB recovery path | reachability, reason RequestTimeout | +| auth token expiry | — | non-issue | clientv3 refreshes transparently | none | +| source version < 3.4 | — | Permanent | never | Failed/`UnsupportedVersion` | +| corrupt/unknown-version checkpoint | — | Permanent (fail closed) | never; operator deletes reserved key | Failed/`CheckpointInvalid` | +| linkUID / cluster-ID mismatch | — | expected transition | genesis + RequireEmpty re-arm | `CheckpointInvalidated` event | +| fence Compare loss (normal apply) | — | optimistic-concurrency loss | re-read, recompute, retry (jitter, not reconnect curve) | internal; persistent genesis-claim loss → FenceError (permanent) | +| N consecutive resyncs, no steady period | — | livelock (meta) | does not self-heal | `ResyncLoopDetected=True` (pages immediately) | + +## Cutover and reversal + +Two-way sync is out of scope permanently: etcd revisions are cluster-local and +there is no per-key provenance channel, so bidirectional sync is structurally +inexpressible without an application-visible format change. + +Cutover (forward): quiesce source writers, set `spec.mode: Drain`, then +`kubectl wait --for=condition=CutoverReady etcdmirror/`. The +`status.cutover` block records the drained revision, verification counts, and +the lease-backed key count for the purge/re-lease step. Once CutoverReady, the +fence key's role is Primary and any straggler mirror write fails its compare. + +Reversal (failback) is delete-and-recreate: delete the CR, create a new one +with swapped endpoints and `initialSync.mode: OverwriteAndPrune` — the +mandatory prune pass removes keys deleted on the new primary since cutover. +Only reverse after the forward mirror reached CutoverReady. + +## Prerequisites (summary) + +- Source etcd >= 3.4 (hard floor, probed at connect); >= 3.4.25 / 3.5.8 + recommended — below that, watch progress notifications are unreliable. +- Target credential range-scoped via etcd RBAC to the effective destination + prefix, **including the reserved checkpoint key** + (default `\x00etcdmirror-checkpoint`; an + override must stay under the effective destination prefix — CEL-enforced). +- Range-defining fields (`source.prefix`, `target.prefix`, `sync.destPrefix`, + `sync.excludePrefixes`, `checkpoint.key`) are immutable: the resume path + never re-scans, so an edited range silently diverges. Change them via + delete-and-recreate with an appropriate `initialSync.mode`. +- Target runs with auto-compaction enabled (resync churn otherwise marches the + default 2GiB quota toward NOSPACE / `TargetQuotaExhausted`). +- Source compaction retention satisfies the formula above for your key count + and rate limit. +- A source running `--client-cert-auth` (RKE1 default) rejects certless + clients at the handshake: supply a client certificate; username/password + auth alone is not viable there. When both are supplied, the token identity + wins and must hold the range-scoped role. diff --git a/internal/controller/etcdmirror_cel_test.go b/internal/controller/etcdmirror_cel_test.go new file mode 100644 index 00000000..56467c5c --- /dev/null +++ b/internal/controller/etcdmirror_cel_test.go @@ -0,0 +1,775 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// validSourceEndpoint and validTargetEndpoint are baseline endpoints that +// satisfy EtcdMirrorEndpoint's own CEL rules (oneOf, scheme-vs-TLS), so each +// test case only needs to perturb the field(s) it's actually exercising. The +// source endpoint is deliberately scheme-less so TLS blocks can be added or +// removed freely without tripping the scheme rules. +func validSourceEndpoint() ecv1alpha1.EtcdMirrorEndpoint { + return ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"etcd-source.example.com:2379"}, + Prefix: "/registry/", + } +} + +func validTargetEndpoint() ecv1alpha1.EtcdMirrorEndpoint { + return ecv1alpha1.EtcdMirrorEndpoint{ + ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-target-client"}, + Prefix: "/mirrored/", + } +} + +func newMirror(prefix string, spec ecv1alpha1.EtcdMirrorSpec) *ecv1alpha1.EtcdMirror { + return &ecv1alpha1.EtcdMirror{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: prefix, + Namespace: "default", + }, + Spec: spec, + } +} + +// createAndCheck applies the mirror and asserts admission matched wantApply, +// deleting it again on success. +func createAndCheck(t *testing.T, em *ecv1alpha1.EtcdMirror, wantApply bool, msg string) { + t.Helper() + err := k8sClient.Create(t.Context(), em) + if wantApply { + require.NoError(t, err, msg) + _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) + } else { + assert.Error(t, err, msg) + } +} + +// TestEtcdMirrorEndpointOneOfCELValidation drives the endpointList/serviceRef +// exactly-one-of XValidation rule on EtcdMirrorEndpoint, exercised on both +// Source and Target. An empty endpointList is deliberately treated as unset +// (k8s list conventions), so [] alongside a serviceRef must be ACCEPTED. +func TestEtcdMirrorEndpointOneOfCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + neither := ecv1alpha1.EtcdMirrorEndpoint{Prefix: "/x/"} + both := ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"etcd.example.com:2379"}, + ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-client"}, + } + emptyListOnly := ecv1alpha1.EtcdMirrorEndpoint{EndpointList: []string{}} + emptyListWithServiceRef := ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{}, + ServiceRef: &ecv1alpha1.EtcdMirrorServiceRef{Name: "etcd-client"}, + } + + tests := []struct { + name string + source ecv1alpha1.EtcdMirrorEndpoint + target ecv1alpha1.EtcdMirrorEndpoint + wantApply bool + }{ + { + name: "valid endpointList source, serviceRef target accepted", + source: validSourceEndpoint(), + target: validTargetEndpoint(), + wantApply: true, + }, + { + name: "source with neither endpointList nor serviceRef rejected", + source: neither, + target: validTargetEndpoint(), + wantApply: false, + }, + { + name: "source with both endpointList and serviceRef rejected", + source: both, + target: validTargetEndpoint(), + wantApply: false, + }, + { + name: "source with empty endpointList and no serviceRef rejected", + source: emptyListOnly, + target: validTargetEndpoint(), + wantApply: false, + }, + { + // Pinned deliberately: empty list == unset, so this is the + // serviceRef-only case, not the both-set case. + name: "source with empty endpointList plus serviceRef accepted", + source: emptyListWithServiceRef, + target: validTargetEndpoint(), + wantApply: true, + }, + { + name: "target with neither endpointList nor serviceRef rejected", + source: validSourceEndpoint(), + target: neither, + wantApply: false, + }, + { + name: "target with both endpointList and serviceRef rejected", + source: validSourceEndpoint(), + target: both, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-endpoint-", ecv1alpha1.EtcdMirrorSpec{ + Source: tt.source, + Target: tt.target, + }) + createAndCheck(t, em, tt.wantApply, "endpointList/serviceRef oneOf") + }) + } +} + +// TestEtcdMirrorEndpointSchemeTLSCELValidation drives the two scheme-vs-TLS +// XValidation rules on EtcdMirrorEndpoint: http:// forbids a tls block, +// https:// requires one (an empty tls block means system trust roots). +func TestEtcdMirrorEndpointSchemeTLSCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tlsBlock := &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: &corev1.LocalObjectReference{Name: "etcd-mirror-tls"}, + } + + tests := []struct { + name string + endpoints []string + tls *ecv1alpha1.EtcdMirrorTLS + wantApply bool + }{ + { + name: "https endpoint with tls block accepted", + endpoints: []string{"https://etcd.example.com:2379"}, + tls: tlsBlock, + wantApply: true, + }, + { + name: "https endpoint with empty tls block (system roots) accepted", + endpoints: []string{"https://etcd.example.com:2379"}, + tls: &ecv1alpha1.EtcdMirrorTLS{}, + wantApply: true, + }, + { + name: "https endpoint without tls block rejected", + endpoints: []string{"https://etcd.example.com:2379"}, + tls: nil, + wantApply: false, + }, + { + name: "http endpoint without tls block accepted", + endpoints: []string{"http://etcd.example.com:2379"}, + tls: nil, + wantApply: true, + }, + { + name: "http endpoint with tls block rejected", + endpoints: []string{"http://etcd.example.com:2379"}, + tls: tlsBlock, + wantApply: false, + }, + { + name: "mixed http and https endpoints with tls block rejected", + endpoints: []string{"https://a.example.com:2379", "http://b.example.com:2379"}, + tls: tlsBlock, + wantApply: false, + }, + { + name: "scheme-less endpoint with tls block accepted", + endpoints: []string{"etcd.example.com:2379"}, + tls: tlsBlock, + wantApply: true, + }, + { + name: "scheme-less endpoint without tls block accepted", + endpoints: []string{"etcd.example.com:2379"}, + tls: nil, + wantApply: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: tt.endpoints, + Prefix: "/registry/", + TLS: tt.tls, + } + em := newMirror("cel-mirror-scheme-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "scheme-vs-TLS rule on source") + }) + } + + t.Run("scheme rules also apply to target", func(t *testing.T) { + em := newMirror("cel-mirror-scheme-", ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: ecv1alpha1.EtcdMirrorEndpoint{ + EndpointList: []string{"https://etcd-target.example.com:2379"}, + Prefix: "/mirrored/", + }, + }) + createAndCheck(t, em, false, "https target without tls must be rejected") + }) +} + +// TestEtcdMirrorTLSInsecureSkipVerifyCELValidation drives the +// insecureSkipVerify/insecureSkipVerifyAcknowledgeRisk companion-field +// XValidation rule on EtcdMirrorTLS. +func TestEtcdMirrorTLSInsecureSkipVerifyCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + secretRef := &corev1.LocalObjectReference{Name: "etcd-mirror-tls"} + + tests := []struct { + name string + tls *ecv1alpha1.EtcdMirrorTLS + wantApply bool + }{ + { + name: "no TLS block accepted", + tls: nil, + wantApply: true, + }, + { + name: "TLS with verification enabled accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: secretRef}, + wantApply: true, + }, + { + name: "insecureSkipVerify with acknowledgement accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: secretRef, + InsecureSkipVerify: true, + InsecureSkipVerifyAcknowledgeRisk: true, + }, + wantApply: true, + }, + { + name: "insecureSkipVerify with acknowledgement and no secretRef accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{ + InsecureSkipVerify: true, + InsecureSkipVerifyAcknowledgeRisk: true, + }, + wantApply: true, + }, + { + name: "insecureSkipVerify without acknowledgement rejected", + tls: &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: secretRef, + InsecureSkipVerify: true, + }, + wantApply: false, + }, + { + name: "acknowledgement without insecureSkipVerify accepted (not the risky case)", + tls: &ecv1alpha1.EtcdMirrorTLS{ + SecretRef: secretRef, + InsecureSkipVerifyAcknowledgeRisk: true, + }, + wantApply: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := validSourceEndpoint() + source.TLS = tt.tls + em := newMirror("cel-mirror-tls-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "insecureSkipVerify companion rule") + }) + } +} + +// TestEtcdMirrorSecretRefCELValidation drives the secretRef rules: on TLS the +// secretRef is optional (nil = system trust roots) but must have a non-empty +// name when present; on Auth it is required with a non-empty name. +func TestEtcdMirrorSecretRefCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + tls *ecv1alpha1.EtcdMirrorTLS + auth *ecv1alpha1.EtcdMirrorAuth + wantApply bool + }{ + { + name: "TLS secretRef with non-empty name accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: &corev1.LocalObjectReference{Name: "etcd-mirror-tls"}}, + wantApply: true, + }, + { + name: "TLS with nil secretRef (system trust roots) accepted", + tls: &ecv1alpha1.EtcdMirrorTLS{}, + wantApply: true, + }, + { + name: "TLS secretRef with empty name rejected", + tls: &ecv1alpha1.EtcdMirrorTLS{SecretRef: &corev1.LocalObjectReference{Name: ""}}, + wantApply: false, + }, + { + name: "Auth secretRef with non-empty name accepted", + auth: &ecv1alpha1.EtcdMirrorAuth{SecretRef: corev1.LocalObjectReference{Name: "etcd-mirror-auth"}}, + wantApply: true, + }, + { + name: "Auth secretRef with empty name rejected", + auth: &ecv1alpha1.EtcdMirrorAuth{SecretRef: corev1.LocalObjectReference{Name: ""}}, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := validSourceEndpoint() + source.TLS = tt.tls + source.Auth = tt.auth + em := newMirror("cel-mirror-secretref-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "secretRef name rules") + }) + } +} + +// TestEtcdMirrorCABundleRefValidation drives EtcdMirrorCABundleRef's schema +// validation (kind enum, required name). +func TestEtcdMirrorCABundleRefValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + ref *ecv1alpha1.EtcdMirrorCABundleRef + wantApply bool + }{ + { + name: "configMap caBundleRef accepted", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "ConfigMap", Name: "mirror-trust"}, + wantApply: true, + }, + { + name: "secret caBundleRef with key accepted", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "Secret", Name: "mirror-trust", Key: "bundle.pem"}, + wantApply: true, + }, + { + name: "caBundleRef with empty name rejected", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "ConfigMap", Name: ""}, + wantApply: false, + }, + { + name: "caBundleRef with bogus kind rejected", + ref: &ecv1alpha1.EtcdMirrorCABundleRef{Kind: "DaemonSet", Name: "mirror-trust"}, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + source := validSourceEndpoint() + source.TLS = &ecv1alpha1.EtcdMirrorTLS{CABundleRef: tt.ref} + em := newMirror("cel-mirror-cabundle-", ecv1alpha1.EtcdMirrorSpec{ + Source: source, + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "caBundleRef schema rules") + }) + } +} + +// TestEtcdMirrorInitialSyncCELValidation drives the initialSync rules: the +// mode enum, and the startRevision-requires-Overwrite guard (a target seeded +// via snapshot restore cannot also be required empty). +func TestEtcdMirrorInitialSyncCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + initialSync *ecv1alpha1.EtcdMirrorInitialSyncSpec + wantApply bool + }{ + { + name: "nil initialSync accepted", + initialSync: nil, + wantApply: true, + }, + { + name: "explicit RequireEmpty accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: ecv1alpha1.EtcdMirrorInitialSyncRequireEmpty}, + wantApply: true, + }, + { + name: "Overwrite accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwrite}, + wantApply: true, + }, + { + name: "OverwriteAndPrune accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwriteAndPrune}, + wantApply: true, + }, + { + name: "bogus mode rejected by enum", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{Mode: "TruncateFirst"}, + wantApply: false, + }, + { + name: "startRevision with Overwrite accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwrite, + StartRevision: 42, + }, + wantApply: true, + }, + { + name: "startRevision with OverwriteAndPrune accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwriteAndPrune, + StartRevision: 42, + }, + wantApply: true, + }, + { + name: "startRevision with explicit RequireEmpty rejected", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncRequireEmpty, + StartRevision: 42, + }, + wantApply: false, + }, + { + name: "startRevision with defaulted mode (RequireEmpty) rejected", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + StartRevision: 42, + }, + wantApply: false, + }, + { + name: "startRevision zero with RequireEmpty accepted", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncRequireEmpty, + StartRevision: 0, + }, + wantApply: true, + }, + { + name: "negative startRevision rejected by minimum", + initialSync: &ecv1alpha1.EtcdMirrorInitialSyncSpec{ + Mode: ecv1alpha1.EtcdMirrorInitialSyncOverwrite, + StartRevision: -1, + }, + wantApply: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-initialsync-", ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + InitialSync: tt.initialSync, + }) + createAndCheck(t, em, tt.wantApply, "initialSync rules") + }) + } +} + +// TestEtcdMirrorModeValidation drives the spec.mode enum. +func TestEtcdMirrorModeValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + mode ecv1alpha1.EtcdMirrorMode + wantApply bool + }{ + {name: "mode unset accepted (defaults to Sync)", mode: "", wantApply: true}, + {name: "mode Sync accepted", mode: ecv1alpha1.EtcdMirrorModeSync, wantApply: true}, + {name: "mode Drain accepted", mode: ecv1alpha1.EtcdMirrorModeDrain, wantApply: true}, + {name: "bogus mode rejected", mode: "Bidirectional", wantApply: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-mode-", ecv1alpha1.EtcdMirrorSpec{ + Mode: tt.mode, + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + }) + createAndCheck(t, em, tt.wantApply, "spec.mode enum") + }) + } +} + +// TestEtcdMirrorImmutabilityCELValidation drives the CEL transition rules: +// source.prefix, target.prefix, sync.destPrefix, sync.excludePrefixes and +// checkpoint.key are immutable; endpoints (and ordinary fields) stay mutable +// — same-cluster endpoint rotation is routine and cross-cluster repoints are +// caught at runtime by the checkpoint's cluster-ID binding, not by spec +// validation. +func TestEtcdMirrorImmutabilityCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + baseSpec := func() ecv1alpha1.EtcdMirrorSpec { + return ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Sync: ecv1alpha1.EtcdMirrorSyncSpec{ + DestPrefix: "registry/", + ExcludePrefixes: []string{"/registry/events/"}, + }, + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + // Must live under the effective destination prefix + // (target.prefix "/mirrored/" + destPrefix "registry/"). + Key: "/mirrored/registry/\x00etcdmirror-checkpoint", + }, + } + } + + tests := []struct { + name string + mutate func(em *ecv1alpha1.EtcdMirror) + wantUpdate bool + }{ + { + name: "changing source.prefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Source.Prefix = "/other/" }, + wantUpdate: false, + }, + { + name: "unsetting source.prefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Source.Prefix = "" }, + wantUpdate: false, + }, + { + name: "changing target.prefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Target.Prefix = "/elsewhere/" }, + wantUpdate: false, + }, + { + name: "changing sync.destPrefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.DestPrefix = "moved/" }, + wantUpdate: false, + }, + { + name: "unsetting sync.destPrefix rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.DestPrefix = "" }, + wantUpdate: false, + }, + { + name: "changing checkpoint.key rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Checkpoint.Key = "/mirrored/registry/\x00other" }, + wantUpdate: false, + }, + { + name: "changing sync.excludePrefixes rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Sync.ExcludePrefixes = []string{"/registry/leases/"} + }, + wantUpdate: false, + }, + { + name: "removing sync.excludePrefixes rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.ExcludePrefixes = nil }, + wantUpdate: false, + }, + { + name: "removing checkpoint block (unsetting key) rejected", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Checkpoint = nil }, + wantUpdate: false, + }, + { + name: "changing source endpoints accepted (endpoints deliberately mutable)", + mutate: func(em *ecv1alpha1.EtcdMirror) { + em.Spec.Source.EndpointList = []string{"etcd-source-b.example.com:2379", "etcd-source-c.example.com:2379"} + }, + wantUpdate: true, + }, + { + name: "changing paused accepted", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Paused = true }, + wantUpdate: true, + }, + { + name: "changing sync.maxOpsPerSecond accepted", + mutate: func(em *ecv1alpha1.EtcdMirror) { em.Spec.Sync.MaxOpsPerSecond = 250 }, + wantUpdate: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-immutable-", baseSpec()) + require.NoError(t, k8sClient.Create(t.Context(), em), "baseline mirror must be accepted") + defer func() { _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) }() + + tt.mutate(em) + err := k8sClient.Update(t.Context(), em) + if tt.wantUpdate { + assert.NoError(t, err, "update should be accepted") + } else { + assert.Error(t, err, "update should be rejected by transition rule") + } + }) + } + + t.Run("setting sync.destPrefix from unset rejected", func(t *testing.T) { + spec := baseSpec() + spec.Sync = ecv1alpha1.EtcdMirrorSyncSpec{} + spec.Checkpoint = nil + em := newMirror("cel-mirror-immutable-", spec) + require.NoError(t, k8sClient.Create(t.Context(), em)) + defer func() { _ = k8sClient.Delete(t.Context(), em, &client.DeleteOptions{}) }() + + em.Spec.Sync.DestPrefix = "late/" + assert.Error(t, k8sClient.Update(t.Context(), em), "unset -> set is still a destPrefix mutation") + }) + + // Present-empty and absent are the same VALUE for these fields (and Go + // typed clients drop "" through omitempty), so a CR created from YAML + // with an explicit destPrefix: "" must stay updatable via typed clients + // — presence-based transition rules would 422 every such update on a + // field it never touched. + t.Run("explicit empty destPrefix stays typed-client updatable", func(t *testing.T) { + u := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "operator.etcd.io/v1alpha1", + "kind": "EtcdMirror", + "metadata": map[string]interface{}{ + "generateName": "cel-mirror-emptydest-", + "namespace": "default", + }, + "spec": map[string]interface{}{ + "source": map[string]interface{}{ + "endpointList": []interface{}{"etcd-source.example.com:2379"}, + "prefix": "/registry/", + }, + "target": map[string]interface{}{ + "serviceRef": map[string]interface{}{"name": "etcd-target-client"}, + "prefix": "/mirrored/", + }, + "sync": map[string]interface{}{ + "destPrefix": "", // stored present-but-empty + }, + }, + }} + require.NoError(t, k8sClient.Create(t.Context(), u), "explicit empty destPrefix must be accepted") + defer func() { _ = k8sClient.Delete(t.Context(), u, &client.DeleteOptions{}) }() + + em := &ecv1alpha1.EtcdMirror{} + require.NoError(t, k8sClient.Get(t.Context(), + client.ObjectKey{Namespace: "default", Name: u.GetName()}, em)) + em.Spec.Paused = true // typed round-trip drops destPrefix to absent + assert.NoError(t, k8sClient.Update(t.Context(), em), + "a typed-client update must not be rejected for a field it never touched") + }) +} + +// TestEtcdMirrorCheckpointKeyCELValidation drives the create-time rule that +// checkpoint.key must live under the effective destination prefix: the +// engine rejects anything else permanently at first start, and the key's own +// immutability would otherwise make the Failed CR unrepairable in place. +func TestEtcdMirrorCheckpointKeyCELValidation(t *testing.T) { + if k8sClient == nil { + t.Skip("envtest apiserver not available") + } + + tests := []struct { + name string + spec ecv1alpha1.EtcdMirrorSpec + wantApply bool + }{ + { + name: "key under effective destination prefix accepted", + spec: ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + Key: "/mirrored/\x00my-checkpoint", + }, + }, + wantApply: true, + }, + { + name: "key outside destination prefix rejected", + spec: ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + Key: "/checkpoints/mirror-a", + }, + }, + wantApply: false, + }, + { + name: "key under target.prefix but outside destPrefix rejected", + spec: ecv1alpha1.EtcdMirrorSpec{ + Source: validSourceEndpoint(), + Target: validTargetEndpoint(), + Sync: ecv1alpha1.EtcdMirrorSyncSpec{DestPrefix: "registry/"}, + Checkpoint: &ecv1alpha1.EtcdMirrorCheckpointSpec{ + Key: "/mirrored/\x00etcdmirror-checkpoint", + }, + }, + wantApply: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + em := newMirror("cel-mirror-ckptkey-", tt.spec) + createAndCheck(t, em, tt.wantApply, "checkpoint.key placement") + }) + } +}