diff --git a/Dockerfile b/Dockerfile index cecd8dbf..9bf1373b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ COPY go.sum go.sum RUN go mod download # Copy the go source -COPY cmd/main.go cmd/main.go +COPY cmd/ cmd/ COPY api/ api/ COPY internal/ internal/ COPY pkg/ pkg/ @@ -22,7 +22,7 @@ COPY pkg/ pkg/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager ./cmd # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/Makefile b/Makefile index 05d9ad1d..518d09ed 100644 --- a/Makefile +++ b/Makefile @@ -126,11 +126,11 @@ verify: verify-mod-tidy lint ## Run static checks against the code. .PHONY: build build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go + go build -o bin/manager ./cmd .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. - go run ./cmd/main.go + go run ./cmd # If you wish to build the manager image targeting other platforms you can use the --platform flag. # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. diff --git a/PROJECT b/PROJECT index f70bcb68..eeba0429 100644 --- a/PROJECT +++ b/PROJECT @@ -17,4 +17,13 @@ resources: kind: EtcdCluster path: go.etcd.io/etcd-operator/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: etcd.io + group: operator + kind: EtcdBackup + path: go.etcd.io/etcd-operator/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/v1alpha1/etcdbackup_types.go b/api/v1alpha1/etcdbackup_types.go new file mode 100644 index 00000000..2045b83f --- /dev/null +++ b/api/v1alpha1/etcdbackup_types.go @@ -0,0 +1,225 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// BackupProvider enumerates the supported object-storage backends a snapshot +// can be uploaded to. The set is intentionally open: the controller dispatches +// on this value to a pluggable provider implementation in pkg/objectstore, so +// adding a new backend is a matter of registering a new provider and a new +// value here. +// +kubebuilder:validation:Enum=s3;gcs +type BackupProvider string + +const ( + // BackupProviderS3 uploads the snapshot to an AWS S3 (or S3-compatible) + // bucket via the aws-sdk-go-v2 provider. + BackupProviderS3 BackupProvider = "s3" + // BackupProviderGCS uploads the snapshot to a Google Cloud Storage bucket + // via the cloud.google.com/go/storage provider. + BackupProviderGCS BackupProvider = "gcs" +) + +// BackupPhase is a high-level summary of where an EtcdBackup is in its +// lifecycle. It is surfaced on .status.phase for at-a-glance reporting. +type BackupPhase string + +const ( + // BackupPhasePending means the backup has been accepted but work has not + // started yet. + BackupPhasePending BackupPhase = "Pending" + // BackupPhaseSnapshotting means a snapshot is being taken from a member. + BackupPhaseSnapshotting BackupPhase = "Snapshotting" + // BackupPhaseUploading means the snapshot is being uploaded to object storage. + BackupPhaseUploading BackupPhase = "Uploading" + // BackupPhaseCompleted means the snapshot was uploaded successfully. + BackupPhaseCompleted BackupPhase = "Completed" + // BackupPhaseFailed means the backup failed; see conditions for details. + BackupPhaseFailed BackupPhase = "Failed" +) + +// Condition types reported on EtcdBackup status. +const ( + // BackupConditionSucceeded is True when the snapshot has been taken and + // uploaded to the destination object store. + BackupConditionSucceeded = "Succeeded" +) + +// S3DestinationSpec describes an AWS S3 (or S3-compatible) upload target. +type S3DestinationSpec struct { + // Bucket is the destination S3 bucket name. + // +kubebuilder:validation:MinLength=1 + Bucket string `json:"bucket"` + + // Region is the AWS region the bucket resides in (e.g. "us-east-1"). + // +optional + Region string `json:"region,omitempty"` + + // Endpoint overrides the S3 endpoint, enabling S3-compatible stores such + // as MinIO or Ceph RGW. If empty, the default AWS endpoint for the region + // is used. + // +optional + Endpoint string `json:"endpoint,omitempty"` + + // ForcePathStyle forces path-style addressing (bucket in the path rather + // than the host). Required by most S3-compatible stores. + // +optional + ForcePathStyle bool `json:"forcePathStyle,omitempty"` +} + +// GCSDestinationSpec describes a Google Cloud Storage upload target. +type GCSDestinationSpec struct { + // Bucket is the destination GCS bucket name. + // +kubebuilder:validation:MinLength=1 + Bucket string `json:"bucket"` + + // Endpoint overrides the GCS endpoint, enabling GCS-compatible emulators + // such as fake-gcs-server or the gcloud storage testbench for hermetic, + // credential-free testing. It must point at the JSON API root the emulator + // serves (e.g. "http://fake-gcs:9000/storage/v1/"). When set, the client is + // pointed at this endpoint and runs unauthenticated, mirroring the S3 + // endpoint override that targets MinIO. If empty, the real Google endpoint + // and the normal credential chain are used. + // +optional + Endpoint string `json:"endpoint,omitempty"` +} + +// BackupDestination describes where a snapshot is uploaded. Exactly one +// provider-specific block must be populated and it must match Provider. +type BackupDestination struct { + // Provider selects the object-storage backend. + Provider BackupProvider `json:"provider"` + + // Prefix is an optional key prefix (a.k.a. "folder") within the bucket + // under which the snapshot object is written. A trailing slash is optional. + // +optional + Prefix string `json:"prefix,omitempty"` + + // SecretRef references a Secret in the EtcdBackup's namespace holding the + // credentials for the provider. The expected keys depend on the provider: + // - s3: accessKeyID / secretAccessKey (and optionally sessionToken) + // - gcs: serviceAccountJSON (a GCP service-account key) + // If unset, the controller falls back to ambient credentials available to + // the operator pod (e.g. IRSA / Workload Identity), which is the + // recommended production posture. + // +optional + SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"` + + // S3 holds S3-specific destination configuration. Required when + // Provider is "s3". + // +optional + S3 *S3DestinationSpec `json:"s3,omitempty"` + + // GCS holds GCS-specific destination configuration. Required when + // Provider is "gcs". + // +optional + GCS *GCSDestinationSpec `json:"gcs,omitempty"` +} + +// EtcdBackupSpec defines the desired state of an EtcdBackup. +type EtcdBackupSpec struct { + // ClusterRef references the EtcdCluster to snapshot. The cluster must live + // in the same namespace as this EtcdBackup. + // +kubebuilder:validation:MinLength=1 + ClusterRef string `json:"clusterRef"` + + // Destination describes the object-storage target for the snapshot. + Destination BackupDestination `json:"destination"` + + // SnapshotTimeout bounds how long the snapshot-save step may run before it + // is considered failed. Defaults to 10m if unset. + // +optional + SnapshotTimeout *metav1.Duration `json:"snapshotTimeout,omitempty"` + + // Retention, when set, asks the controller to delete older snapshots under + // the same bucket/prefix once more than RetainCount snapshots exist. A + // value of 0 (the default) disables retention pruning. + // +optional + Retention *RetentionPolicy `json:"retention,omitempty"` +} + +// RetentionPolicy controls automatic pruning of old snapshots. +type RetentionPolicy struct { + // RetainCount is the number of most-recent snapshots to keep under the + // destination bucket/prefix. Older snapshots are deleted after a + // successful upload. Zero disables pruning. + // +kubebuilder:validation:Minimum=0 + RetainCount int32 `json:"retainCount,omitempty"` +} + +// EtcdBackupStatus defines the observed state of an EtcdBackup. +type EtcdBackupStatus struct { + // ObservedGeneration is the most recent generation observed by the controller. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Phase is a high-level summary of the backup lifecycle. + // +optional + Phase BackupPhase `json:"phase,omitempty"` + + // SnapshotLocation is the fully-qualified URI of the uploaded snapshot + // (e.g. "s3://my-bucket/etcd/backup-...db" or "gs://my-bucket/..."). + // +optional + SnapshotLocation string `json:"snapshotLocation,omitempty"` + + // SnapshotSizeBytes is the size of the uploaded snapshot in bytes. + // +optional + SnapshotSizeBytes int64 `json:"snapshotSizeBytes,omitempty"` + + // CompletionTime is when the snapshot finished uploading successfully. + // +optional + CompletionTime *metav1.Time `json:"completionTime,omitempty"` + + // Conditions represent the latest available observations of the backup's state. + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Cluster",type=string,JSONPath=`.spec.clusterRef` +// +kubebuilder:printcolumn:name="Provider",type=string,JSONPath=`.spec.destination.provider` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Location",type=string,JSONPath=`.status.snapshotLocation` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// EtcdBackup is the Schema for the etcdbackups API. It represents a single +// point-in-time snapshot of an EtcdCluster uploaded to object storage. +type EtcdBackup struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec EtcdBackupSpec `json:"spec,omitempty"` + Status EtcdBackupStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// EtcdBackupList contains a list of EtcdBackup. +type EtcdBackupList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EtcdBackup `json:"items"` +} diff --git a/api/v1alpha1/etcdrestore_types.go b/api/v1alpha1/etcdrestore_types.go new file mode 100644 index 00000000..ebc73c4a --- /dev/null +++ b/api/v1alpha1/etcdrestore_types.go @@ -0,0 +1,198 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// RestorePhase is a high-level summary of where an EtcdRestore is in its +// lifecycle. It is surfaced on .status.phase for at-a-glance reporting. +type RestorePhase string + +const ( + // RestorePhasePending means the restore has been accepted but work has not + // started yet. + RestorePhasePending RestorePhase = "Pending" + // RestorePhaseDownloading means the snapshot is being fetched from object + // storage. + RestorePhaseDownloading RestorePhase = "Downloading" + // RestorePhaseRestoring means the snapshot is being written into a fresh + // data directory on the target member and the new cluster is bootstrapping. + RestorePhaseRestoring RestorePhase = "Restoring" + // RestorePhaseCompleted means the snapshot was restored into the target + // cluster successfully. + RestorePhaseCompleted RestorePhase = "Completed" + // RestorePhaseFailed means the restore failed; see conditions for details. + RestorePhaseFailed RestorePhase = "Failed" +) + +// Condition types reported on EtcdRestore status. +const ( + // RestoreConditionSucceeded is True when the snapshot has been downloaded + // and restored into the target cluster. + RestoreConditionSucceeded = "Succeeded" +) + +// SnapshotSource describes where the snapshot to restore comes from. Exactly +// one of BackupRef or Location must be set; the controller rejects a source +// that sets both or neither. +type SnapshotSource struct { + // BackupRef names a completed EtcdBackup in the same namespace whose + // uploaded snapshot should be restored. The controller reads that backup's + // destination (bucket/prefix/provider/secretRef) and recorded + // snapshotLocation, so this is the convenient path when restoring a backup + // the operator itself produced. + // +optional + BackupRef *BackupReference `json:"backupRef,omitempty"` + + // Location fully describes a snapshot object independently of any + // EtcdBackup resource. Use this to restore a snapshot taken out-of-band or + // after the originating EtcdBackup has been deleted. + // +optional + Location *SnapshotLocation `json:"location,omitempty"` +} + +// BackupReference points at an EtcdBackup in the same namespace. +type BackupReference struct { + // Name is the metadata.name of the EtcdBackup to restore from. + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` +} + +// SnapshotLocation describes an explicit snapshot object in object storage. It +// reuses BackupDestination (provider, bucket, prefix, secretRef) and adds the +// object key relative to the destination prefix. +type SnapshotLocation struct { + // Destination is the object-storage location (provider, bucket, prefix and + // optional secretRef) the snapshot lives in. The same secretRef semantics + // as EtcdBackup apply: omit it to use ambient credentials. + Destination BackupDestination `json:"destination"` + + // Key is the object key of the snapshot, relative to the destination + // Prefix (e.g. "my-cluster/backup-1-20260617T010203Z.db"). It is joined + // with the destination prefix exactly as the backup path joins them, so a + // value copied verbatim from an EtcdBackup's status round-trips. + // +kubebuilder:validation:MinLength=1 + Key string `json:"key"` +} + +// RestoreTarget describes the new EtcdCluster the snapshot is restored into. A +// restore always targets a fresh, empty cluster: the restored data directory +// must be the cluster's genesis, never overlaid onto an existing member, so the +// controller refuses to proceed if a non-empty EtcdCluster of this name already +// exists. +type RestoreTarget struct { + // Name is the metadata.name of the EtcdCluster to create (in the + // EtcdRestore's namespace) and restore into. It must not already exist + // unless it exists and reports zero ready members (an empty shell). + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Size is the desired size of the restored cluster. Defaults to 1 if unset; + // a single-member restore is the safest default because etcd's snapshot + // restore bootstraps a single-node cluster which is then grown. + // +optional + // +kubebuilder:validation:Minimum=1 + Size int `json:"size,omitempty"` + + // Version is the etcd version for the restored cluster. If empty, the + // controller inherits the version recorded on the source EtcdBackup (when + // restoring via backupRef) or requires it to be set explicitly. + // +optional + Version string `json:"version,omitempty"` + + // StorageSpec optionally requests persistent storage for the restored + // cluster, mirroring EtcdClusterSpec.StorageSpec. When omitted the restored + // cluster uses ephemeral container storage. + // +optional + StorageSpec *StorageSpec `json:"storageSpec,omitempty"` +} + +// EtcdRestoreSpec defines the desired state of an EtcdRestore: take a snapshot +// from object storage and bootstrap a new EtcdCluster from it. +type EtcdRestoreSpec struct { + // Source selects the snapshot to restore. + Source SnapshotSource `json:"source"` + + // Target describes the EtcdCluster to create and restore into. + Target RestoreTarget `json:"target"` + + // RestoreTimeout bounds how long the download+restore step may run before it + // is considered failed. Defaults to 10m if unset. + // +optional + RestoreTimeout *metav1.Duration `json:"restoreTimeout,omitempty"` +} + +// EtcdRestoreStatus defines the observed state of an EtcdRestore. +type EtcdRestoreStatus struct { + // ObservedGeneration is the most recent generation observed by the controller. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Phase is a high-level summary of the restore lifecycle. + // +optional + Phase RestorePhase `json:"phase,omitempty"` + + // SnapshotLocation is the fully-qualified URI of the snapshot that was + // restored (e.g. "s3://my-bucket/etcd/backups/...db"). + // +optional + SnapshotLocation string `json:"snapshotLocation,omitempty"` + + // RestoredCluster is the name of the EtcdCluster the snapshot was restored + // into, once created. + // +optional + RestoredCluster string `json:"restoredCluster,omitempty"` + + // CompletionTime is when the restore finished successfully. + // +optional + CompletionTime *metav1.Time `json:"completionTime,omitempty"` + + // Conditions represent the latest available observations of the restore's state. + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Target",type=string,JSONPath=`.spec.target.name` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Source",type=string,JSONPath=`.status.snapshotLocation` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// EtcdRestore is the Schema for the etcdrestores API. It represents restoring a +// snapshot from object storage into a new EtcdCluster. +type EtcdRestore struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec EtcdRestoreSpec `json:"spec,omitempty"` + Status EtcdRestoreStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// EtcdRestoreList contains a list of EtcdRestore. +type EtcdRestoreList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EtcdRestore `json:"items"` +} diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go index 9a178715..183c9581 100644 --- a/api/v1alpha1/groupversion_info.go +++ b/api/v1alpha1/groupversion_info.go @@ -40,6 +40,10 @@ func addKnownTypes(s *runtime.Scheme) error { s.AddKnownTypes(GroupVersion, &EtcdCluster{}, &EtcdClusterList{}, + &EtcdBackup{}, + &EtcdBackupList{}, + &EtcdRestore{}, + &EtcdRestoreList{}, ) metav1.AddToGroupVersion(s, GroupVersion) return nil diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9db6ca12..46231872 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" netx "net" ) @@ -58,6 +58,51 @@ func (in *AltNames) DeepCopy() *AltNames { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupDestination) DeepCopyInto(out *BackupDestination) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.S3 != nil { + in, out := &in.S3, &out.S3 + *out = new(S3DestinationSpec) + **out = **in + } + if in.GCS != nil { + in, out := &in.GCS, &out.GCS + *out = new(GCSDestinationSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupDestination. +func (in *BackupDestination) DeepCopy() *BackupDestination { + if in == nil { + return nil + } + out := new(BackupDestination) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupReference) DeepCopyInto(out *BackupReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupReference. +func (in *BackupReference) DeepCopy() *BackupReference { + if in == nil { + return nil + } + out := new(BackupReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CommonConfig) DeepCopyInto(out *CommonConfig) { *out = *in @@ -79,6 +124,117 @@ func (in *CommonConfig) DeepCopy() *CommonConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdBackup) DeepCopyInto(out *EtcdBackup) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdBackup. +func (in *EtcdBackup) DeepCopy() *EtcdBackup { + if in == nil { + return nil + } + out := new(EtcdBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdBackup) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdBackupList) DeepCopyInto(out *EtcdBackupList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EtcdBackup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdBackupList. +func (in *EtcdBackupList) DeepCopy() *EtcdBackupList { + if in == nil { + return nil + } + out := new(EtcdBackupList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdBackupList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdBackupSpec) DeepCopyInto(out *EtcdBackupSpec) { + *out = *in + in.Destination.DeepCopyInto(&out.Destination) + if in.SnapshotTimeout != nil { + in, out := &in.SnapshotTimeout, &out.SnapshotTimeout + *out = new(metav1.Duration) + **out = **in + } + if in.Retention != nil { + in, out := &in.Retention, &out.Retention + *out = new(RetentionPolicy) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdBackupSpec. +func (in *EtcdBackupSpec) DeepCopy() *EtcdBackupSpec { + if in == nil { + return nil + } + out := new(EtcdBackupSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdBackupStatus) DeepCopyInto(out *EtcdBackupStatus) { + *out = *in + if in.CompletionTime != nil { + in, out := &in.CompletionTime, &out.CompletionTime + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdBackupStatus. +func (in *EtcdBackupStatus) DeepCopy() *EtcdBackupStatus { + if in == nil { + return nil + } + out := new(EtcdBackupStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EtcdCluster) DeepCopyInto(out *EtcdCluster) { *out = *in @@ -200,6 +356,128 @@ func (in *EtcdClusterStatus) DeepCopy() *EtcdClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdRestore) DeepCopyInto(out *EtcdRestore) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdRestore. +func (in *EtcdRestore) DeepCopy() *EtcdRestore { + if in == nil { + return nil + } + out := new(EtcdRestore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdRestore) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdRestoreList) DeepCopyInto(out *EtcdRestoreList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EtcdRestore, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdRestoreList. +func (in *EtcdRestoreList) DeepCopy() *EtcdRestoreList { + if in == nil { + return nil + } + out := new(EtcdRestoreList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EtcdRestoreList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdRestoreSpec) DeepCopyInto(out *EtcdRestoreSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + in.Target.DeepCopyInto(&out.Target) + if in.RestoreTimeout != nil { + in, out := &in.RestoreTimeout, &out.RestoreTimeout + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdRestoreSpec. +func (in *EtcdRestoreSpec) DeepCopy() *EtcdRestoreSpec { + if in == nil { + return nil + } + out := new(EtcdRestoreSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EtcdRestoreStatus) DeepCopyInto(out *EtcdRestoreStatus) { + *out = *in + if in.CompletionTime != nil { + in, out := &in.CompletionTime, &out.CompletionTime + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EtcdRestoreStatus. +func (in *EtcdRestoreStatus) DeepCopy() *EtcdRestoreStatus { + if in == nil { + return nil + } + out := new(EtcdRestoreStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GCSDestinationSpec) DeepCopyInto(out *GCSDestinationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCSDestinationSpec. +func (in *GCSDestinationSpec) DeepCopy() *GCSDestinationSpec { + if in == nil { + return nil + } + out := new(GCSDestinationSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemberStatus) DeepCopyInto(out *MemberStatus) { *out = *in @@ -360,6 +638,97 @@ func (in *ProviderConfig) DeepCopy() *ProviderConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RestoreTarget) DeepCopyInto(out *RestoreTarget) { + *out = *in + if in.StorageSpec != nil { + in, out := &in.StorageSpec, &out.StorageSpec + *out = new(StorageSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreTarget. +func (in *RestoreTarget) DeepCopy() *RestoreTarget { + if in == nil { + return nil + } + out := new(RestoreTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RetentionPolicy) DeepCopyInto(out *RetentionPolicy) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RetentionPolicy. +func (in *RetentionPolicy) DeepCopy() *RetentionPolicy { + if in == nil { + return nil + } + out := new(RetentionPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *S3DestinationSpec) DeepCopyInto(out *S3DestinationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new S3DestinationSpec. +func (in *S3DestinationSpec) DeepCopy() *S3DestinationSpec { + if in == nil { + return nil + } + out := new(S3DestinationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SnapshotLocation) DeepCopyInto(out *SnapshotLocation) { + *out = *in + in.Destination.DeepCopyInto(&out.Destination) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SnapshotLocation. +func (in *SnapshotLocation) DeepCopy() *SnapshotLocation { + if in == nil { + return nil + } + out := new(SnapshotLocation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SnapshotSource) DeepCopyInto(out *SnapshotSource) { + *out = *in + if in.BackupRef != nil { + in, out := &in.BackupRef, &out.BackupRef + *out = new(BackupReference) + **out = **in + } + if in.Location != nil { + in, out := &in.Location, &out.Location + *out = new(SnapshotLocation) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SnapshotSource. +func (in *SnapshotSource) DeepCopy() *SnapshotSource { + if in == nil { + return nil + } + out := new(SnapshotSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageSpec) DeepCopyInto(out *StorageSpec) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index a04983ec..e0460d48 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -58,7 +58,17 @@ func init() { } func main() { + // Subcommand dispatch: the operator image doubles as the restore + // download helper. When invoked as `manager restore-download` (by the + // restore download init-container the cluster controller injects), stream + // the snapshot out of object storage and exit, never starting the manager. + if len(os.Args) > 1 && os.Args[1] == "restore-localize" { + runRestoreLocalize() + return + } + var imageRegistry string + var operatorImage string var metricsAddr string var enableLeaderElection bool var probeAddr string @@ -67,6 +77,9 @@ func main() { var tlsOpts []func(*tls.Config) flag.StringVar(&imageRegistry, "image-registry", "gcr.io/etcd-development/etcd", "The container registry to pull etcd images from. Defaults to gcr.io/etcd-development/etcd.") + flag.StringVar(&operatorImage, "operator-image", os.Getenv("OPERATOR_IMAGE"), + "The operator's own image, used as the restore init-container that bootstraps a "+ + "restore-target member from a snapshot. Defaults to the OPERATOR_IMAGE env var.") flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -158,6 +171,23 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "EtcdCluster") os.Exit(1) } + if err = (&controller.EtcdBackupReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + RESTConfig: mgr.GetConfig(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EtcdBackup") + os.Exit(1) + } + if err = (&controller.EtcdRestoreReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + RESTConfig: mgr.GetConfig(), + OperatorImage: operatorImage, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EtcdRestore") + os.Exit(1) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/cmd/restore_localize.go b/cmd/restore_localize.go new file mode 100644 index 00000000..99a50cd4 --- /dev/null +++ b/cmd/restore_localize.go @@ -0,0 +1,238 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "go.uber.org/zap" + + "go.etcd.io/etcd/client/pkg/v3/logutil" + "go.etcd.io/etcd/etcdutl/v3/snapshot" + + "go.etcd.io/etcd-operator/pkg/objectstore" +) + +// The restore-localize subcommand runs as the SINGLE restore init-container the +// cluster controller injects into a restore-target member pod. It runs the +// operator image (the only image we control that already links the cloud SDKs +// and etcd's snapshot-restore library), so a restore needs no extra image and no +// shell. It performs, in order: +// +// 1. Ordinal guard: restore ONLY the genesis member (pod name ends in "-0"). +// Members 1..N exit 0 immediately and join via etcd's normal existing- +// cluster path; restoring into every member would lay N divergent single- +// member genesis dirs. +// 2. Idempotency guard: if the data dir already carries a marker file matching +// this restore's generation, exit 0 without touching the data — so a pod +// restart / reschedule never re-wipes-and-restores (which would silently +// lose everything written since the restore). +// 3. Download the snapshot from object storage into a temp file. +// 4. In-process `snapshot.Restore` (the etcdutl snapshot-restore library) into +// the etcd data dir, laying a single-member genesis whose member identity +// matches exactly what the etcd container will advertise on boot. A corrupt +// snapshot fails the library's hash check here, exiting non-zero — which is +// how the corrupt-snapshot rejection surfaces for the RIGHT reason. +// 5. Write the marker so step (2) short-circuits future restarts. +// +// Env contract (stamped by applyRestoreInitContainers / the EtcdCluster +// annotation): see the RESTORE_* / cred env names below. +const ( + envRLProvider = "RESTORE_PROVIDER" + envRLBucket = "RESTORE_BUCKET" + envRLPrefix = "RESTORE_PREFIX" + envRLKey = "RESTORE_KEY" + envRLRegion = "RESTORE_REGION" + envRLEndpoint = "RESTORE_ENDPOINT" + envRLForcePathStyle = "RESTORE_FORCE_PATH_STYLE" + envRLDataDir = "RESTORE_DATA_DIR" + envRLGeneration = "RESTORE_GENERATION" + envRLMemberName = "RESTORE_MEMBER_NAME" + envRLPeerURL = "RESTORE_PEER_URL" + envRLClusterToken = "RESTORE_CLUSTER_TOKEN" + envRLPodName = "POD_NAME" + + // S3 credential env (optional => ambient identity). + envRLS3AccessKeyID = "RESTORE_S3_ACCESS_KEY_ID" + envRLS3SecretAccessKey = "RESTORE_S3_SECRET_ACCESS_KEY" + envRLS3SessionToken = "RESTORE_S3_SESSION_TOKEN" + // GCS credential env (optional => ambient/unauthenticated). + envRLGCSServiceAccountJSON = "RESTORE_GCS_SERVICE_ACCOUNT_JSON" + + restoreDownloadTimeout = 10 * time.Minute + // restoreMarkerFile lives in the data dir and records the restore generation + // that produced it. + restoreMarkerFile = ".etcd-operator-restore-complete" +) + +// runRestoreLocalize is the `manager restore-localize` entrypoint. Any error +// exits non-zero so the init-container (and the member pod) fails loudly rather +// than the etcd container booting an empty / half-restored data dir. +func runRestoreLocalize() { + if err := doRestoreLocalize(context.Background()); err != nil { + fmt.Fprintf(os.Stderr, "restore-localize: %v\n", err) + os.Exit(1) + } +} + +func doRestoreLocalize(ctx context.Context) error { + podName := os.Getenv(envRLPodName) + dataDir := os.Getenv(envRLDataDir) + generation := os.Getenv(envRLGeneration) + memberName := os.Getenv(envRLMemberName) + peerURL := os.Getenv(envRLPeerURL) + if podName == "" || dataDir == "" || memberName == "" || peerURL == "" { + return fmt.Errorf("missing required env (pod=%q dataDir=%q member=%q peerURL=%q)", + podName, dataDir, memberName, peerURL) + } + + // (1) Ordinal guard: only the genesis member (-0) is restored. + if !strings.HasSuffix(podName, "-0") { + fmt.Fprintf(os.Stdout, "restore-localize: %s is not the genesis member; skipping restore\n", podName) + return nil + } + + // (2) Idempotency guard: skip if a marker for this generation already exists. + markerPath := filepath.Join(dataDir, restoreMarkerFile) + if existing, err := os.ReadFile(markerPath); err == nil { + if strings.TrimSpace(string(existing)) == generation { + fmt.Fprintf(os.Stdout, + "restore-localize: marker for generation %q already present; skipping restore\n", generation) + return nil + } + // A marker for a DIFFERENT generation means this data dir was restored + // from another snapshot and then (per the empty-target guard) should not + // be re-restored under us. Refuse rather than clobber. + return fmt.Errorf("data dir %q already restored for a different generation (have %q, want %q)", + dataDir, strings.TrimSpace(string(existing)), generation) + } + + // (3) Download the snapshot to a temp file. + snapPath := filepath.Join(os.TempDir(), "restore-snapshot.db") + if err := downloadSnapshot(ctx, snapPath); err != nil { + return err + } + defer func() { _ = os.Remove(snapPath) }() + + // (4) In-process restore into the etcd data dir. + if err := restoreSnapshotToDataDir(snapPath, dataDir, memberName, peerURL, + os.Getenv(envRLClusterToken)); err != nil { + return err + } + + // (5) Mark success so a restart is a no-op. + if err := os.WriteFile(markerPath, []byte(generation+"\n"), 0o600); err != nil { + return fmt.Errorf("write restore marker %q: %w", markerPath, err) + } + fmt.Fprintf(os.Stdout, "restore-localize: restored genesis member %q into %s\n", memberName, dataDir) + return nil +} + +// downloadSnapshot streams the configured snapshot object into snapPath. +func downloadSnapshot(ctx context.Context, snapPath string) error { + provider := os.Getenv(envRLProvider) + bucket := os.Getenv(envRLBucket) + key := os.Getenv(envRLKey) + if provider == "" || bucket == "" || key == "" { + return fmt.Errorf("missing snapshot source env (provider=%q bucket=%q key=%q)", provider, bucket, key) + } + + forcePathStyle, _ := strconv.ParseBool(os.Getenv(envRLForcePathStyle)) + dst := objectstore.Destination{ + Provider: objectstore.Provider(provider), + Bucket: bucket, + Prefix: os.Getenv(envRLPrefix), + Region: os.Getenv(envRLRegion), + Endpoint: os.Getenv(envRLEndpoint), + ForcePathStyle: forcePathStyle, + } + + var creds objectstore.Credentials + switch objectstore.Provider(provider) { + case objectstore.ProviderS3: + creds.AccessKeyID = os.Getenv(envRLS3AccessKeyID) + creds.SecretAccessKey = os.Getenv(envRLS3SecretAccessKey) + creds.SessionToken = os.Getenv(envRLS3SessionToken) + case objectstore.ProviderGCS: + if v := os.Getenv(envRLGCSServiceAccountJSON); v != "" { + creds.ServiceAccountJSON = []byte(v) + } + } + + dlCtx, cancel := context.WithTimeout(ctx, restoreDownloadTimeout) + defer cancel() + + store, err := objectstore.New(dlCtx, dst, creds) + if err != nil { + return fmt.Errorf("build object store: %w", err) + } + body, err := store.Download(dlCtx, key) + if err != nil { + return fmt.Errorf("download snapshot %q from %s://%s: %w", key, provider, bucket, err) + } + defer func() { _ = body.Close() }() + + out, err := os.Create(snapPath) + if err != nil { + return fmt.Errorf("create %q: %w", snapPath, err) + } + n, copyErr := io.Copy(out, body) + closeErr := out.Close() + if copyErr != nil { + return fmt.Errorf("stream snapshot to %q: %w", snapPath, copyErr) + } + if closeErr != nil { + return fmt.Errorf("close %q: %w", snapPath, closeErr) + } + if n == 0 { + return fmt.Errorf("downloaded snapshot %q is empty", key) + } + return nil +} + +// restoreSnapshotToDataDir runs the etcdutl snapshot-restore library in-process, +// laying a single-member genesis data directory into dataDir. The member name, +// peer URL and a single-member initial cluster must match exactly what the etcd +// container advertises on boot, or the bootstrapped member rejects itself. A +// corrupt/truncated snapshot fails the library's integrity check and returns an +// error here, which is the corrupt-snapshot rejection path. +func restoreSnapshotToDataDir(snapPath, dataDir, memberName, peerURL, clusterToken string) error { + lg, err := logutil.CreateDefaultZapLogger(zap.InfoLevel) + if err != nil { + lg = zap.NewNop() + } + if clusterToken == "" { + clusterToken = "etcd-cluster" + } + mgr := snapshot.NewV3(lg) + return mgr.Restore(snapshot.RestoreConfig{ + SnapshotPath: snapPath, + Name: memberName, + OutputDataDir: dataDir, + PeerURLs: []string{peerURL}, + InitialCluster: fmt.Sprintf("%s=%s", memberName, peerURL), + InitialClusterToken: clusterToken, + SkipHashCheck: false, + }) +} diff --git a/config/crd/bases/operator.etcd.io_etcdbackups.yaml b/config/crd/bases/operator.etcd.io_etcdbackups.yaml new file mode 100644 index 00000000..e41d6318 --- /dev/null +++ b/config/crd/bases/operator.etcd.io_etcdbackups.yaml @@ -0,0 +1,270 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: etcdbackups.operator.etcd.io +spec: + group: operator.etcd.io + names: + kind: EtcdBackup + listKind: EtcdBackupList + plural: etcdbackups + singular: etcdbackup + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.clusterRef + name: Cluster + type: string + - jsonPath: .spec.destination.provider + name: Provider + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.snapshotLocation + name: Location + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + EtcdBackup is the Schema for the etcdbackups API. It represents a single + point-in-time snapshot of an EtcdCluster uploaded to object storage. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: EtcdBackupSpec defines the desired state of an EtcdBackup. + properties: + clusterRef: + description: |- + ClusterRef references the EtcdCluster to snapshot. The cluster must live + in the same namespace as this EtcdBackup. + minLength: 1 + type: string + destination: + description: Destination describes the object-storage target for the + snapshot. + properties: + gcs: + description: |- + GCS holds GCS-specific destination configuration. Required when + Provider is "gcs". + properties: + bucket: + description: Bucket is the destination GCS bucket name. + minLength: 1 + type: string + endpoint: + description: |- + Endpoint overrides the GCS endpoint, enabling GCS-compatible emulators + such as fake-gcs-server or the gcloud storage testbench for hermetic, + credential-free testing. It must point at the JSON API root the emulator + serves (e.g. "http://fake-gcs:9000/storage/v1/"). When set, the client is + pointed at this endpoint and runs unauthenticated, mirroring the S3 + endpoint override that targets MinIO. If empty, the real Google endpoint + and the normal credential chain are used. + type: string + required: + - bucket + type: object + prefix: + description: |- + Prefix is an optional key prefix (a.k.a. "folder") within the bucket + under which the snapshot object is written. A trailing slash is optional. + type: string + provider: + description: Provider selects the object-storage backend. + enum: + - s3 + - gcs + type: string + s3: + description: |- + S3 holds S3-specific destination configuration. Required when + Provider is "s3". + properties: + bucket: + description: Bucket is the destination S3 bucket name. + minLength: 1 + type: string + endpoint: + description: |- + Endpoint overrides the S3 endpoint, enabling S3-compatible stores such + as MinIO or Ceph RGW. If empty, the default AWS endpoint for the region + is used. + type: string + forcePathStyle: + description: |- + ForcePathStyle forces path-style addressing (bucket in the path rather + than the host). Required by most S3-compatible stores. + type: boolean + region: + description: Region is the AWS region the bucket resides in + (e.g. "us-east-1"). + type: string + required: + - bucket + type: object + secretRef: + description: |- + SecretRef references a Secret in the EtcdBackup's namespace holding the + credentials for the provider. The expected keys depend on the provider: + - s3: accessKeyID / secretAccessKey (and optionally sessionToken) + - gcs: serviceAccountJSON (a GCP service-account key) + If unset, the controller falls back to ambient credentials available to + the operator pod (e.g. IRSA / Workload Identity), which is the + recommended production posture. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - provider + type: object + retention: + description: |- + Retention, when set, asks the controller to delete older snapshots under + the same bucket/prefix once more than RetainCount snapshots exist. A + value of 0 (the default) disables retention pruning. + properties: + retainCount: + description: |- + RetainCount is the number of most-recent snapshots to keep under the + destination bucket/prefix. Older snapshots are deleted after a + successful upload. Zero disables pruning. + format: int32 + minimum: 0 + type: integer + type: object + snapshotTimeout: + description: |- + SnapshotTimeout bounds how long the snapshot-save step may run before it + is considered failed. Defaults to 10m if unset. + type: string + required: + - clusterRef + - destination + type: object + status: + description: EtcdBackupStatus defines the observed state of an EtcdBackup. + properties: + completionTime: + description: CompletionTime is when the snapshot finished uploading + successfully. + format: date-time + type: string + conditions: + description: Conditions represent the latest available observations + of the backup's state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + description: ObservedGeneration is the most recent generation observed + by the controller. + format: int64 + type: integer + phase: + description: Phase is a high-level summary of the backup lifecycle. + type: string + snapshotLocation: + description: |- + SnapshotLocation is the fully-qualified URI of the uploaded snapshot + (e.g. "s3://my-bucket/etcd/backup-...db" or "gs://my-bucket/..."). + type: string + snapshotSizeBytes: + description: SnapshotSizeBytes is the size of the uploaded snapshot + in bytes. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/operator.etcd.io_etcdrestores.yaml b/config/crd/bases/operator.etcd.io_etcdrestores.yaml new file mode 100644 index 00000000..3b580f69 --- /dev/null +++ b/config/crd/bases/operator.etcd.io_etcdrestores.yaml @@ -0,0 +1,342 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: etcdrestores.operator.etcd.io +spec: + group: operator.etcd.io + names: + kind: EtcdRestore + listKind: EtcdRestoreList + plural: etcdrestores + singular: etcdrestore + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.target.name + name: Target + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.snapshotLocation + name: Source + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + EtcdRestore is the Schema for the etcdrestores API. It represents restoring a + snapshot from object storage into a new EtcdCluster. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EtcdRestoreSpec defines the desired state of an EtcdRestore: take a snapshot + from object storage and bootstrap a new EtcdCluster from it. + properties: + restoreTimeout: + description: |- + RestoreTimeout bounds how long the download+restore step may run before it + is considered failed. Defaults to 10m if unset. + type: string + source: + description: Source selects the snapshot to restore. + properties: + backupRef: + description: |- + BackupRef names a completed EtcdBackup in the same namespace whose + uploaded snapshot should be restored. The controller reads that backup's + destination (bucket/prefix/provider/secretRef) and recorded + snapshotLocation, so this is the convenient path when restoring a backup + the operator itself produced. + properties: + name: + description: Name is the metadata.name of the EtcdBackup to + restore from. + minLength: 1 + type: string + required: + - name + type: object + location: + description: |- + Location fully describes a snapshot object independently of any + EtcdBackup resource. Use this to restore a snapshot taken out-of-band or + after the originating EtcdBackup has been deleted. + properties: + destination: + description: |- + Destination is the object-storage location (provider, bucket, prefix and + optional secretRef) the snapshot lives in. The same secretRef semantics + as EtcdBackup apply: omit it to use ambient credentials. + properties: + gcs: + description: |- + GCS holds GCS-specific destination configuration. Required when + Provider is "gcs". + properties: + bucket: + description: Bucket is the destination GCS bucket + name. + minLength: 1 + type: string + endpoint: + description: |- + Endpoint overrides the GCS endpoint, enabling GCS-compatible emulators + such as fake-gcs-server or the gcloud storage testbench for hermetic, + credential-free testing. It must point at the JSON API root the emulator + serves (e.g. "http://fake-gcs:9000/storage/v1/"). When set, the client is + pointed at this endpoint and runs unauthenticated, mirroring the S3 + endpoint override that targets MinIO. If empty, the real Google endpoint + and the normal credential chain are used. + type: string + required: + - bucket + type: object + prefix: + description: |- + Prefix is an optional key prefix (a.k.a. "folder") within the bucket + under which the snapshot object is written. A trailing slash is optional. + type: string + provider: + description: Provider selects the object-storage backend. + enum: + - s3 + - gcs + type: string + s3: + description: |- + S3 holds S3-specific destination configuration. Required when + Provider is "s3". + properties: + bucket: + description: Bucket is the destination S3 bucket name. + minLength: 1 + type: string + endpoint: + description: |- + Endpoint overrides the S3 endpoint, enabling S3-compatible stores such + as MinIO or Ceph RGW. If empty, the default AWS endpoint for the region + is used. + type: string + forcePathStyle: + description: |- + ForcePathStyle forces path-style addressing (bucket in the path rather + than the host). Required by most S3-compatible stores. + type: boolean + region: + description: Region is the AWS region the bucket resides + in (e.g. "us-east-1"). + type: string + required: + - bucket + type: object + secretRef: + description: |- + SecretRef references a Secret in the EtcdBackup's namespace holding the + credentials for the provider. The expected keys depend on the provider: + - s3: accessKeyID / secretAccessKey (and optionally sessionToken) + - gcs: serviceAccountJSON (a GCP service-account key) + If unset, the controller falls back to ambient credentials available to + the operator pod (e.g. IRSA / Workload Identity), which is the + recommended production posture. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - provider + type: object + key: + description: |- + Key is the object key of the snapshot, relative to the destination + Prefix (e.g. "my-cluster/backup-1-20260617T010203Z.db"). It is joined + with the destination prefix exactly as the backup path joins them, so a + value copied verbatim from an EtcdBackup's status round-trips. + minLength: 1 + type: string + required: + - destination + - key + type: object + type: object + target: + description: Target describes the EtcdCluster to create and restore + into. + properties: + name: + description: |- + Name is the metadata.name of the EtcdCluster to create (in the + EtcdRestore's namespace) and restore into. It must not already exist + unless it exists and reports zero ready members (an empty shell). + minLength: 1 + type: string + size: + description: |- + Size is the desired size of the restored cluster. Defaults to 1 if unset; + a single-member restore is the safest default because etcd's snapshot + restore bootstraps a single-node cluster which is then grown. + minimum: 1 + type: integer + storageSpec: + description: |- + StorageSpec optionally requests persistent storage for the restored + cluster, mirroring EtcdClusterSpec.StorageSpec. When omitted the restored + cluster uses ephemeral container storage. + properties: + accessModes: + type: string + pvcName: + type: string + storageClassName: + type: string + volumeSizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + volumeSizeRequest: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - volumeSizeRequest + type: object + version: + description: |- + Version is the etcd version for the restored cluster. If empty, the + controller inherits the version recorded on the source EtcdBackup (when + restoring via backupRef) or requires it to be set explicitly. + type: string + required: + - name + type: object + required: + - source + - target + type: object + status: + description: EtcdRestoreStatus defines the observed state of an EtcdRestore. + properties: + completionTime: + description: CompletionTime is when the restore finished successfully. + format: date-time + type: string + conditions: + description: Conditions represent the latest available observations + of the restore's state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + description: ObservedGeneration is the most recent generation observed + by the controller. + format: int64 + type: integer + phase: + description: Phase is a high-level summary of the restore lifecycle. + type: string + restoredCluster: + description: |- + RestoredCluster is the name of the EtcdCluster the snapshot was restored + into, once created. + type: string + snapshotLocation: + description: |- + SnapshotLocation is the fully-qualified URI of the snapshot that was + restored (e.g. "s3://my-bucket/etcd/backups/...db"). + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index ca614b9f..6f8e7aef 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,6 +3,8 @@ # It should be run by config/default resources: - bases/operator.etcd.io_etcdclusters.yaml +- bases/operator.etcd.io_etcdbackups.yaml +- bases/operator.etcd.io_etcdrestores.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/e2e/patch-env.yaml b/config/e2e/patch-env.yaml index ceff2e89..37d190da 100644 --- a/config/e2e/patch-env.yaml +++ b/config/e2e/patch-env.yaml @@ -10,3 +10,9 @@ spec: env: - name: GOFAIL_HTTP value: ":22381" + # The restore controller stamps this image onto restore-target + # clusters as the restore init-container. It must match the operator + # image the e2e builds and loads into kind (see config/manager + # images: transform). Kept in sync manually with that tag. + - name: OPERATOR_IMAGE + value: "etcd-operator:v0.1" diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c3164c44..67d74770 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -65,6 +65,15 @@ spec: - --health-probe-bind-address=:8081 image: controller:latest name: manager + env: + # OPERATOR_IMAGE is stamped onto restore-target EtcdClusters as the + # restore init-container (it runs `manager restore-localize` to fetch + # the snapshot and lay the genesis data dir). It must equal the image + # this Deployment runs. Override it (or pass --operator-image) to match + # your deployed operator image; kustomize's images: transform rewrites + # the container image: above but not this env value. + - name: OPERATOR_IMAGE + value: "gcr.io/etcd-io/etcd-operator:latest" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/config/rbac/etcdbackup_editor_role.yaml b/config/rbac/etcdbackup_editor_role.yaml new file mode 100644 index 00000000..b9c5240a --- /dev/null +++ b/config/rbac/etcdbackup_editor_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to edit etcdbackups. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdbackup-editor-role +rules: +- apiGroups: + - operator.etcd.io + resources: + - etcdbackups + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - operator.etcd.io + resources: + - etcdbackups/status + verbs: + - get diff --git a/config/rbac/etcdbackup_viewer_role.yaml b/config/rbac/etcdbackup_viewer_role.yaml new file mode 100644 index 00000000..74943623 --- /dev/null +++ b/config/rbac/etcdbackup_viewer_role.yaml @@ -0,0 +1,23 @@ +# permissions for end users to view etcdbackups. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdbackup-viewer-role +rules: +- apiGroups: + - operator.etcd.io + resources: + - etcdbackups + verbs: + - get + - list + - watch +- apiGroups: + - operator.etcd.io + resources: + - etcdbackups/status + verbs: + - get diff --git a/config/rbac/etcdrestore_editor_role.yaml b/config/rbac/etcdrestore_editor_role.yaml new file mode 100644 index 00000000..12d02fd2 --- /dev/null +++ b/config/rbac/etcdrestore_editor_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to edit etcdrestores. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdrestore-editor-role +rules: +- apiGroups: + - operator.etcd.io + resources: + - etcdrestores + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - operator.etcd.io + resources: + - etcdrestores/status + verbs: + - get diff --git a/config/rbac/etcdrestore_viewer_role.yaml b/config/rbac/etcdrestore_viewer_role.yaml new file mode 100644 index 00000000..bbb562db --- /dev/null +++ b/config/rbac/etcdrestore_viewer_role.yaml @@ -0,0 +1,23 @@ +# permissions for end users to view etcdrestores. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdrestore-viewer-role +rules: +- apiGroups: + - operator.etcd.io + resources: + - etcdrestores + verbs: + - get + - list + - watch +- apiGroups: + - operator.etcd.io + resources: + - etcdrestores/status + verbs: + - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index ae91bf79..b6d28b66 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -24,4 +24,8 @@ resources: # if you do not want those helpers be installed with your Project. - etcdcluster_editor_role.yaml - etcdcluster_viewer_role.yaml +- etcdbackup_editor_role.yaml +- etcdbackup_viewer_role.yaml +- etcdrestore_editor_role.yaml +- etcdrestore_viewer_role.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6c9ac65e..d9b4ba10 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -28,6 +28,20 @@ rules: - list - patch - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create - apiGroups: - apps resources: @@ -64,7 +78,9 @@ rules: - apiGroups: - operator.etcd.io resources: + - etcdbackups - etcdclusters + - etcdrestores verbs: - create - delete @@ -76,13 +92,17 @@ rules: - apiGroups: - operator.etcd.io resources: + - etcdbackups/finalizers - etcdclusters/finalizers + - etcdrestores/finalizers verbs: - update - apiGroups: - operator.etcd.io resources: + - etcdbackups/status - etcdclusters/status + - etcdrestores/status verbs: - get - patch diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index f3c97ab2..b744b9ce 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -1,4 +1,6 @@ ## Append samples of your project ## resources: - operator_v1alpha1_etcdcluster.yaml +- operator_v1alpha1_etcdbackup.yaml +- operator_v1alpha1_etcdrestore.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/operator_v1alpha1_etcdbackup.yaml b/config/samples/operator_v1alpha1_etcdbackup.yaml new file mode 100644 index 00000000..c4a4dd75 --- /dev/null +++ b/config/samples/operator_v1alpha1_etcdbackup.yaml @@ -0,0 +1,55 @@ +# Sample EtcdBackup CRs for the two built-in object-storage providers. +# +# Each backup is a one-shot snapshot of the referenced EtcdCluster, uploaded to +# the configured bucket/prefix. Credentials are optional: when secretRef is +# omitted the operator falls back to ambient cloud credentials (IRSA on AWS, +# Workload Identity on GCP), which is the recommended production posture. +--- +apiVersion: operator.etcd.io/v1alpha1 +kind: EtcdBackup +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdbackup-s3-sample +spec: + # Name of an EtcdCluster in the same namespace. + clusterRef: etcdcluster-sample + # Bound the snapshot-save step (defaults to 10m if omitted). + snapshotTimeout: 10m + destination: + provider: s3 + # Optional "folder" within the bucket. + prefix: etcd/backups + s3: + bucket: my-etcd-backups + region: us-east-1 + # endpoint + forcePathStyle enable S3-compatible stores (MinIO, Ceph RGW). + # endpoint: http://minio.storage.svc.cluster.local:9000 + # forcePathStyle: true + # Optional credentials secret. Keys: accessKeyID / secretAccessKey + # (and optionally sessionToken). Omit to use ambient credentials (IRSA). + secretRef: + name: s3-backup-credentials + # Optional retention: keep the 7 most-recent snapshots under bucket/prefix. + retention: + retainCount: 7 +--- +apiVersion: operator.etcd.io/v1alpha1 +kind: EtcdBackup +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdbackup-gcs-sample +spec: + clusterRef: etcdcluster-sample + destination: + provider: gcs + prefix: etcd/backups + gcs: + bucket: my-etcd-backups + # Optional credentials secret. Key: serviceAccountJSON (a GCP SA key). + # Omit to use Application Default Credentials (Workload Identity). + secretRef: + name: gcs-backup-credentials diff --git a/config/samples/operator_v1alpha1_etcdrestore.yaml b/config/samples/operator_v1alpha1_etcdrestore.yaml new file mode 100644 index 00000000..8c269649 --- /dev/null +++ b/config/samples/operator_v1alpha1_etcdrestore.yaml @@ -0,0 +1,65 @@ +# Sample EtcdRestore CRs. +# +# An EtcdRestore is a one-shot job: it downloads a snapshot from object storage +# and bootstraps a NEW, empty EtcdCluster from it. It never restores over a +# cluster that already has members. Credentials follow the same rules as +# EtcdBackup: they come ONLY from the destination's secretRef (or the operator's +# ambient identity when secretRef is omitted) and are never logged. +--- +# Restore from a completed EtcdBackup by reference. The destination, snapshot +# key, and (best-effort) etcd version are derived from the referenced backup. +apiVersion: operator.etcd.io/v1alpha1 +kind: EtcdRestore +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdrestore-from-backup-sample +spec: + source: + backupRef: + # Name of a completed EtcdBackup in the same namespace. + name: etcdbackup-s3-sample + target: + # The EtcdCluster to create and restore into. Must not already exist with + # members; the operator creates it as a fresh, single-member cluster. + name: etcdcluster-restored + size: 1 + # version is optional here: it is inherited from the source cluster when it + # still exists. Set it explicitly to pin the restored cluster's etcd version. + # version: v3.6.1 + # Bound the download+restore step (defaults to 10m if omitted). + restoreTimeout: 10m +--- +# Restore from an explicit object-storage location, independent of any +# EtcdBackup resource (e.g. the originating backup was deleted). +apiVersion: operator.etcd.io/v1alpha1 +kind: EtcdRestore +metadata: + labels: + app.kubernetes.io/name: etcd-operator + app.kubernetes.io/managed-by: kustomize + name: etcdrestore-from-location-sample +spec: + source: + location: + destination: + provider: s3 + prefix: etcd/backups + s3: + bucket: my-etcd-backups + region: us-east-1 + # Credentials secret. Keys: accessKeyID / secretAccessKey (and optionally + # sessionToken). Omit to use ambient credentials (IRSA). The same secret + # used for backups is reused here; restore needs only read access. + secretRef: + name: s3-backup-credentials + # Object key relative to the destination prefix. Copy this verbatim from a + # backup's .status.snapshotLocation (the part after the bucket/prefix). + key: etcdcluster-sample/etcdbackup-s3-sample-20260617T010203Z.db + target: + name: etcdcluster-restored-from-location + size: 1 + # version is REQUIRED on the explicit-location path (no source cluster to + # inherit from). + version: v3.6.1 diff --git a/docs/snapshot-backups.md b/docs/snapshot-backups.md new file mode 100644 index 00000000..72630dc7 --- /dev/null +++ b/docs/snapshot-backups.md @@ -0,0 +1,227 @@ +# EtcdBackup: on-demand snapshots to object storage + +This document describes the `EtcdBackup` custom resource, which implements the +roadmap item *"Create on-demand backup of a cluster"* by taking a point-in-time +etcd snapshot and uploading it to an object store (S3, GCS, or a pluggable +provider). + +## Custom resource + +An `EtcdBackup` is a one-shot job: it references an `EtcdCluster` in the same +namespace and a destination, the controller snapshots the cluster and uploads +the result, and the resource then serves as the immutable record of that +snapshot. + +```yaml +apiVersion: operator.etcd.io/v1alpha1 +kind: EtcdBackup +metadata: + name: nightly-2026-06-17 +spec: + clusterRef: my-cluster # EtcdCluster in the same namespace + snapshotTimeout: 10m # optional, bounds the snapshot-save step + destination: + provider: s3 # s3 | gcs + prefix: etcd/backups # optional key prefix within the bucket + secretRef: # optional; omit to use ambient credentials + name: s3-backup-credentials + s3: + bucket: my-etcd-backups + region: us-east-1 + # endpoint + forcePathStyle enable S3-compatible stores (MinIO, Ceph RGW) + retention: + retainCount: 7 # keep the 7 newest snapshots under bucket/prefix +status: + phase: Completed # Pending | Snapshotting | Uploading | Completed | Failed + snapshotLocation: s3://my-etcd-backups/etcd/backups/my-cluster/nightly-2026-06-17-20260617T010203Z.db + snapshotSizeBytes: 20480 + completionTime: "2026-06-17T01:02:05Z" + conditions: + - type: Succeeded + status: "True" + reason: SnapshotUploaded +``` + +## Architecture + +The backup feature is implemented as a **separate controller plus an isolated +provider package**, wired into the existing operator binary: + +``` +api/v1alpha1/etcdbackup_types.go # EtcdBackup CRD +api/v1alpha1/etcdrestore_types.go # EtcdRestore CRD +internal/controller/etcdbackup_*.go # backup reconciler + snapshotter (no cloud SDK imports) +internal/controller/etcdrestore_*.go # restore reconciler + restorer (no cloud SDK imports) +internal/controller/objectstore_creds.go # shared, audit-logged credential seam +pkg/objectstore/{interface,s3,gcs}.go # cloud SDKs isolated behind a Store interface +``` + +Two seams keep the cloud SDKs and the exec machinery out of the core +reconciliation path and make the orchestration unit-testable without cloud +credentials or a live cluster: + +1. **`Snapshotter`** produces the snapshot byte stream. The default + implementation execs `etcdctl snapshot save` inside a member pod via the + Kubernetes exec subresource and streams its stdout. +2. **`pkg/objectstore.Store`** is the minimal object-storage surface + (`Upload` / `Download` / `List` / `Delete`). A `Factory` registry dispatches + on the provider name, so a new backend is added by dropping a file in the package + and calling `objectstore.Register`. The AWS and GCP SDKs are imported **only** + from this package. + +The snapshot is streamed from the snapshotter into the uploader through an +`io.Pipe`, giving backpressure so memory stays bounded. + +### Why in-binary (vs a standalone backup-manager) + +The heavy cloud SDKs are isolated to `pkg/objectstore` behind an interface, and +the controller is a self-contained file set, so the code is trivially liftable +into a future `cmd/backup-manager` if the CVE surface of the cloud SDKs becomes +a concern for the core operator. Shipping it in the existing binary now keeps +this a single, independently-mergeable PR with one Deployment, one RBAC set, and +one image — the standard kubebuilder multi-controller layout — and avoids the +operational cost of a second binary before there is a reason for it. + +## Credentials + +`secretRef` is optional. When omitted, providers use their ambient credential +chain — IRSA on AWS, Workload Identity on GCP — which is the recommended +production posture. When present, the expected secret keys are: + +- **s3**: `accessKeyID`, `secretAccessKey`, and optionally `sessionToken` +- **gcs**: `serviceAccountJSON` (a GCP service-account key) + +# EtcdRestore: restore a snapshot into a new cluster + +`EtcdRestore` implements the roadmap item *"Create a new cluster from a +backup"*: it downloads a snapshot from object storage and bootstraps a **new, +empty** `EtcdCluster` from it. A restore is always a genesis — it never overlays +a snapshot onto a cluster that already has members. + +## Custom resource + +The snapshot source is one of two mutually-exclusive forms: + +- **`backupRef`** — the name of a completed `EtcdBackup` in the same namespace. + The destination (bucket/prefix/provider/secretRef), the object key, and a + best-effort etcd version (inherited from the still-existing source cluster) + are all derived from that backup. This is the convenient path for restoring a + backup the operator itself produced. +- **`location`** — an explicit object-storage destination plus the object + `key`, independent of any `EtcdBackup` (e.g. the originating backup was + deleted). On this path `target.version` is required. + +```yaml +apiVersion: operator.etcd.io/v1alpha1 +kind: EtcdRestore +metadata: + name: restore-2026-06-18 +spec: + source: + backupRef: + name: nightly-2026-06-17 # a Completed EtcdBackup + target: + name: my-cluster-restored # NEW EtcdCluster to create + size: 1 # restore bootstraps a single seed member + # version: v3.6.1 # optional; inherited from source cluster + restoreTimeout: 10m # optional, bounds download+restore +status: + phase: Completed # Pending | Downloading | Restoring | Completed | Failed + snapshotLocation: s3://my-etcd-backups/etcd/backups/my-cluster/nightly-...db + restoredCluster: my-cluster-restored + conditions: + - type: Succeeded + status: "True" + reason: SnapshotRestored +``` + +## How it works + +The restore controller mirrors the backup controller's two-seam design so its +orchestration is unit-testable with no cloud creds and no live etcd: + +1. **`pkg/objectstore.Store.Download`** streams the snapshot back out of the + bucket. Unlike the upload path (which spools to a temp file because S3 + `PutObject` needs a known length), `Download` streams the body, so the + snapshot is never staged on the operator's disk on the read path. +2. **`Restorer`** writes the snapshot into the target member and bootstraps the + new cluster. The default implementation streams the snapshot bytes into the + genesis pod (`-0`) over the exec subresource's stdin and runs + `etcdctl snapshot restore` into a fresh data directory, stamping the member + identity (`--name` / `--initial-advertise-peer-urls` / `--initial-cluster`) + with the exact name and peer URL the bootstrapping pod will advertise. + `etcdctl snapshot restore` refuses to overwrite a populated data directory, + which is a second line of defence behind the controller's empty-target check. + +**Empty-target guarantee.** Before restoring, the controller either creates the +target `EtcdCluster` fresh (owning it, so deleting the `EtcdRestore` can GC the +restored cluster) or, if a cluster of that name already exists, verifies its +backing StatefulSet reports **zero** ready members. Any ready member aborts the +restore — a restore must never silently diverge from a live cluster. + +# Security + +The backup and restore paths share one credential seam +(`resolveStoreCredentials`) with the following guarantees, asserted directly by +unit tests: + +- **Credentials come only from `secretRef`** (or the operator's ambient + identity when `secretRef` is omitted). No other source is consulted. +- **Credential values are never logged.** The resolver emits a structured audit + log line recording the namespace, provider, and the secret *name* — never any + key value — on every resolution (including the ambient-identity case). Secret + bytes never reach status, conditions, or Events either. +- **Errors never echo secret material.** A missing or malformed secret yields an + error mentioning only the secret name and the provider's required key names. +- **Validation before I/O.** The destination's provider-specific block must be + present and consistent, a referenced secret name must be non-empty, and an + `EtcdRestore` must set exactly one snapshot source — all checked before any + network call so misconfiguration is a clean, terminal failure. + +## Least-privilege IAM + +The credentials a backup/restore needs are narrow; scope them accordingly rather +than reusing a broad cluster role. + +**AWS S3.** Restore needs only read; backup needs write and (for retention) +list/delete. A combined policy, scoped to the backup prefix: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "BackupRestoreObjects", + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": "arn:aws:s3:::my-etcd-backups/etcd/backups/*" + }, + { + "Sid": "BackupRetentionList", + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": "arn:aws:s3:::my-etcd-backups", + "Condition": {"StringLike": {"s3:prefix": ["etcd/backups/*"]}} + } + ] +} +``` + +A restore-only identity can drop `s3:PutObject` and `s3:DeleteObject`. Prefer +IRSA (no stored secret) over a long-lived access key whenever the operator runs +on EKS. + +**GCS.** Grant `roles/storage.objectAdmin` on the bucket (or +`roles/storage.objectViewer` for a restore-only identity). Prefer Workload +Identity over a downloaded service-account key. + +# Out of scope (follow-ups) + +- **Cron / scheduled backups** (`EtcdBackupSchedule`) are deferred; `EtcdBackup` + and the provider interface are the foundation for them. +- **Encryption at rest** of the snapshot object (SSE-KMS / CMEK or + operator-side envelope encryption) is deferred; today the snapshot is written + with the bucket's default encryption. +- TLS-enabled clusters: the snapshotter and restorer currently issue plain + `etcdctl` commands; cert-flag injection for mTLS clusters is a small + follow-up that both paths share. diff --git a/go.mod b/go.mod index 4cd5ea7e..5af44e67 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,21 @@ go 1.26.0 toolchain go1.26.4 require ( + cloud.google.com/go/storage v1.62.3 + github.com/aws/aws-sdk-go-v2 v1.42.0 + github.com/aws/aws-sdk-go-v2/config v1.32.25 + github.com/aws/aws-sdk-go-v2/credentials v1.19.24 + github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0 github.com/cert-manager/cert-manager v1.20.3 github.com/go-logr/logr v1.4.3 github.com/stretchr/testify v1.11.1 go.etcd.io/etcd/api/v3 v3.6.12 go.etcd.io/etcd/client/pkg/v3 v3.6.12 go.etcd.io/etcd/client/v3 v3.6.12 + go.etcd.io/etcd/etcdutl/v3 v3.6.12 go.etcd.io/etcd/server/v3 v3.6.12 go.uber.org/zap v1.28.0 + google.golang.org/api v0.285.0 k8s.io/api v0.36.2 k8s.io/apimachinery v0.36.2 k8s.io/client-go v0.36.2 @@ -21,17 +28,52 @@ require ( ) require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.7.0 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 // indirect + github.com/aws/smithy-go v1.27.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-openapi/swag/jsonname v0.25.4 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.16 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 // indirect github.com/moby/spdystream v0.5.1 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/vladimirvivien/gexe v0.5.0 // indirect go.etcd.io/raft/v3 v3.6.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.42.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect k8s.io/streaming v0.36.2 // indirect k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect sigs.k8s.io/gateway-api v1.5.0 // indirect @@ -91,8 +133,8 @@ require ( github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 // indirect go.etcd.io/bbolt v1.4.3 // indirect go.etcd.io/etcd/pkg/v3 v3.6.12 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect @@ -101,19 +143,19 @@ require ( go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.52.0 // indirect + golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/oauth2 v0.35.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/term v0.43.0 // indirect - golang.org/x/text v0.37.0 // indirect - golang.org/x/time v0.14.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/grpc v1.79.3 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad // indirect + google.golang.org/grpc v1.81.1 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index b9df057d..32c4102f 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,75 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= +cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= +cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= +cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY= +cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.62.3 h1:SZq1t23NCI+e96dH77Dg3PEfsNNEjqO8zE5AnD8gVD0= +cloud.google.com/go/storage v1.62.3/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go-v2 v1.42.0 h1:XvXMJTkFQtpBKIWZnmr9ZEOc2InWM2yldjXEJ/bymhA= +github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls= +github.com/aws/aws-sdk-go-v2/config v1.32.25 h1:ACCejvStYoilgwrfegSt5ZntCbPrk52qfwyNcnl3omM= +github.com/aws/aws-sdk-go-v2/config v1.32.25/go.mod h1:LJyU8sDRbXUxFn8xMJIGP+v9QYYwveNLI8a/giAOiAs= +github.com/aws/aws-sdk-go-v2/credentials v1.19.24 h1:2hQqYCV9yqyePQ9o6dCrZc/zO8U3TwPr9mIKlZnPu/I= +github.com/aws/aws-sdk-go-v2/credentials v1.19.24/go.mod h1:IDwpACtwqHLISdzfwUUNq4P9DsB/h5BLg4FwJPNfqFY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 h1:r6qZHbT+wxgWO/e9vYNUEtg7lv5+UN3pRqKhLXvnArg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29/go.mod h1:QRnaRcTVGKPGRy8w78HMQtKUGRYcnMZAANATkeVA6Mo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 h1:f3vKqSo13fhTYb+JEcXwXefZQE26I1FB5eTSniU67ko= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29/go.mod h1:MzoLFUArKGpGD+ukmPiTPG1X5x4o6M2kq4v2dr1FiEc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 h1:RdwIf/CuUsvJX3RgJagbOyotl/cxoLY4xviKuE7p2GY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29/go.mod h1:71wt8W2EgswdZy9Mf9KNnzxZ3TiZlv4caKghPktDOkA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 h1:VTGy885W5DKBxWRUJbym9hytNaYzsyaPkCHGRRMAOhU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30/go.mod h1:AS0HycUvJRFvTt613AYDOgO2jzw+00cVSMny8XB3yMY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 h1:ZD2+BSw9vFsNlKYIasSNt3uDbjqqXIBcM13UJv/Lx2k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12/go.mod h1:Ms4zlcVBbXbiP7EVLhl+lgjvA/a7YphqQ3Ih3174EmI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 h1:V51LGlOq/1VsDsHUdoklAQi7rMmx4qQubvFYAlP2254= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22/go.mod h1:4Pzhyz8hJOm2bepgl+NjvRx8vlUFAIIvJnZ/MkcNPpU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 h1:DRebniUGZ2MqiiIVmQJ04vIXr918hubdHMnarSLEWyU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29/go.mod h1:LfRkPCD8YHDM2E5eTkos2UpwYeZnBcVarTa8L59bJHA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 h1:hiME6pBzC7OTl9LMtlyTWBuEl1f4QBcUmFDKC7MLXtc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29/go.mod h1:G7RP+uhagpKtKhd1BM9N6JQqjCcGEU47K5lBVZQyRQw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0 h1:ta8csKy5vN91F3i5gGR85lFV0srBqySEji7Jroes6rE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 h1:3nXpRcFwRCW8n7HgO2QGy0Dc20eQNfBuUemGQhpF8m8= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.0/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 h1:ey1XLTYXb9PcLt4535632o5kCGXNXEhNb620Dqwuylo= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.3/go.mod h1:Lk7PlmoTYryQmyBG0EXqj5BcUbj3whXdU2s3yGI3EAc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 h1:yLr03zQE/5Eu5l3QU0Si+xMbLMbSDF2YXsigqXngs6g= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6/go.mod h1:Q5N6icH+KJZDLh+ESNwzdv6cZ6vLFF/egy3IOxWhmz4= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 h1:VrIhKRCSK1umelSgB9RghvA9RTUYeQffyAS5ApXehNI= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.3/go.mod h1:r8wkDOuLaaMFqFiYAb8dGY2A3gJCOujMc6CFOVC4Zhc= +github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= +github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -16,6 +80,8 @@ github.com/cert-manager/cert-manager v1.20.3 h1:7zgThbjfRBNjN2/cM/Wdo/vl/oeFQybI github.com/cert-manager/cert-manager v1.20.3/go.mod h1:Aqf5P0xRh9aey1p10m2c3UAk/Vb/FBPyH3WQxJRm+7Y= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= @@ -31,6 +97,14 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= @@ -41,6 +115,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -77,10 +153,18 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.16 h1:F/VPrx0YPBdksZJQdCAp0WUsqnNmZpUZszzfYt0M5Dw= +github.com/googleapis/enterprise-certificate-proxy v0.3.16/go.mod h1:9Yb0eAkH/Xqhvv3zbeKf/+wMJqCeocWc6KIhDvEAuYE= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= @@ -126,6 +210,8 @@ github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -150,6 +236,8 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -183,6 +271,8 @@ go.etcd.io/etcd/client/pkg/v3 v3.6.12 h1:36zzB+pQOdHbhN+kH2iJz/K8bJn0ZLtLfPPO7jo go.etcd.io/etcd/client/pkg/v3 v3.6.12/go.mod h1:hh2+ZXtfLzs3o6mn92ntgNPBrTJJOvXqICM5g3L3DMY= go.etcd.io/etcd/client/v3 v3.6.12 h1:kMSP6JcPZMqSJiX+TXdUIBU/4eXEZWBAaui4VihMbIc= go.etcd.io/etcd/client/v3 v3.6.12/go.mod h1:CMs6fJWYiZQk4ytFjd4lE1diOvvRMmtbbn/alZXd3dQ= +go.etcd.io/etcd/etcdutl/v3 v3.6.12 h1:LvTFNBzRYO8m9MlRIeYKi6MEAADMOkffAfc0B7nqEYE= +go.etcd.io/etcd/etcdutl/v3 v3.6.12/go.mod h1:ZhxWJgE0yEslHH8XVKY1dzB8uZpkUyACmfOGJ56Va7I= go.etcd.io/etcd/pkg/v3 v3.6.12 h1:rewjbWPC/H5GHK0yxPbU0lzdFdQR9RlpZL7XmLYm2BE= go.etcd.io/etcd/pkg/v3 v3.6.12/go.mod h1:qDFIetmpC8TTZfkZkDzpNrXtVqVsyYumRWNPFXFhcpQ= go.etcd.io/etcd/server/v3 v3.6.12 h1:PAcIHCcTjPM1sbePiu7fCzNKQvOBFEaGnu2JFhgaGJQ= @@ -191,16 +281,20 @@ go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0 h1:XmiuHzgJt067+a6kwyAzkhXooYVv3/TOw9cM2VfJgUM= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.65.0/go.mod h1:KDgtbWKTQs4bM+VPUr6WlL9m/WXcmkCcBlIzqxPGzmI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= +go.opentelemetry.io/contrib/detectors/gcp v1.42.0/go.mod h1:W9zQ439utxymRrXsUOzZbFX4JhLxXU4+ZnCt8GG7yA8= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= @@ -224,29 +318,29 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= -golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -255,38 +349,42 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= -google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.285.0 h1:B7eHHoKGAX/LrPkQvhQqnGwjgWxofbdGwCTQvpm8FkM= +google.golang.org/api v0.285.0/go.mod h1:NlOlUIr8MPoIhT9Bb/oUnRuHbJOLwxb6JSYJM8Yz+jQ= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad h1:45WmJvIV6C2+O/jjLkPUH+F3aOj/1miDoU2DD0+NWbg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/controller/etcdbackup_controller.go b/internal/controller/etcdbackup_controller.go new file mode 100644 index 00000000..331a8639 --- /dev/null +++ b/internal/controller/etcdbackup_controller.go @@ -0,0 +1,325 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "io" + "time" + + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/objectstore" +) + +const ( + // defaultSnapshotTimeout bounds the snapshot-save step when the spec omits one. + defaultSnapshotTimeout = 10 * time.Minute + // backupContainerName is the etcd container name in member pods. + backupContainerName = "etcd" +) + +// EtcdBackupReconciler reconciles an EtcdBackup object: it takes a point-in-time +// snapshot of the referenced EtcdCluster and uploads it to object storage. +// +// The reconciler is intentionally thin: all cloud-specific behavior lives +// behind two seams — Snapshotter (produce the snapshot stream) and +// objectstore.Factory (write it to a bucket). Both are injectable so the +// orchestration is exercised by unit tests with no cloud creds and no live +// etcd. +type EtcdBackupReconciler struct { + client.Client + Scheme *runtime.Scheme + + // RESTConfig is retained for parity with the restore reconciler and any + // future in-pod tooling. The default snapshotter is client-API based and + // does not use it. + RESTConfig *rest.Config + + // Snapshotter produces the snapshot stream. Defaults to an exec-based + // implementation in SetupWithManager; overridden in tests. + Snapshotter Snapshotter + + // NewStore builds an object store from a destination and credentials. + // Defaults to objectstore.New; overridden in tests with a fake. + NewStore objectstore.Factory +} + +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdbackups,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdbackups/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdbackups/finalizers,verbs=update +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdclusters,verbs=get;list;watch +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch + +// Reconcile drives a single EtcdBackup to completion. Because a backup is a +// one-shot job rather than a continuously-reconciled desired state, a terminal +// phase (Completed/Failed) short-circuits further work; the resource is the +// audit record of that snapshot. +func (r *EtcdBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + var backup ecv1alpha1.EtcdBackup + if err := r.Get(ctx, req.NamespacedName, &backup); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Terminal backups are immutable records; nothing more to do. + if backup.Status.Phase == ecv1alpha1.BackupPhaseCompleted || + backup.Status.Phase == ecv1alpha1.BackupPhaseFailed { + return ctrl.Result{}, nil + } + + backup.Status.ObservedGeneration = backup.Generation + + result, err := r.runBackup(ctx, &backup) + if err != nil { + logger.Error(err, "backup failed", "backup", req.NamespacedName) + r.markFailed(&backup, err) + } + + if statusErr := r.Status().Update(ctx, &backup); statusErr != nil { + // Prefer surfacing the original error if both occurred. + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, statusErr + } + return result, err +} + +// runBackup performs the snapshot+upload, mutating backup.Status as it +// progresses. It returns a non-nil error on any failure; the caller records it. +func (r *EtcdBackupReconciler) runBackup(ctx context.Context, backup *ecv1alpha1.EtcdBackup) (ctrl.Result, error) { + // 1. Resolve the target cluster and a member pod to snapshot from. + pod, err := r.selectMemberPod(ctx, backup) + if err != nil { + return ctrl.Result{}, err + } + + // 2. Build the object store (resolving credentials from the optional secret). + store, err := r.buildStore(ctx, backup) + if err != nil { + return ctrl.Result{}, err + } + + // 3. Take the snapshot and stream it into the uploader via a pipe. The GCS + // provider streams straight through; the S3 provider spools the stream to + // an ephemeral temp file first (PutObject needs a known length), so for + // S3 the snapshot does briefly land in the operator pod's scratch space. + backup.Status.Phase = ecv1alpha1.BackupPhaseSnapshotting + + timeout := defaultSnapshotTimeout + if backup.Spec.SnapshotTimeout != nil { + timeout = backup.Spec.SnapshotTimeout.Duration + } + snapCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + key := snapshotObjectKey(backup) + res, err := r.snapshotAndUpload(snapCtx, backup, store, pod, key) + if err != nil { + return ctrl.Result{}, err + } + + // 4. Record success. + now := metav1.Now() + backup.Status.Phase = ecv1alpha1.BackupPhaseCompleted + backup.Status.SnapshotLocation = res.URI + backup.Status.SnapshotSizeBytes = res.Size + backup.Status.CompletionTime = &now + meta.SetStatusCondition(&backup.Status.Conditions, metav1.Condition{ + Type: ecv1alpha1.BackupConditionSucceeded, + Status: metav1.ConditionTrue, + Reason: "SnapshotUploaded", + Message: fmt.Sprintf("snapshot uploaded to %s (%d bytes)", res.URI, res.Size), + ObservedGeneration: backup.Generation, + }) + + // 5. Best-effort retention pruning. A pruning failure does not fail the + // backup itself — the snapshot is already safely uploaded. + if err := r.applyRetention(ctx, backup, store); err != nil { + log.FromContext(ctx).Error(err, "retention pruning failed (snapshot upload still succeeded)") + } + + return ctrl.Result{}, nil +} + +// snapshotAndUpload wires the snapshotter's stdout pipe into the store's +// Upload reader. The pipe gives backpressure: the snapshot only advances as +// fast as the upload drains it, bounding memory to the io.Copy buffer. +func (r *EtcdBackupReconciler) snapshotAndUpload( + ctx context.Context, + backup *ecv1alpha1.EtcdBackup, + store objectstore.Store, + pod types.NamespacedName, + key string, +) (objectstore.UploadResult, error) { + pr, pw := io.Pipe() + + // Producer: snapshot -> pipe writer. + go func() { + _, serr := r.Snapshotter.Snapshot(ctx, pod, pw) + // Closing with the error propagates it to the uploader's Read. + _ = pw.CloseWithError(serr) + }() + + // Consumer: pipe reader -> object store. Drives both goroutines. + backup.Status.Phase = ecv1alpha1.BackupPhaseUploading + res, err := store.Upload(ctx, key, pr, -1) + // Ensure the producer is unblocked if Upload returned early. + _ = pr.CloseWithError(err) + if err != nil { + return objectstore.UploadResult{}, fmt.Errorf("snapshot/upload: %w", err) + } + return res, nil +} + +// selectMemberPod resolves the referenced EtcdCluster's StatefulSet and returns +// a member pod to snapshot from. Any ready member yields a consistent +// cluster-wide snapshot, so the lowest-ordinal ready pod is chosen for +// determinism. +func (r *EtcdBackupReconciler) selectMemberPod(ctx context.Context, backup *ecv1alpha1.EtcdBackup) (types.NamespacedName, error) { + var cluster ecv1alpha1.EtcdCluster + clusterKey := types.NamespacedName{Namespace: backup.Namespace, Name: backup.Spec.ClusterRef} + if err := r.Get(ctx, clusterKey, &cluster); err != nil { + if apierrors.IsNotFound(err) { + return types.NamespacedName{}, fmt.Errorf("referenced EtcdCluster %q not found", backup.Spec.ClusterRef) + } + return types.NamespacedName{}, fmt.Errorf("get EtcdCluster %q: %w", backup.Spec.ClusterRef, err) + } + + var sts appsv1.StatefulSet + if err := r.Get(ctx, clusterKey, &sts); err != nil { + return types.NamespacedName{}, fmt.Errorf("get StatefulSet for cluster %q: %w", backup.Spec.ClusterRef, err) + } + if sts.Status.ReadyReplicas < 1 { + return types.NamespacedName{}, fmt.Errorf("cluster %q has no ready members to snapshot", backup.Spec.ClusterRef) + } + + // StatefulSet pods are deterministically named -. + return types.NamespacedName{ + Namespace: backup.Namespace, + Name: fmt.Sprintf("%s-0", sts.Name), + }, nil +} + +// buildStore constructs an objectstore.Store from the backup's destination, +// resolving credentials from the optional secretRef. +func (r *EtcdBackupReconciler) buildStore(ctx context.Context, backup *ecv1alpha1.EtcdBackup) (objectstore.Store, error) { + dst := backup.Spec.Destination + + if err := validateDestination(dst); err != nil { + return nil, err + } + + osDst, err := toObjectStoreDestination(dst) + if err != nil { + return nil, err + } + + creds, err := r.resolveCredentials(ctx, backup.Namespace, dst) + if err != nil { + return nil, err + } + + store, err := r.NewStore(ctx, osDst, creds) + if err != nil { + return nil, fmt.Errorf("build object store: %w", err) + } + return store, nil +} + +// resolveCredentials reads the provider credentials from the referenced secret, +// if any, delegating to the shared, audit-logged resolver so the backup and +// restore paths handle credentials identically (and never log their values). +// With no secretRef the returned Credentials is empty and providers fall back +// to ambient credentials (IRSA/Workload Identity). +func (r *EtcdBackupReconciler) resolveCredentials( + ctx context.Context, namespace string, dst ecv1alpha1.BackupDestination, +) (objectstore.Credentials, error) { + return resolveStoreCredentials(ctx, r.Client, log.FromContext(ctx), namespace, dst) +} + +// applyRetention deletes snapshots beyond the configured RetainCount under the +// backup's bucket/prefix, newest-first. +func (r *EtcdBackupReconciler) applyRetention(ctx context.Context, backup *ecv1alpha1.EtcdBackup, store objectstore.Store) error { + pol := backup.Spec.Retention + if pol == nil || pol.RetainCount <= 0 { + return nil + } + + objs, err := store.List(ctx, snapshotKeyPrefix(backup)) + if err != nil { + return fmt.Errorf("list for retention: %w", err) + } + if int32(len(objs)) <= pol.RetainCount { + return nil + } + + // Listed keys are absolute (they include the destination prefix). Delete + // joins its argument with the destination prefix again, so strip the prefix + // to address each object relative to it. + prefix := backup.Spec.Destination.Prefix + for _, o := range objs[pol.RetainCount:] { + relKey := relativeKey(prefix, o.Key) + if err := store.Delete(ctx, relKey); err != nil { + return fmt.Errorf("delete %q: %w", o.Key, err) + } + } + return nil +} + +func (r *EtcdBackupReconciler) markFailed(backup *ecv1alpha1.EtcdBackup, err error) { + backup.Status.Phase = ecv1alpha1.BackupPhaseFailed + meta.SetStatusCondition(&backup.Status.Conditions, metav1.Condition{ + Type: ecv1alpha1.BackupConditionSucceeded, + Status: metav1.ConditionFalse, + Reason: "BackupFailed", + Message: err.Error(), + ObservedGeneration: backup.Generation, + }) +} + +// SetupWithManager registers the reconciler and wires the default Snapshotter +// and object-store factory if the caller did not inject them. +func (r *EtcdBackupReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.NewStore == nil { + r.NewStore = objectstore.New + } + if r.Snapshotter == nil { + // The default snapshotter talks to etcd over the client API (no in-pod + // exec), so it needs no rest.Config and works against distroless images. + r.Snapshotter = newClientSnapshotter() + } + return ctrl.NewControllerManagedBy(mgr). + For(&ecv1alpha1.EtcdBackup{}). + Complete(r) +} diff --git a/internal/controller/etcdbackup_controller_test.go b/internal/controller/etcdbackup_controller_test.go new file mode 100644 index 00000000..17663294 --- /dev/null +++ b/internal/controller/etcdbackup_controller_test.go @@ -0,0 +1,551 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "bytes" + "context" + "fmt" + "io" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/objectstore" +) + +// --- test doubles ----------------------------------------------------------- + +// fakeSnapshotter returns canned snapshot bytes (or an error) instead of +// exec'ing into a pod. +type fakeSnapshotter struct { + data []byte + err error + gotPod types.NamespacedName + calls int + mu sync.Mutex + delay time.Duration +} + +func (f *fakeSnapshotter) Snapshot(ctx context.Context, pod types.NamespacedName, w io.Writer) (int64, error) { + f.mu.Lock() + f.calls++ + f.gotPod = pod + f.mu.Unlock() + if f.delay > 0 { + select { + case <-time.After(f.delay): + case <-ctx.Done(): + return 0, ctx.Err() + } + } + if f.err != nil { + return 0, f.err + } + n, err := w.Write(f.data) + return int64(n), err +} + +// fakeStore is an in-memory objectstore.Store for controller tests. +type fakeStore struct { + mu sync.Mutex + prefix string + objects map[string][]byte + times map[string]time.Time + clock time.Time + uploadErr error + downloadErr error + deleted []string +} + +func newFakeStore(prefix string) *fakeStore { + return &fakeStore{ + prefix: prefix, + objects: map[string][]byte{}, + times: map[string]time.Time{}, + clock: time.Unix(0, 0), + } +} + +func (s *fakeStore) key(k string) string { return objectstore.JoinKey(s.prefix, k) } + +func (s *fakeStore) Upload(_ context.Context, key string, r io.Reader, _ int64) (objectstore.UploadResult, error) { + if s.uploadErr != nil { + return objectstore.UploadResult{}, s.uploadErr + } + data, err := io.ReadAll(r) + if err != nil { + return objectstore.UploadResult{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + full := s.key(key) + s.objects[full] = data + s.clock = s.clock.Add(time.Second) + s.times[full] = s.clock + return objectstore.UploadResult{URI: "test://" + full, Size: int64(len(data))}, nil +} + +func (s *fakeStore) Download(_ context.Context, key string) (io.ReadCloser, error) { + if s.downloadErr != nil { + return nil, s.downloadErr + } + s.mu.Lock() + defer s.mu.Unlock() + full := s.key(key) + data, ok := s.objects[full] + if !ok { + return nil, fmt.Errorf("fakeStore: %q: %w", full, objectstore.ErrNotFound) + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (s *fakeStore) List(_ context.Context, keyPrefix string) ([]objectstore.ObjectInfo, error) { + s.mu.Lock() + defer s.mu.Unlock() + full := s.key(keyPrefix) + var out []objectstore.ObjectInfo + for k, v := range s.objects { + if len(full) == 0 || hasStorePrefix(k, full) { + out = append(out, objectstore.ObjectInfo{Key: k, Size: int64(len(v)), LastModified: s.times[k]}) + } + } + // newest first + for i := 0; i < len(out); i++ { + for j := i + 1; j < len(out); j++ { + if out[j].LastModified.After(out[i].LastModified) { + out[i], out[j] = out[j], out[i] + } + } + } + return out, nil +} + +func (s *fakeStore) Delete(_ context.Context, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + full := s.key(key) + s.deleted = append(s.deleted, full) + delete(s.objects, full) + delete(s.times, full) + return nil +} + +func (s *fakeStore) Scheme() string { return "test" } + +func hasStorePrefix(s, p string) bool { return len(s) >= len(p) && s[:len(p)] == p } + +// --- fixtures --------------------------------------------------------------- + +func backupScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, ecv1alpha1.AddToScheme(s)) + require.NoError(t, corev1.AddToScheme(s)) + require.NoError(t, appsv1.AddToScheme(s)) + return s +} + +// Shared identifiers for the controller test fixtures. +const ( + testNS = "ns1" + testClusterName = "etcd-a" + testBackupName = "backup-1" +) + +func readyCluster() (*ecv1alpha1.EtcdCluster, *appsv1.StatefulSet) { + c := &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: testClusterName, Namespace: testNS}, + Spec: ecv1alpha1.EtcdClusterSpec{Size: 3, Version: "v3.6.1"}, + } + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: testClusterName, Namespace: testNS}, + Status: appsv1.StatefulSetStatus{ReadyReplicas: 3}, + } + return c, sts +} + +func s3Backup(clusterRef string) *ecv1alpha1.EtcdBackup { + return &ecv1alpha1.EtcdBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: testBackupName, + Namespace: testNS, + CreationTimestamp: metav1.NewTime(time.Date(2026, 6, 17, 1, 2, 3, 0, time.UTC)), + }, + Spec: ecv1alpha1.EtcdBackupSpec{ + ClusterRef: clusterRef, + Destination: ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + Prefix: "etcd/backups", + S3: &ecv1alpha1.S3DestinationSpec{Bucket: "b", Region: "us-east-1"}, + }, + }, + } +} + +func newReconciler(t *testing.T, store *fakeStore, snap *fakeSnapshotter, objs ...client.Object) *EtcdBackupReconciler { + t.Helper() + s := backupScheme(t) + cl := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(objs...). + WithStatusSubresource(&ecv1alpha1.EtcdBackup{}). + Build() + return &EtcdBackupReconciler{ + Client: cl, + Scheme: s, + Snapshotter: snap, + NewStore: func(_ context.Context, _ objectstore.Destination, _ objectstore.Credentials) (objectstore.Store, error) { + return store, nil + }, + } +} + +// --- tests ------------------------------------------------------------------ + +func TestReconcile_HappyPath(t *testing.T) { + cluster, sts := readyCluster() + backup := s3Backup("etcd-a") + store := newFakeStore("etcd/backups") + snap := &fakeSnapshotter{data: []byte("SNAPSHOTDATA")} + + r := newReconciler(t, store, snap, cluster, sts, backup) + + res, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, + }) + require.NoError(t, err) + assert.True(t, res.IsZero()) + + // Status reflects success. + var got ecv1alpha1.EtcdBackup + require.NoError(t, r.Get(context.Background(), types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, &got)) + assert.Equal(t, ecv1alpha1.BackupPhaseCompleted, got.Status.Phase) + assert.Equal(t, int64(len("SNAPSHOTDATA")), got.Status.SnapshotSizeBytes) + assert.NotNil(t, got.Status.CompletionTime) + assert.Contains(t, got.Status.SnapshotLocation, "etcd/backups/etcd-a/backup-1-") + cond := meta.FindStatusCondition(got.Status.Conditions, ecv1alpha1.BackupConditionSucceeded) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionTrue, cond.Status) + + // Snapshotter was asked for the lowest-ordinal pod. + assert.Equal(t, "etcd-a-0", snap.gotPod.Name) + assert.Equal(t, "ns1", snap.gotPod.Namespace) + + // The bytes actually landed in the store under the joined key. + require.Len(t, store.objects, 1) + for k, v := range store.objects { + assert.Contains(t, k, "etcd/backups/etcd-a/backup-1-") + assert.Equal(t, []byte("SNAPSHOTDATA"), v) + } +} + +func TestReconcile_RetentionKeepsJustUploadedSnapshot(t *testing.T) { + // End-to-end: with RetainCount=1, after a successful snapshot the only + // surviving object must be the one this reconcile just uploaded — never an + // older pre-existing snapshot, and the new one must not be pruned. + cluster, sts := readyCluster() + backup := s3Backup("etcd-a") + backup.Spec.Retention = &ecv1alpha1.RetentionPolicy{RetainCount: 1} + store := newFakeStore("etcd/backups") + + // Seed two older snapshots (clock advances per upload, so these predate the + // reconcile's upload). + for _, k := range []string{"etcd-a/backup-1-20200101T000000Z.db", "etcd-a/backup-1-20210101T000000Z.db"} { + _, err := store.Upload(context.Background(), k, readerOf("old"), -1) + require.NoError(t, err) + } + + snap := &fakeSnapshotter{data: []byte("FRESH")} + r := newReconciler(t, store, snap, cluster, sts, backup) + + _, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, + }) + require.NoError(t, err) + + require.Len(t, store.objects, 1, "RetainCount=1 keeps exactly one snapshot") + for k, v := range store.objects { + assert.Equal(t, []byte("FRESH"), v, "the surviving snapshot must be the just-uploaded one") + assert.NotContains(t, k, "20200101") + assert.NotContains(t, k, "20210101") + } +} + +func TestReconcile_MissingClusterFails(t *testing.T) { + backup := s3Backup("absent") + store := newFakeStore("") + snap := &fakeSnapshotter{data: []byte("x")} + r := newReconciler(t, store, snap, backup) + + _, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, + }) + require.Error(t, err) + + var got ecv1alpha1.EtcdBackup + require.NoError(t, r.Get(context.Background(), types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, &got)) + assert.Equal(t, ecv1alpha1.BackupPhaseFailed, got.Status.Phase) + assert.Equal(t, 0, snap.calls, "snapshot must not run when cluster is missing") +} + +func TestReconcile_NoReadyMembersFails(t *testing.T) { + cluster, sts := readyCluster() + sts.Status.ReadyReplicas = 0 + backup := s3Backup("etcd-a") + store := newFakeStore("") + snap := &fakeSnapshotter{data: []byte("x")} + r := newReconciler(t, store, snap, cluster, sts, backup) + + _, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, + }) + require.Error(t, err) + assert.Equal(t, 0, snap.calls) +} + +func TestReconcile_SnapshotErrorMarksFailed(t *testing.T) { + cluster, sts := readyCluster() + backup := s3Backup("etcd-a") + store := newFakeStore("etcd/backups") + snap := &fakeSnapshotter{err: fmt.Errorf("etcdctl boom")} + r := newReconciler(t, store, snap, cluster, sts, backup) + + _, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, + }) + require.Error(t, err) + + var got ecv1alpha1.EtcdBackup + require.NoError(t, r.Get(context.Background(), types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, &got)) + assert.Equal(t, ecv1alpha1.BackupPhaseFailed, got.Status.Phase) + cond := meta.FindStatusCondition(got.Status.Conditions, ecv1alpha1.BackupConditionSucceeded) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Empty(t, store.objects, "nothing should be uploaded on snapshot failure") +} + +func TestReconcile_UploadErrorMarksFailed(t *testing.T) { + cluster, sts := readyCluster() + backup := s3Backup("etcd-a") + store := newFakeStore("etcd/backups") + store.uploadErr = fmt.Errorf("s3 unavailable") + snap := &fakeSnapshotter{data: []byte("data")} + r := newReconciler(t, store, snap, cluster, sts, backup) + + _, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, + }) + require.Error(t, err) + + var got ecv1alpha1.EtcdBackup + require.NoError(t, r.Get(context.Background(), types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, &got)) + assert.Equal(t, ecv1alpha1.BackupPhaseFailed, got.Status.Phase) +} + +func TestReconcile_TerminalIsNoOp(t *testing.T) { + cluster, sts := readyCluster() + backup := s3Backup("etcd-a") + backup.Status.Phase = ecv1alpha1.BackupPhaseCompleted + store := newFakeStore("") + snap := &fakeSnapshotter{data: []byte("x")} + r := newReconciler(t, store, snap, cluster, sts, backup) + + _, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, + }) + require.NoError(t, err) + assert.Equal(t, 0, snap.calls, "completed backup must not re-run") +} + +func TestReconcile_SnapshotTimeout(t *testing.T) { + cluster, sts := readyCluster() + backup := s3Backup("etcd-a") + backup.Spec.SnapshotTimeout = &metav1.Duration{Duration: 20 * time.Millisecond} + store := newFakeStore("etcd/backups") + snap := &fakeSnapshotter{data: []byte("data"), delay: 500 * time.Millisecond} + r := newReconciler(t, store, snap, cluster, sts, backup) + + _, err := r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, + }) + require.Error(t, err) + var got ecv1alpha1.EtcdBackup + require.NoError(t, r.Get(context.Background(), types.NamespacedName{Name: "backup-1", Namespace: "ns1"}, &got)) + assert.Equal(t, ecv1alpha1.BackupPhaseFailed, got.Status.Phase) +} + +func TestResolveCredentials_S3FromSecret(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "creds", Namespace: "ns1"}, + Data: map[string][]byte{ + "accessKeyID": []byte("AKID"), + "secretAccessKey": []byte("SECRET"), + "sessionToken": []byte("TOKEN"), + }, + } + r := newReconciler(t, newFakeStore(""), &fakeSnapshotter{}, secret) + + creds, err := r.resolveCredentials(context.Background(), "ns1", ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + SecretRef: &corev1.LocalObjectReference{Name: "creds"}, + }) + require.NoError(t, err) + assert.Equal(t, "AKID", creds.AccessKeyID) + assert.Equal(t, "SECRET", creds.SecretAccessKey) + assert.Equal(t, "TOKEN", creds.SessionToken) +} + +func TestResolveCredentials_GCSFromSecret(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "gcskey", Namespace: "ns1"}, + Data: map[string][]byte{"serviceAccountJSON": []byte(`{"type":"service_account"}`)}, + } + r := newReconciler(t, newFakeStore(""), &fakeSnapshotter{}, secret) + + creds, err := r.resolveCredentials(context.Background(), "ns1", ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderGCS, + SecretRef: &corev1.LocalObjectReference{Name: "gcskey"}, + }) + require.NoError(t, err) + assert.JSONEq(t, `{"type":"service_account"}`, string(creds.ServiceAccountJSON)) +} + +func TestResolveCredentials_NoSecretIsAmbient(t *testing.T) { + r := newReconciler(t, newFakeStore(""), &fakeSnapshotter{}) + creds, err := r.resolveCredentials(context.Background(), "ns1", ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + }) + require.NoError(t, err) + assert.Empty(t, creds.AccessKeyID) +} + +func TestResolveCredentials_MissingSecretErrors(t *testing.T) { + r := newReconciler(t, newFakeStore(""), &fakeSnapshotter{}) + _, err := r.resolveCredentials(context.Background(), "ns1", ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + SecretRef: &corev1.LocalObjectReference{Name: "nope"}, + }) + assert.Error(t, err) +} + +func TestApplyRetention_PrunesOldest(t *testing.T) { + backup := s3Backup("etcd-a") + backup.Spec.Retention = &ecv1alpha1.RetentionPolicy{RetainCount: 2} + store := newFakeStore("etcd/backups") + + // Seed three existing snapshots under the cluster prefix with increasing + // modtimes (store advances its clock per upload). + for _, k := range []string{"etcd-a/old1.db", "etcd-a/old2.db", "etcd-a/old3.db"} { + _, err := store.Upload(context.Background(), k, readerOf("x"), -1) + require.NoError(t, err) + } + + r := &EtcdBackupReconciler{} + require.NoError(t, r.applyRetention(context.Background(), backup, store)) + + // RetainCount=2 keeps the two newest; exactly one (the oldest) deleted. + require.Len(t, store.deleted, 1) + assert.Equal(t, "etcd/backups/etcd-a/old1.db", store.deleted[0]) + assert.Len(t, store.objects, 2) +} + +func TestApplyRetention_DisabledByDefault(t *testing.T) { + backup := s3Backup("etcd-a") + store := newFakeStore("etcd/backups") + for _, k := range []string{"etcd-a/a.db", "etcd-a/b.db"} { + _, err := store.Upload(context.Background(), k, readerOf("x"), -1) + require.NoError(t, err) + } + r := &EtcdBackupReconciler{} + require.NoError(t, r.applyRetention(context.Background(), backup, store)) + assert.Empty(t, store.deleted) +} + +func TestToObjectStoreDestination(t *testing.T) { + t.Run("s3 ok", func(t *testing.T) { + dst, err := toObjectStoreDestination(ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + Prefix: "p", + S3: &ecv1alpha1.S3DestinationSpec{Bucket: "b", Region: "r", Endpoint: "http://e", ForcePathStyle: true}, + }) + require.NoError(t, err) + assert.Equal(t, objectstore.ProviderS3, dst.Provider) + assert.Equal(t, "b", dst.Bucket) + assert.Equal(t, "r", dst.Region) + assert.True(t, dst.ForcePathStyle) + }) + t.Run("s3 missing block", func(t *testing.T) { + _, err := toObjectStoreDestination(ecv1alpha1.BackupDestination{Provider: ecv1alpha1.BackupProviderS3}) + assert.Error(t, err) + }) + t.Run("gcs ok", func(t *testing.T) { + dst, err := toObjectStoreDestination(ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderGCS, + GCS: &ecv1alpha1.GCSDestinationSpec{Bucket: "gb"}, + }) + require.NoError(t, err) + assert.Equal(t, objectstore.ProviderGCS, dst.Provider) + assert.Equal(t, "gb", dst.Bucket) + }) + t.Run("gcs missing block", func(t *testing.T) { + _, err := toObjectStoreDestination(ecv1alpha1.BackupDestination{Provider: ecv1alpha1.BackupProviderGCS}) + assert.Error(t, err) + }) + t.Run("unknown provider", func(t *testing.T) { + _, err := toObjectStoreDestination(ecv1alpha1.BackupDestination{Provider: "azure"}) + assert.Error(t, err) + }) +} + +func TestRelativeKey(t *testing.T) { + assert.Equal(t, "etcd-a/x.db", relativeKey("etcd/backups", "etcd/backups/etcd-a/x.db")) + assert.Equal(t, "etcd-a/x.db", relativeKey("/etcd/backups/", "etcd/backups/etcd-a/x.db")) + assert.Equal(t, "etcd-a/x.db", relativeKey("", "etcd-a/x.db")) + // key not under prefix returns as-is (trimmed) + assert.Equal(t, "other/x.db", relativeKey("etcd", "other/x.db")) +} + +// readerOf is a tiny helper returning an io.Reader over a string. +func readerOf(s string) io.Reader { return &stringReader{s: s} } + +type stringReader struct { + s string + i int +} + +func (r *stringReader) Read(p []byte) (int, error) { + if r.i >= len(r.s) { + return 0, io.EOF + } + n := copy(p, r.s[r.i:]) + r.i += n + return n, nil +} diff --git a/internal/controller/etcdbackup_helpers.go b/internal/controller/etcdbackup_helpers.go new file mode 100644 index 00000000..2f1bcee7 --- /dev/null +++ b/internal/controller/etcdbackup_helpers.go @@ -0,0 +1,92 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + "strings" + "time" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/objectstore" +) + +// snapshotKeyPrefix is the per-cluster prefix under which a backup's snapshots +// are written. Keeping all snapshots for a cluster under a shared prefix lets +// retention list and prune them as a group. +func snapshotKeyPrefix(backup *ecv1alpha1.EtcdBackup) string { + return backup.Spec.ClusterRef +} + +// snapshotObjectKey is the object key (relative to the destination prefix) for +// this backup's snapshot. It embeds the cluster, the backup name, and the +// creation time so keys are unique and sortable. +func snapshotObjectKey(backup *ecv1alpha1.EtcdBackup) string { + ts := backup.CreationTimestamp.Time + if ts.IsZero() { + ts = time.Now() + } + return fmt.Sprintf("%s/%s-%s.db", + backup.Spec.ClusterRef, + backup.Name, + ts.UTC().Format("20060102T150405Z"), + ) +} + +// relativeKey strips the destination prefix from an absolute (listed) object +// key so the result can be re-joined with the same prefix by Store.Delete +// without double-prefixing. Both sides are normalized for slashes. +func relativeKey(prefix, absKey string) string { + prefix = strings.Trim(prefix, "/") + absKey = strings.TrimLeft(absKey, "/") + if prefix == "" { + return absKey + } + if strings.HasPrefix(absKey, prefix+"/") { + return absKey[len(prefix)+1:] + } + return absKey +} + +// toObjectStoreDestination converts the API destination into the +// provider-agnostic objectstore.Destination, validating that the +// provider-specific block matches the selected provider. +func toObjectStoreDestination(dst ecv1alpha1.BackupDestination) (objectstore.Destination, error) { + out := objectstore.Destination{ + Provider: objectstore.Provider(dst.Provider), + Prefix: dst.Prefix, + } + switch dst.Provider { + case ecv1alpha1.BackupProviderS3: + if dst.S3 == nil { + return out, fmt.Errorf("destination.s3 is required when provider is %q", dst.Provider) + } + out.Bucket = dst.S3.Bucket + out.Region = dst.S3.Region + out.Endpoint = dst.S3.Endpoint + out.ForcePathStyle = dst.S3.ForcePathStyle + case ecv1alpha1.BackupProviderGCS: + if dst.GCS == nil { + return out, fmt.Errorf("destination.gcs is required when provider is %q", dst.Provider) + } + out.Bucket = dst.GCS.Bucket + out.Endpoint = dst.GCS.Endpoint + default: + return out, fmt.Errorf("unsupported backup provider %q", dst.Provider) + } + return out, nil +} diff --git a/internal/controller/etcdrestore_controller.go b/internal/controller/etcdrestore_controller.go new file mode 100644 index 00000000..1e167895 --- /dev/null +++ b/internal/controller/etcdrestore_controller.go @@ -0,0 +1,567 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/objectstore" +) + +const ( + // defaultRestoreTimeout bounds the download+restore step when the spec omits one. + defaultRestoreTimeout = 10 * time.Minute + // restoreRequeueInterval is how often the reconciler re-checks a stamped + // restore-target cluster while waiting for its genesis member to boot from + // the restored data dir. + restoreRequeueInterval = 5 * time.Second +) + +// EtcdRestoreReconciler reconciles an EtcdRestore object: it downloads a +// snapshot from object storage and bootstraps a new EtcdCluster from it. +// +// Like EtcdBackupReconciler, the reconciler is intentionally thin: the +// cloud-specific behaviour lives behind two seams — objectstore.Factory (read +// the snapshot from a bucket) and Restorer (write it into a fresh member and +// bootstrap). Both are injectable so the orchestration is exercised by unit +// tests with no cloud creds and no live etcd. +type EtcdRestoreReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + // RESTConfig is retained for parity with the other controllers (and any + // future in-pod operations); the restore data path no longer execs into pods. + RESTConfig *rest.Config + + // OperatorImage is the operator's own image, stamped onto the restore-target + // EtcdCluster so the cluster controller can inject a restore init-container + // (the operator image, `manager restore-localize`) that downloads the + // snapshot and lays the genesis data dir before etcd starts. Required at + // runtime; tests set it explicitly. + OperatorImage string + + // NewStore builds an object store from a destination and credentials. + // Defaults to objectstore.New; overridden in tests with a fake. It is used + // only to eagerly verify the snapshot object is reachable (so a missing + // object fails the restore terminally and promptly), not to stream bytes — + // the member's init-container performs the actual download+restore. + NewStore objectstore.Factory +} + +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdrestores,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdrestores/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdrestores/finalizers,verbs=update +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdclusters,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=operator.etcd.io,resources=etcdbackups,verbs=get;list;watch +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch + +// Reconcile drives a single EtcdRestore to completion. A restore is a one-shot +// job, so a terminal phase (Completed/Failed) short-circuits further work. +func (r *EtcdRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + var restore ecv1alpha1.EtcdRestore + if err := r.Get(ctx, req.NamespacedName, &restore); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Terminal restores are immutable records; nothing more to do. + if restore.Status.Phase == ecv1alpha1.RestorePhaseCompleted || + restore.Status.Phase == ecv1alpha1.RestorePhaseFailed { + return ctrl.Result{}, nil + } + + restore.Status.ObservedGeneration = restore.Generation + + result, err := r.runRestore(ctx, &restore) + if err != nil { + logger.Error(err, "restore failed", "restore", req.NamespacedName) + r.markFailed(&restore, err) + } + + if statusErr := r.Status().Update(ctx, &restore); statusErr != nil { + if err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, statusErr + } + return result, err +} + +// runRestore drives the restore as: validate -> resolve source -> verify the +// snapshot object is reachable -> ensure+STAMP the target cluster -> WATCH the +// genesis member boot from the restored data. +// +// This is the init-container (pull) model: the controller no longer streams +// snapshot bytes into a member. Instead it stamps the snapshot source onto the +// target EtcdCluster (restore-source annotation); the cluster controller injects +// a restore init-container that downloads the snapshot and lays the genesis data +// dir before etcd starts. The controller's job shrinks to stamping the source +// and observing the member become Ready (or fail), so Completed truthfully means +// "a member booted from the restored data", not merely "the spec was injected". +func (r *EtcdRestoreReconciler) runRestore(ctx context.Context, restore *ecv1alpha1.EtcdRestore) (ctrl.Result, error) { + // 1. Validate the spec up front so misconfiguration is a clean, terminal + // failure rather than a partial restore. + if err := validateRestoreSpec(&restore.Spec); err != nil { + return ctrl.Result{}, err + } + + // 2. Resolve the snapshot source into a concrete destination + object key + + // cluster version (the latter from the referenced backup, when used). + src, err := r.resolveSource(ctx, restore) + if err != nil { + return ctrl.Result{}, err + } + + // 3. Build the object store and eagerly verify the snapshot object is + // reachable. This surfaces a missing object as a terminal, prompt failure + // here (the message contains "download"/"not found") rather than letting a + // member pod crash-loop on a 404 download. We open the body and close it + // immediately — the member's init-container performs the real download. + store, err := r.buildStore(ctx, restore.Namespace, src.destination) + if err != nil { + return ctrl.Result{}, err + } + timeout := defaultRestoreTimeout + if restore.Spec.RestoreTimeout != nil { + timeout = restore.Spec.RestoreTimeout.Duration + } + // Verify reachability only on the first pass (before we enter Restoring and + // stamp the target); subsequent requeues are just watching the member boot, + // so re-opening the object every interval would be wasteful. + if restore.Status.Phase != ecv1alpha1.RestorePhaseRestoring { + vctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + if err := r.verifySnapshotReachable(vctx, store, src.key); err != nil { + return ctrl.Result{}, err + } + } + + // 4. Ensure the target is a fresh, empty cluster (create if absent, reject if + // it already has members) and STAMP the restore-source annotation on it so + // the cluster controller injects the restore init-container. + if restore.Status.Phase == "" || restore.Status.Phase == ecv1alpha1.RestorePhasePending { + restore.Status.Phase = ecv1alpha1.RestorePhaseDownloading + } + target, err := r.ensureStampedTargetCluster(ctx, restore, src, store) + if err != nil { + return ctrl.Result{}, err + } + + // 5. WATCH: the genesis member must boot from the restored data dir. Detect a + // wedged restore init-container (corrupt snapshot, bad creds) as a terminal + // failure; mark Completed once the StatefulSet reports a ready member. + restore.Status.Phase = ecv1alpha1.RestorePhaseRestoring + ready, failErr, err := r.observeRestoreProgress(ctx, target) + if err != nil { + return ctrl.Result{}, err + } + if failErr != nil { + return ctrl.Result{}, failErr + } + if !ready { + // Still booting; requeue without churning the status. + return ctrl.Result{RequeueAfter: restoreRequeueInterval}, nil + } + + // 6. Record success — a member booted from the restored data. + now := metav1.Now() + restore.Status.Phase = ecv1alpha1.RestorePhaseCompleted + restore.Status.SnapshotLocation = fmt.Sprintf("%s://%s", store.Scheme(), + objectstore.JoinKey(src.destination.Prefix, src.key)) + restore.Status.RestoredCluster = target.Name + restore.Status.CompletionTime = &now + meta.SetStatusCondition(&restore.Status.Conditions, metav1.Condition{ + Type: ecv1alpha1.RestoreConditionSucceeded, + Status: metav1.ConditionTrue, + Reason: "SnapshotRestored", + Message: fmt.Sprintf("snapshot %q restored into cluster %q", src.key, target.Name), + ObservedGeneration: restore.Generation, + }) + r.event(restore, corev1.EventTypeNormal, "Restored", + fmt.Sprintf("restored snapshot into EtcdCluster %q", target.Name)) + + return ctrl.Result{}, nil +} + +// verifySnapshotReachable opens the snapshot object and reads a byte to confirm +// it exists and is fetchable, then closes it. A missing object surfaces as a +// terminal failure whose message contains "download"/"not found", satisfying the +// missing-object contract without waiting for a member pod to crash-loop. +func (r *EtcdRestoreReconciler) verifySnapshotReachable( + ctx context.Context, store objectstore.Store, key string, +) error { + body, err := store.Download(ctx, key) + if err != nil { + if errors.Is(err, objectstore.ErrNotFound) { + return fmt.Errorf("download snapshot %q: not found: %w", key, err) + } + return fmt.Errorf("download snapshot %q: %w", key, err) + } + defer func() { _ = body.Close() }() + buf := make([]byte, 1) + if _, rerr := body.Read(buf); rerr != nil && !errors.Is(rerr, io.EOF) { + return fmt.Errorf("download snapshot %q: %w", key, rerr) + } + return nil +} + +// resolvedSource is the flattened source of a restore: where the snapshot lives +// and the etcd version to stamp the target cluster with. +type resolvedSource struct { + destination ecv1alpha1.BackupDestination + key string + version string +} + +// resolveSource turns the spec's Source (backupRef | location) into a concrete +// destination + object key. For a backupRef it reads the referenced EtcdBackup, +// requiring it to have completed, and derives the key from its recorded +// snapshot location relative to the destination prefix. +func (r *EtcdRestoreReconciler) resolveSource(ctx context.Context, restore *ecv1alpha1.EtcdRestore) (resolvedSource, error) { + src := restore.Spec.Source + if src.Location != nil { + return resolvedSource{ + destination: src.Location.Destination, + key: src.Location.Key, + }, nil + } + + // backupRef path. + var backup ecv1alpha1.EtcdBackup + key := types.NamespacedName{Namespace: restore.Namespace, Name: src.BackupRef.Name} + if err := r.Get(ctx, key, &backup); err != nil { + if apierrors.IsNotFound(err) { + return resolvedSource{}, fmt.Errorf("referenced EtcdBackup %q not found", src.BackupRef.Name) + } + return resolvedSource{}, fmt.Errorf("get EtcdBackup %q: %w", src.BackupRef.Name, err) + } + if backup.Status.Phase != ecv1alpha1.BackupPhaseCompleted { + return resolvedSource{}, fmt.Errorf( + "referenced EtcdBackup %q has not completed (phase %q)", src.BackupRef.Name, backup.Status.Phase) + } + + objKey, err := snapshotKeyFromBackup(&backup) + if err != nil { + return resolvedSource{}, err + } + return resolvedSource{ + destination: backup.Spec.Destination, + key: objKey, + // Best-effort version hint: if the source EtcdCluster the backup was + // taken from still exists, inherit its version so the operator need not + // restate it. The caller falls back to target.Version (and errors if + // neither is available). + version: r.sourceClusterVersion(ctx, restore.Namespace, backup.Spec.ClusterRef), + }, nil +} + +// sourceClusterVersion returns the etcd version of the named cluster if it +// still exists, or "" otherwise. It is a best-effort hint, never an error: the +// originating cluster is commonly gone by restore time. +func (r *EtcdRestoreReconciler) sourceClusterVersion(ctx context.Context, namespace, name string) string { + var cluster ecv1alpha1.EtcdCluster + if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &cluster); err != nil { + return "" + } + return cluster.Spec.Version +} + +// buildStore constructs an objectstore.Store from a destination, resolving and +// audit-logging credentials. +func (r *EtcdRestoreReconciler) buildStore( + ctx context.Context, namespace string, dst ecv1alpha1.BackupDestination, +) (objectstore.Store, error) { + osDst, err := toObjectStoreDestination(dst) + if err != nil { + return nil, err + } + creds, err := resolveStoreCredentials(ctx, r.Client, log.FromContext(ctx), namespace, dst) + if err != nil { + return nil, err + } + store, err := r.NewStore(ctx, osDst, creds) + if err != nil { + return nil, fmt.Errorf("build object store: %w", err) + } + return store, nil +} + +// ensureStampedTargetCluster creates the target EtcdCluster if it does not +// exist, or verifies that an existing one is empty (no ready members), and in +// both cases ensures it carries the restore-source annotation so the cluster +// controller injects the restore init-container. It returns the cluster. +// +// The annotation is the single gate that turns a normal cluster into a +// restore-target; it is set at creation (so the very first StatefulSet is a +// restore pod template) and reconciled onto an existing empty shell. +func (r *EtcdRestoreReconciler) ensureStampedTargetCluster( + ctx context.Context, restore *ecv1alpha1.EtcdRestore, src resolvedSource, store objectstore.Store, +) (*ecv1alpha1.EtcdCluster, error) { + t := restore.Spec.Target + clusterKey := types.NamespacedName{Namespace: restore.Namespace, Name: t.Name} + + annotationValue, err := r.buildRestoreAnnotation(src, store) + if err != nil { + return nil, err + } + + var existing ecv1alpha1.EtcdCluster + getErr := r.Get(ctx, clusterKey, &existing) + switch { + case getErr == nil: + // A cluster already exists: it must be an empty shell, never an active + // cluster with data, or the restore would silently diverge from it. + if err := r.assertClusterEmpty(ctx, &existing); err != nil { + return nil, err + } + // Reconcile the restore annotation onto the empty shell if missing. + if existing.Annotations[restoreSourceAnnotation] != annotationValue { + if existing.Annotations == nil { + existing.Annotations = map[string]string{} + } + existing.Annotations[restoreSourceAnnotation] = annotationValue + if err := r.Update(ctx, &existing); err != nil { + return nil, fmt.Errorf("stamp restore annotation on EtcdCluster %q: %w", t.Name, err) + } + } + return &existing, nil + case apierrors.IsNotFound(getErr): + // Fall through to create a fresh, stamped cluster. + default: + return nil, fmt.Errorf("get target EtcdCluster %q: %w", t.Name, getErr) + } + + version := t.Version + if version == "" { + version = src.version + } + if version == "" { + return nil, fmt.Errorf("target.version must be set when restoring from an explicit location") + } + size := t.Size + if size == 0 { + size = 1 + } + + cluster := &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: t.Name, + Namespace: restore.Namespace, + Annotations: map[string]string{restoreSourceAnnotation: annotationValue}, + }, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: size, + Version: version, + StorageSpec: t.StorageSpec, + }, + } + // Own the created cluster so deleting the EtcdRestore can GC the restored + // cluster if desired, and so the lineage is discoverable. + if err := ctrl.SetControllerReference(restore, cluster, r.Scheme); err != nil { + return nil, fmt.Errorf("set owner on target cluster: %w", err) + } + if err := r.Create(ctx, cluster); err != nil { + return nil, fmt.Errorf("create target EtcdCluster %q: %w", t.Name, err) + } + r.event(restore, corev1.EventTypeNormal, "ClusterCreated", + fmt.Sprintf("created EtcdCluster %q for restore", t.Name)) + return cluster, nil +} + +// buildRestoreAnnotation assembles the JSON restore-source annotation the +// cluster controller reads to inject the restore init-container. It carries the +// snapshot addressing, the creds Secret *name* (never values), the operator +// image, and a generation token (the snapshot key) for the init-container's +// idempotency marker. +func (r *EtcdRestoreReconciler) buildRestoreAnnotation( + src resolvedSource, store objectstore.Store, +) (string, error) { + if r.OperatorImage == "" { + return "", fmt.Errorf("operator image is not configured; cannot build restore init-container " + + "(set --operator-image / OPERATOR_IMAGE)") + } + osDst, err := toObjectStoreDestination(src.destination) + if err != nil { + return "", err + } + credsName := "" + if src.destination.SecretRef != nil { + credsName = src.destination.SecretRef.Name + } + rs := restoreSource{ + Provider: string(osDst.Provider), + Bucket: osDst.Bucket, + Prefix: osDst.Prefix, + Key: src.key, + Region: osDst.Region, + Endpoint: osDst.Endpoint, + ForcePathStyle: osDst.ForcePathStyle, + CredsSecretName: credsName, + OperatorImage: r.OperatorImage, + Generation: src.key, + } + return encodeRestoreSource(rs) +} + +// observeRestoreProgress inspects the restore-target cluster's StatefulSet and +// genesis pod. It returns (ready=true) once a member reports Ready (the restore +// init-container ran and etcd booted from the restored data); a non-nil failErr +// when the restore init-container is wedged (corrupt snapshot, bad creds, +// CrashLoopBackOff) so the restore is marked Failed for the right reason; and +// (false,nil,nil) while the member is still coming up (caller requeues). +func (r *EtcdRestoreReconciler) observeRestoreProgress( + ctx context.Context, target *ecv1alpha1.EtcdCluster, +) (ready bool, failErr error, err error) { + var sts appsv1.StatefulSet + stsKey := types.NamespacedName{Namespace: target.Namespace, Name: target.Name} + if getErr := r.Get(ctx, stsKey, &sts); getErr != nil { + if apierrors.IsNotFound(getErr) { + // StatefulSet not created yet; keep waiting. + return false, nil, nil + } + return false, nil, fmt.Errorf("get StatefulSet for restore target %q: %w", target.Name, getErr) + } + if sts.Status.ReadyReplicas > 0 { + return true, nil, nil + } + + // Not ready yet: check the genesis pod for a wedged restore init-container so + // a corrupt snapshot / bad creds fails terminally instead of looping forever. + genesisPod := fmt.Sprintf("%s-0", target.Name) + var pod corev1.Pod + if getErr := r.Get(ctx, types.NamespacedName{Namespace: target.Namespace, Name: genesisPod}, &pod); getErr != nil { + // No pod yet (or transient): keep waiting. + return false, nil, nil + } + if reason := restoreInitContainerFailure(&pod); reason != "" { + return false, fmt.Errorf("restore init-container failed: %s", reason), nil + } + return false, nil, nil +} + +// restoreInitContainerFailure returns a non-empty diagnosis if the restore +// init-container has terminated non-zero or is stuck in a known-fatal waiting +// state. A snapshot that is corrupt makes `restore-localize`'s in-process +// snapshot.Restore exit non-zero, surfacing here. +func restoreInitContainerFailure(pod *corev1.Pod) string { + for _, cs := range pod.Status.InitContainerStatuses { + if cs.Name != restoreInitContainerName { + continue + } + if t := cs.LastTerminationState.Terminated; t != nil && t.ExitCode != 0 { + return fmt.Sprintf("%s exited %d: %s", cs.Name, t.ExitCode, t.Reason) + } + if t := cs.State.Terminated; t != nil && t.ExitCode != 0 { + return fmt.Sprintf("%s exited %d: %s", cs.Name, t.ExitCode, t.Reason) + } + if w := cs.State.Waiting; w != nil && w.Reason == "CrashLoopBackOff" { + return fmt.Sprintf("%s in CrashLoopBackOff: %s", cs.Name, w.Message) + } + } + return "" +} + +// assertClusterEmpty rejects restoring into a cluster that already has members, +// UNLESS that cluster is already stamped as this restore's target. Emptiness is +// judged by the backing StatefulSet's ready replicas: a restore must be the +// cluster's genesis, so a ready member on an UNSTAMPED cluster means foreign data +// already exists (reject). A ready member on an already-STAMPED cluster is our +// own restored genesis booting — the success signal, not a populated-target +// rejection — so it is allowed (this reconciles the controller's "empty only" +// guard with the init-container's "skip if already restored" idempotency). +func (r *EtcdRestoreReconciler) assertClusterEmpty(ctx context.Context, cluster *ecv1alpha1.EtcdCluster) error { + if _, alreadyStamped := cluster.Annotations[restoreSourceAnnotation]; alreadyStamped { + // Restore already in progress against this cluster; ready members are our + // own restored member, not a foreign populated cluster. + return nil + } + var sts appsv1.StatefulSet + key := types.NamespacedName{Namespace: cluster.Namespace, Name: cluster.Name} + if err := r.Get(ctx, key, &sts); err != nil { + if apierrors.IsNotFound(err) { + // Cluster object exists but no StatefulSet yet: an empty shell, OK. + return nil + } + return fmt.Errorf("get StatefulSet for target cluster %q: %w", cluster.Name, err) + } + if sts.Status.ReadyReplicas > 0 { + return fmt.Errorf( + "target EtcdCluster %q already has %d ready member(s); refusing to restore over a non-empty cluster", + cluster.Name, sts.Status.ReadyReplicas) + } + return nil +} + +func (r *EtcdRestoreReconciler) markFailed(restore *ecv1alpha1.EtcdRestore, err error) { + restore.Status.Phase = ecv1alpha1.RestorePhaseFailed + meta.SetStatusCondition(&restore.Status.Conditions, metav1.Condition{ + Type: ecv1alpha1.RestoreConditionSucceeded, + Status: metav1.ConditionFalse, + Reason: "RestoreFailed", + Message: err.Error(), + ObservedGeneration: restore.Generation, + }) + r.event(restore, corev1.EventTypeWarning, "RestoreFailed", err.Error()) +} + +// event records a Kubernetes Event when a Recorder is wired (it is nil in unit +// tests that do not exercise events). +func (r *EtcdRestoreReconciler) event(restore *ecv1alpha1.EtcdRestore, eventtype, reason, message string) { + if r.Recorder == nil { + return + } + r.Recorder.Event(restore, eventtype, reason, message) +} + +// SetupWithManager registers the reconciler and wires the default object-store +// factory if the caller did not inject one. The reconciler owns the target +// EtcdCluster, so a periodic requeue (RequeueAfter) drives the watch for the +// genesis member becoming Ready; no extra StatefulSet watch is required for +// correctness. +func (r *EtcdRestoreReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.NewStore == nil { + r.NewStore = objectstore.New + } + if r.Recorder == nil { + r.Recorder = mgr.GetEventRecorderFor("etcdrestore-controller") + } + return ctrl.NewControllerManagedBy(mgr). + For(&ecv1alpha1.EtcdRestore{}). + Owns(&ecv1alpha1.EtcdCluster{}). + Complete(r) +} diff --git a/internal/controller/etcdrestore_controller_test.go b/internal/controller/etcdrestore_controller_test.go new file mode 100644 index 00000000..64ff2b95 --- /dev/null +++ b/internal/controller/etcdrestore_controller_test.go @@ -0,0 +1,474 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/objectstore" +) + +// --- fixtures --------------------------------------------------------------- + +const ( + testRestoreName = "restore-1" + testOperatorImg = "ghcr.io/etcd/etcd-operator:test" + restoreReadySize = 1 +) + +func newRestoreReconciler( + t *testing.T, store *fakeStore, objs ...client.Object, +) *EtcdRestoreReconciler { + t.Helper() + s := backupScheme(t) + cl := fake.NewClientBuilder(). + WithScheme(s). + WithObjects(objs...). + WithStatusSubresource(&ecv1alpha1.EtcdRestore{}). + Build() + return &EtcdRestoreReconciler{ + Client: cl, + Scheme: s, + OperatorImage: testOperatorImg, + NewStore: func(_ context.Context, _ objectstore.Destination, _ objectstore.Credentials) (objectstore.Store, error) { + return store, nil + }, + } +} + +// locationRestore builds an EtcdRestore that restores an explicit object key. +func locationRestore(targetName, version string) *ecv1alpha1.EtcdRestore { + return &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: testRestoreName, Namespace: testNS}, + Spec: ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{ + Location: &ecv1alpha1.SnapshotLocation{ + Destination: ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + Prefix: "etcd/backups", + S3: &ecv1alpha1.S3DestinationSpec{Bucket: "b", Region: "us-east-1"}, + }, + Key: "etcd-a/snap.db", + }, + }, + Target: ecv1alpha1.RestoreTarget{Name: targetName, Size: 1, Version: version}, + }, + } +} + +func reconcileRestore(t *testing.T, r *EtcdRestoreReconciler) (ctrl.Result, error) { + t.Helper() + return r.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: testRestoreName, Namespace: testNS}, + }) +} + +func getRestore(t *testing.T, r *EtcdRestoreReconciler) ecv1alpha1.EtcdRestore { + t.Helper() + var got ecv1alpha1.EtcdRestore + require.NoError(t, r.Get(context.Background(), + types.NamespacedName{Name: testRestoreName, Namespace: testNS}, &got)) + return got +} + +func getCluster(t *testing.T, r *EtcdRestoreReconciler, name string) ecv1alpha1.EtcdCluster { + t.Helper() + var c ecv1alpha1.EtcdCluster + require.NoError(t, r.Get(context.Background(), + types.NamespacedName{Name: name, Namespace: testNS}, &c)) + return c +} + +// --- tests ------------------------------------------------------------------ + +// TestRestore_StampsTargetAndRequeues proves the controller's core init-container +// model: it creates the target EtcdCluster, stamps it with the restore-source +// annotation (so the cluster controller injects the restore init-container), and +// requeues in Restoring (NOT Completed) until the genesis member boots. +func TestRestore_StampsTargetAndRequeues(t *testing.T) { + store := newFakeStore("etcd/backups") + _, err := store.Upload(context.Background(), "etcd-a/snap.db", readerOf("SNAPSHOT"), -1) + require.NoError(t, err) + + restore := locationRestore("restored-a", "v3.6.1") + r := newRestoreReconciler(t, store, restore) + + res, err := reconcileRestore(t, r) + require.NoError(t, err) + assert.Positive(t, res.RequeueAfter, "should requeue while the member boots") + + got := getRestore(t, r) + assert.Equal(t, ecv1alpha1.RestorePhaseRestoring, got.Status.Phase) + assert.NotEqual(t, ecv1alpha1.RestorePhaseCompleted, got.Status.Phase) + + // Target cluster created, owned by the restore, and stamped. + cluster := getCluster(t, r, "restored-a") + assert.Equal(t, "v3.6.1", cluster.Spec.Version) + assert.Equal(t, 1, cluster.Spec.Size) + require.Len(t, cluster.OwnerReferences, 1) + assert.Equal(t, "EtcdRestore", cluster.OwnerReferences[0].Kind) + + // The annotation must decode to a valid restore source carrying our snapshot + // addressing, the operator image, and a generation token. + raw := cluster.Annotations[restoreSourceAnnotation] + require.NotEmpty(t, raw, "target cluster must carry the restore-source annotation") + src, decErr := decodeRestoreSource(raw) + require.NoError(t, decErr) + assert.Equal(t, "s3", src.Provider) + assert.Equal(t, "b", src.Bucket) + assert.Equal(t, "etcd-a/snap.db", src.Key) + assert.Equal(t, "etcd/backups", src.Prefix) + assert.Equal(t, testOperatorImg, src.OperatorImage) + assert.NotEmpty(t, src.Generation) +} + +// TestRestore_CompletesWhenMemberReady proves the controller marks Completed +// only once the restore-target StatefulSet reports a ready member — Completed +// truthfully means "a member booted from the restored data". +func TestRestore_CompletesWhenMemberReady(t *testing.T) { + store := newFakeStore("etcd/backups") + _, err := store.Upload(context.Background(), "etcd-a/snap.db", readerOf("SNAPSHOT"), -1) + require.NoError(t, err) + + restore := locationRestore("restored-ready", "v3.6.1") + r := newRestoreReconciler(t, store, restore) + + // First reconcile: stamps + requeues. + _, err = reconcileRestore(t, r) + require.NoError(t, err) + require.Equal(t, ecv1alpha1.RestorePhaseRestoring, getRestore(t, r).Status.Phase) + + // Simulate the cluster controller bringing a member up: create a ready STS. + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "restored-ready", Namespace: testNS}, + Status: appsv1.StatefulSetStatus{ReadyReplicas: 1}, + } + require.NoError(t, r.Create(context.Background(), sts)) + + // Second reconcile: observes the ready member and completes. + _, err = reconcileRestore(t, r) + require.NoError(t, err) + got := getRestore(t, r) + assert.Equal(t, ecv1alpha1.RestorePhaseCompleted, got.Status.Phase) + assert.Equal(t, "restored-ready", got.Status.RestoredCluster) + assert.NotNil(t, got.Status.CompletionTime) + assert.Contains(t, got.Status.SnapshotLocation, "etcd/backups/etcd-a/snap.db") + cond := meta.FindStatusCondition(got.Status.Conditions, ecv1alpha1.RestoreConditionSucceeded) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionTrue, cond.Status) +} + +// TestRestore_BackupRefStampsInheritedVersion proves the backupRef path resolves +// the snapshot key + inherits the source cluster's version into the stamped +// target. +func TestRestore_BackupRefStampsInheritedVersion(t *testing.T) { + cluster, _ := readyCluster() // source cluster still exists -> version hint + backup := s3Backup("etcd-a") + backup.Status.Phase = ecv1alpha1.BackupPhaseCompleted + backup.Status.SnapshotLocation = "test://etcd/backups/etcd-a/backup-1-...db" + + store := newFakeStore("etcd/backups") + objKey := snapshotObjectKey(backup) + _, err := store.Upload(context.Background(), objKey, readerOf("FROMBACKUP"), -1) + require.NoError(t, err) + + restore := &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: testRestoreName, Namespace: testNS}, + Spec: ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{ + BackupRef: &ecv1alpha1.BackupReference{Name: testBackupName}, + }, + Target: ecv1alpha1.RestoreTarget{Name: "restored-b"}, // no version -> inherit + }, + } + r := newRestoreReconciler(t, store, cluster, backup, restore) + + _, err = reconcileRestore(t, r) + require.NoError(t, err) + + created := getCluster(t, r, "restored-b") + assert.Equal(t, cluster.Spec.Version, created.Spec.Version) + raw := created.Annotations[restoreSourceAnnotation] + require.NotEmpty(t, raw) + src, decErr := decodeRestoreSource(raw) + require.NoError(t, decErr) + assert.Equal(t, objKey, src.Key) +} + +func TestRestore_BackupRefNotCompletedFails(t *testing.T) { + backup := s3Backup("etcd-a") // phase is empty (Pending) + store := newFakeStore("etcd/backups") + restore := &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: testRestoreName, Namespace: testNS}, + Spec: ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{BackupRef: &ecv1alpha1.BackupReference{Name: testBackupName}}, + Target: ecv1alpha1.RestoreTarget{Name: "restored-c", Version: "v3.6.1"}, + }, + } + r := newRestoreReconciler(t, store, backup, restore) + + _, err := reconcileRestore(t, r) + require.Error(t, err) + assert.Equal(t, ecv1alpha1.RestorePhaseFailed, getRestore(t, r).Status.Phase) +} + +func TestRestore_MissingBackupRefFails(t *testing.T) { + store := newFakeStore("etcd/backups") + restore := &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: testRestoreName, Namespace: testNS}, + Spec: ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{BackupRef: &ecv1alpha1.BackupReference{Name: "absent"}}, + Target: ecv1alpha1.RestoreTarget{Name: "restored-d", Version: "v3.6.1"}, + }, + } + r := newRestoreReconciler(t, store, restore) + _, err := reconcileRestore(t, r) + require.Error(t, err) + assert.Equal(t, ecv1alpha1.RestorePhaseFailed, getRestore(t, r).Status.Phase) +} + +func TestRestore_BothSourcesFails(t *testing.T) { + restore := locationRestore("restored-e", "v3.6.1") + restore.Spec.Source.BackupRef = &ecv1alpha1.BackupReference{Name: "x"} + r := newRestoreReconciler(t, newFakeStore(""), restore) + _, err := reconcileRestore(t, r) + require.Error(t, err) + assert.Equal(t, ecv1alpha1.RestorePhaseFailed, getRestore(t, r).Status.Phase) +} + +func TestRestore_NoSourceFails(t *testing.T) { + restore := &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: testRestoreName, Namespace: testNS}, + Spec: ecv1alpha1.EtcdRestoreSpec{ + Target: ecv1alpha1.RestoreTarget{Name: "restored-f", Version: "v3.6.1"}, + }, + } + r := newRestoreReconciler(t, newFakeStore(""), restore) + _, err := reconcileRestore(t, r) + require.Error(t, err) + assert.Equal(t, ecv1alpha1.RestorePhaseFailed, getRestore(t, r).Status.Phase) +} + +func TestRestore_VersionRequiredForExplicitLocation(t *testing.T) { + store := newFakeStore("etcd/backups") + _, err := store.Upload(context.Background(), "etcd-a/snap.db", readerOf("S"), -1) + require.NoError(t, err) + restore := locationRestore("restored-g", "") // no version, no source cluster -> error + r := newRestoreReconciler(t, store, restore) + _, err = reconcileRestore(t, r) + require.Error(t, err) + assert.Equal(t, ecv1alpha1.RestorePhaseFailed, getRestore(t, r).Status.Phase) +} + +// TestRestore_NonEmptyUnstampedTargetRejected proves a restore into a cluster +// that already has a ready member AND is NOT already a restore target is +// rejected, and is never stamped (so the cluster controller never mutates that +// live cluster's pod template). +func TestRestore_NonEmptyUnstampedTargetRejected(t *testing.T) { + store := newFakeStore("etcd/backups") + _, err := store.Upload(context.Background(), "etcd-a/snap.db", readerOf("S"), -1) + require.NoError(t, err) + + existing := &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "restored-h", Namespace: testNS}, + Spec: ecv1alpha1.EtcdClusterSpec{Size: 3, Version: "v3.6.1"}, + } + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "restored-h", Namespace: testNS}, + Status: appsv1.StatefulSetStatus{ReadyReplicas: 3}, + } + restore := locationRestore("restored-h", "v3.6.1") + r := newRestoreReconciler(t, store, existing, sts, restore) + + _, err = reconcileRestore(t, r) + require.Error(t, err) + got := getRestore(t, r) + assert.Equal(t, ecv1alpha1.RestorePhaseFailed, got.Status.Phase) + // The live cluster must NOT have been stamped. + cluster := getCluster(t, r, "restored-h") + _, stamped := cluster.Annotations[restoreSourceAnnotation] + assert.False(t, stamped, "must not stamp (and thus mutate) a non-empty live cluster") +} + +// TestRestore_EmptyExistingTargetStamped proves an existing empty shell cluster +// is accepted and gets stamped. +func TestRestore_EmptyExistingTargetStamped(t *testing.T) { + store := newFakeStore("etcd/backups") + _, err := store.Upload(context.Background(), "etcd-a/snap.db", readerOf("S"), -1) + require.NoError(t, err) + + existing := &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "restored-i", Namespace: testNS}, + Spec: ecv1alpha1.EtcdClusterSpec{Size: 1, Version: "v3.6.1"}, + } + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "restored-i", Namespace: testNS}, + Status: appsv1.StatefulSetStatus{ReadyReplicas: 0}, + } + restore := locationRestore("restored-i", "v3.6.1") + r := newRestoreReconciler(t, store, existing, sts, restore) + + _, err = reconcileRestore(t, r) + require.NoError(t, err) + assert.Equal(t, ecv1alpha1.RestorePhaseRestoring, getRestore(t, r).Status.Phase) + cluster := getCluster(t, r, "restored-i") + _, stamped := cluster.Annotations[restoreSourceAnnotation] + assert.True(t, stamped) +} + +func TestRestore_SnapshotNotFoundFails(t *testing.T) { + store := newFakeStore("etcd/backups") // empty: object absent + restore := locationRestore("restored-j", "v3.6.1") + r := newRestoreReconciler(t, store, restore) + + _, err := reconcileRestore(t, r) + require.Error(t, err) + got := getRestore(t, r) + assert.Equal(t, ecv1alpha1.RestorePhaseFailed, got.Status.Phase) + // The target cluster must not have been created/stamped — verification fails + // before the cluster is touched. + var cluster ecv1alpha1.EtcdCluster + getErr := r.Get(context.Background(), types.NamespacedName{Name: "restored-j", Namespace: testNS}, &cluster) + assert.Error(t, getErr, "no cluster should be created when the snapshot is missing") +} + +func TestRestore_DownloadErrorFails(t *testing.T) { + store := newFakeStore("etcd/backups") + store.downloadErr = fmt.Errorf("network down") + restore := locationRestore("restored-k", "v3.6.1") + r := newRestoreReconciler(t, store, restore) + + _, err := reconcileRestore(t, r) + require.Error(t, err) + assert.Equal(t, ecv1alpha1.RestorePhaseFailed, getRestore(t, r).Status.Phase) +} + +// TestRestore_NoOperatorImageFails proves the controller refuses to stamp a +// restore target when it does not know its own image (it could not build a +// working restore init-container). +func TestRestore_NoOperatorImageFails(t *testing.T) { + store := newFakeStore("etcd/backups") + _, err := store.Upload(context.Background(), "etcd-a/snap.db", readerOf("S"), -1) + require.NoError(t, err) + restore := locationRestore("restored-noimg", "v3.6.1") + r := newRestoreReconciler(t, store, restore) + r.OperatorImage = "" // simulate a misconfigured operator + _, err = reconcileRestore(t, r) + require.Error(t, err) + assert.Equal(t, ecv1alpha1.RestorePhaseFailed, getRestore(t, r).Status.Phase) +} + +func TestRestore_TerminalIsNoOp(t *testing.T) { + store := newFakeStore("etcd/backups") + restore := locationRestore("restored-m", "v3.6.1") + restore.Status.Phase = ecv1alpha1.RestorePhaseCompleted + r := newRestoreReconciler(t, store, restore) + + _, err := reconcileRestore(t, r) + require.NoError(t, err) + // No target cluster should have been created by a terminal restore. + var cluster ecv1alpha1.EtcdCluster + getErr := r.Get(context.Background(), types.NamespacedName{Name: "restored-m", Namespace: testNS}, &cluster) + assert.Error(t, getErr) +} + +func TestValidateRestoreSpec(t *testing.T) { + loc := &ecv1alpha1.SnapshotLocation{ + Destination: ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + S3: &ecv1alpha1.S3DestinationSpec{Bucket: "b"}, + }, + Key: "k.db", + } + cases := []struct { + name string + spec ecv1alpha1.EtcdRestoreSpec + wantErr bool + }{ + {"location ok", ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{Location: loc}, + Target: ecv1alpha1.RestoreTarget{Name: "c"}, + }, false}, + {"backupRef ok", ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{BackupRef: &ecv1alpha1.BackupReference{Name: "b"}}, + Target: ecv1alpha1.RestoreTarget{Name: "c"}, + }, false}, + {"both sources", ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{Location: loc, BackupRef: &ecv1alpha1.BackupReference{Name: "b"}}, + Target: ecv1alpha1.RestoreTarget{Name: "c"}, + }, true}, + {"no source", ecv1alpha1.EtcdRestoreSpec{ + Target: ecv1alpha1.RestoreTarget{Name: "c"}, + }, true}, + {"no target name", ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{Location: loc}, + }, true}, + {"location missing key", ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{Location: &ecv1alpha1.SnapshotLocation{Destination: loc.Destination}}, + Target: ecv1alpha1.RestoreTarget{Name: "c"}, + }, true}, + {"location bad destination", ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{Location: &ecv1alpha1.SnapshotLocation{ + Destination: ecv1alpha1.BackupDestination{Provider: ecv1alpha1.BackupProviderS3}, // no S3 block + Key: "k", + }}, + Target: ecv1alpha1.RestoreTarget{Name: "c"}, + }, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err := validateRestoreSpec(&c.spec) + if c.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +// TestRestoreSourceAnnotationRoundTrip proves encode/decode of the annotation is +// lossless and that decode rejects incomplete payloads (which would otherwise +// build a data-less restore pod). +func TestRestoreSourceAnnotationRoundTrip(t *testing.T) { + src := restoreSource{ + Provider: "s3", Bucket: "b", Prefix: "p", Key: "k.db", + Region: "us-east-1", Endpoint: "http://minio:9000", ForcePathStyle: true, + CredsSecretName: "creds", OperatorImage: testOperatorImg, Generation: "k.db", + } + raw, err := encodeRestoreSource(src) + require.NoError(t, err) + got, err := decodeRestoreSource(raw) + require.NoError(t, err) + assert.Equal(t, src, got) + + _, err = decodeRestoreSource(`{"provider":"s3","bucket":"b"}`) // missing key + image + require.Error(t, err) +} diff --git a/internal/controller/etcdrestore_helpers.go b/internal/controller/etcdrestore_helpers.go new file mode 100644 index 00000000..9c06f859 --- /dev/null +++ b/internal/controller/etcdrestore_helpers.go @@ -0,0 +1,68 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "fmt" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// validateRestoreSpec checks the cross-field invariants the CRD's OpenAPI +// schema cannot express: exactly one snapshot source, a present and consistent +// destination on the explicit-location path, and a target name. Validation runs +// before any I/O so a misconfigured EtcdRestore fails cleanly and terminally. +func validateRestoreSpec(spec *ecv1alpha1.EtcdRestoreSpec) error { + src := spec.Source + hasRef := src.BackupRef != nil + hasLoc := src.Location != nil + switch { + case hasRef && hasLoc: + return fmt.Errorf("spec.source must set exactly one of backupRef or location, not both") + case !hasRef && !hasLoc: + return fmt.Errorf("spec.source must set one of backupRef or location") + } + + if hasRef && src.BackupRef.Name == "" { + return fmt.Errorf("spec.source.backupRef.name must not be empty") + } + if hasLoc { + if src.Location.Key == "" { + return fmt.Errorf("spec.source.location.key must not be empty") + } + if err := validateDestination(src.Location.Destination); err != nil { + return fmt.Errorf("spec.source.location.destination invalid: %w", err) + } + } + + if spec.Target.Name == "" { + return fmt.Errorf("spec.target.name must not be empty") + } + return nil +} + +// snapshotKeyFromBackup derives the object key (relative to the destination +// prefix) of a completed EtcdBackup's snapshot. It re-derives the key the +// backup controller wrote rather than parsing status.snapshotLocation, so the +// key construction stays in one place (snapshotObjectKey) and round-trips +// exactly. A backup that has not completed has no stable key and is rejected. +func snapshotKeyFromBackup(backup *ecv1alpha1.EtcdBackup) (string, error) { + if backup.Status.SnapshotLocation == "" { + return "", fmt.Errorf("EtcdBackup %q has no recorded snapshotLocation", backup.Name) + } + return snapshotObjectKey(backup), nil +} diff --git a/internal/controller/objectstore_creds.go b/internal/controller/objectstore_creds.go new file mode 100644 index 00000000..e59dfc38 --- /dev/null +++ b/internal/controller/objectstore_creds.go @@ -0,0 +1,113 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/go-logr/logr" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" + "go.etcd.io/etcd-operator/pkg/objectstore" +) + +// resolveStoreCredentials reads object-store credentials for a destination from +// its referenced Secret, if any, and returns them for the objectstore factory. +// +// SECURITY: object-store credentials originate *only* from the destination's +// secretRef (or the operator's ambient identity when secretRef is unset). This +// function is the single seam that reads that secret, and it is written so the +// credential *values* are never logged or returned in an error: +// +// - It emits a structured audit log line recording the namespace, provider, +// and the secret *name* — never any secret key value. +// - On a missing/unreadable secret it wraps only the secret name and the API +// error (which never contains the secret's data), never the bytes it failed +// to read. +// - The returned objectstore.Credentials is passed straight to the provider +// factory and is never rendered into status, conditions, or events. +// +// Tests assert this guarantee directly (see the cred-never-logged test). +func resolveStoreCredentials( + ctx context.Context, + c client.Client, + logger logr.Logger, + namespace string, + dst ecv1alpha1.BackupDestination, +) (objectstore.Credentials, error) { + var creds objectstore.Credentials + + if dst.SecretRef == nil { + // Ambient-credential path (IRSA / Workload Identity). Record that no + // secret was used so an auditor can see the operator's own identity is + // in play, with nothing sensitive to redact. + logger.Info("objectstore credentials: using ambient operator identity (no secretRef)", + "namespace", namespace, "provider", string(dst.Provider)) + return creds, nil + } + + // Audit the *reference* before reading it. Only the secret name is logged; + // the values pulled below are never logged anywhere. + logger.Info("objectstore credentials: loading from secretRef", + "namespace", namespace, "provider", string(dst.Provider), "secretName", dst.SecretRef.Name) + + var secret corev1.Secret + key := types.NamespacedName{Namespace: namespace, Name: dst.SecretRef.Name} + if err := c.Get(ctx, key, &secret); err != nil { + return creds, fmt.Errorf("get credentials secret %q: %w", dst.SecretRef.Name, err) + } + + switch dst.Provider { + case ecv1alpha1.BackupProviderS3: + creds.AccessKeyID = string(secret.Data["accessKeyID"]) + creds.SecretAccessKey = string(secret.Data["secretAccessKey"]) + creds.SessionToken = string(secret.Data["sessionToken"]) + if creds.AccessKeyID == "" || creds.SecretAccessKey == "" { + return creds, fmt.Errorf( + "secret %q is missing required s3 keys accessKeyID/secretAccessKey", dst.SecretRef.Name) + } + case ecv1alpha1.BackupProviderGCS: + creds.ServiceAccountJSON = secret.Data["serviceAccountJSON"] + if len(creds.ServiceAccountJSON) == 0 { + return creds, fmt.Errorf( + "secret %q is missing required gcs key serviceAccountJSON", dst.SecretRef.Name) + } + default: + return creds, fmt.Errorf("unsupported provider %q for secret-based credentials", dst.Provider) + } + return creds, nil +} + +// validateDestination performs structural validation of a BackupDestination +// shared by the backup and restore paths: the provider-specific block must be +// present and consistent, and a referenced secret name must be non-empty. It is +// the controller-side complement to the API-level CRD validation, catching the +// cross-field invariants OpenAPI cannot express. +func validateDestination(dst ecv1alpha1.BackupDestination) error { + if _, err := toObjectStoreDestination(dst); err != nil { + return err + } + if dst.SecretRef != nil && dst.SecretRef.Name == "" { + return fmt.Errorf("destination.secretRef.name must not be empty when secretRef is set") + } + return nil +} diff --git a/internal/controller/objectstore_creds_test.go b/internal/controller/objectstore_creds_test.go new file mode 100644 index 00000000..2a7d0fed --- /dev/null +++ b/internal/controller/objectstore_creds_test.go @@ -0,0 +1,202 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "strings" + "sync" + "testing" + + "github.com/go-logr/logr" + "github.com/go-logr/logr/funcr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// capturingLogger records every formatted log line so a test can assert that +// secret material never appears in any of them. +type capturingLogger struct { + mu sync.Mutex + lines []string +} + +func (c *capturingLogger) sink() logr.Logger { + return funcr.New(func(prefix, args string) { + c.mu.Lock() + defer c.mu.Unlock() + c.lines = append(c.lines, prefix+" "+args) + }, funcr.Options{}) +} + +func (c *capturingLogger) all() string { + c.mu.Lock() + defer c.mu.Unlock() + return strings.Join(c.lines, "\n") +} + +// These are the secret values that must NEVER be logged. +const ( + secretAK = "AKIAEXAMPLESENSITIVE" + secretSK = "sUperSecret/Key+Value==" + secretTok = "FwoGZXIvSESSIONTOKEN" + secretJSON = `{"type":"service_account","private_key":"-----BEGIN PRIVATE KEY-----TOPSECRET"}` +) + +func credSecret(name string, data map[string][]byte) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNS}, + Data: data, + } +} + +// TestResolveStoreCredentials_NeverLogsSecretValues is the core security +// guarantee: regardless of provider, the resolver may log the secret *name* but +// never any credential value. +func TestResolveStoreCredentials_NeverLogsSecretValues(t *testing.T) { + s := backupScheme(t) + + t.Run("s3", func(t *testing.T) { + secret := credSecret("s3creds", map[string][]byte{ + "accessKeyID": []byte(secretAK), + "secretAccessKey": []byte(secretSK), + "sessionToken": []byte(secretTok), + }) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(secret).Build() + cap := &capturingLogger{} + + creds, err := resolveStoreCredentials(context.Background(), cl, cap.sink(), testNS, + ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + SecretRef: &corev1.LocalObjectReference{Name: "s3creds"}, + }) + require.NoError(t, err) + // Sanity: the values were actually read (so the test is meaningful). + require.Equal(t, secretAK, creds.AccessKeyID) + require.Equal(t, secretSK, creds.SecretAccessKey) + + logged := cap.all() + assert.Contains(t, logged, "s3creds", "the secret name is allowed (and expected) in audit logs") + assert.NotContains(t, logged, secretAK, "access key ID must never be logged") + assert.NotContains(t, logged, secretSK, "secret access key must never be logged") + assert.NotContains(t, logged, secretTok, "session token must never be logged") + }) + + t.Run("gcs", func(t *testing.T) { + secret := credSecret("gcskey", map[string][]byte{"serviceAccountJSON": []byte(secretJSON)}) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(secret).Build() + cap := &capturingLogger{} + + creds, err := resolveStoreCredentials(context.Background(), cl, cap.sink(), testNS, + ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderGCS, + SecretRef: &corev1.LocalObjectReference{Name: "gcskey"}, + }) + require.NoError(t, err) + require.Equal(t, secretJSON, string(creds.ServiceAccountJSON)) + + logged := cap.all() + assert.Contains(t, logged, "gcskey") + assert.NotContains(t, logged, "TOPSECRET", "service-account key material must never be logged") + assert.NotContains(t, logged, "private_key") + }) +} + +// TestResolveStoreCredentials_Validation covers the presence/format checks the +// resolver performs so misconfiguration surfaces before any network I/O. +func TestResolveStoreCredentials_Validation(t *testing.T) { + s := backupScheme(t) + + t.Run("ambient when no secretRef", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(s).Build() + creds, err := resolveStoreCredentials(context.Background(), cl, logr.Discard(), testNS, + ecv1alpha1.BackupDestination{Provider: ecv1alpha1.BackupProviderS3}) + require.NoError(t, err) + assert.Empty(t, creds.AccessKeyID) + }) + + t.Run("missing secret errors without leaking", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(s).Build() + _, err := resolveStoreCredentials(context.Background(), cl, logr.Discard(), testNS, + ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + SecretRef: &corev1.LocalObjectReference{Name: "absent"}, + }) + require.Error(t, err) + }) + + t.Run("s3 secret missing required keys errors", func(t *testing.T) { + secret := credSecret("partial", map[string][]byte{"accessKeyID": []byte("only-ak")}) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(secret).Build() + _, err := resolveStoreCredentials(context.Background(), cl, logr.Discard(), testNS, + ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + SecretRef: &corev1.LocalObjectReference{Name: "partial"}, + }) + require.Error(t, err) + assert.NotContains(t, err.Error(), "only-ak", "error must not echo secret values") + }) + + t.Run("gcs secret missing key errors", func(t *testing.T) { + secret := credSecret("emptygcs", map[string][]byte{}) + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(secret).Build() + _, err := resolveStoreCredentials(context.Background(), cl, logr.Discard(), testNS, + ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderGCS, + SecretRef: &corev1.LocalObjectReference{Name: "emptygcs"}, + }) + require.Error(t, err) + }) +} + +func TestValidateDestination(t *testing.T) { + cases := []struct { + name string + dst ecv1alpha1.BackupDestination + wantErr bool + }{ + {"s3 ok", ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + S3: &ecv1alpha1.S3DestinationSpec{Bucket: "b"}, + }, false}, + {"s3 missing block", ecv1alpha1.BackupDestination{Provider: ecv1alpha1.BackupProviderS3}, true}, + {"gcs ok", ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderGCS, + GCS: &ecv1alpha1.GCSDestinationSpec{Bucket: "b"}, + }, false}, + {"empty secretRef name", ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + S3: &ecv1alpha1.S3DestinationSpec{Bucket: "b"}, + SecretRef: &corev1.LocalObjectReference{Name: ""}, + }, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err := validateDestination(c.dst) + if c.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/internal/controller/restore_initcontainers.go b/internal/controller/restore_initcontainers.go new file mode 100644 index 00000000..560cfb73 --- /dev/null +++ b/internal/controller/restore_initcontainers.go @@ -0,0 +1,220 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// restoreSourceAnnotation is the annotation the EtcdRestore controller stamps on +// the target EtcdCluster to mark it as a restore-target. Its value is a +// JSON-encoded restoreSource. The cluster controller reads it at StatefulSet +// build time: when ABSENT, the pod template is byte-identical to a normal +// cluster (zero restore init-containers, no extra volumes) so the existing +// cluster e2e is unaffected; when PRESENT, the controller injects ONE restore +// init-container (the operator image, `manager restore-localize`) that +// bootstraps the genesis member from the snapshot before etcd starts. +const restoreSourceAnnotation = "operator.etcd.io/restore-source" + +const ( + // restoreInitContainerName is the single restore init-container injected on a + // restore-target cluster. It downloads the snapshot AND restores it + // in-process (the operator image links etcd's snapshot-restore library), so + // there is no second image hop and no shell. + restoreInitContainerName = "restore-localize" + + // restoreCredsVolumeName is the (optional) projected mount of the object-store + // credentials Secret into the restore init-container only. + restoreCredsVolumeName = "restore-creds" +) + +// restoreSource is the JSON payload carried by restoreSourceAnnotation. It fully +// describes the snapshot object to fetch and the operator image to fetch+restore +// it with. Credential *values* are NOT carried here (that would leak them into +// the EtcdCluster object); the credentials Secret is referenced by name and its +// keys are surfaced as env vars into the restore init-container only. +type restoreSource struct { + Provider string `json:"provider"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix,omitempty"` + Key string `json:"key"` + Region string `json:"region,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + ForcePathStyle bool `json:"forcePathStyle,omitempty"` + + // CredsSecretName names the Secret holding object-store credentials, or "" for + // ambient/unauthenticated access. Its keys are mapped to RESTORE_* env vars on + // the restore init-container only (never the long-running etcd container). + CredsSecretName string `json:"credsSecretName,omitempty"` + + // OperatorImage is the image the restore init-container runs (the operator + // image, invoked as `manager restore-localize`). The cluster controller knows + // its own image and stamps it here so the member pod need not guess it. + OperatorImage string `json:"operatorImage"` + + // Generation ties the restore to the idempotency marker the init-container + // writes on success: a pod restart with the same generation is a no-op + // (never re-wipes-and-restores), while a genuinely new restore into a reused + // (empty) cluster name carries a new generation. Derived from the snapshot key. + Generation string `json:"generation"` +} + +// encodeRestoreSource serializes a restoreSource for the annotation value. +func encodeRestoreSource(src restoreSource) (string, error) { + b, err := json.Marshal(src) + if err != nil { + return "", fmt.Errorf("encode restore source: %w", err) + } + return string(b), nil +} + +// decodeRestoreSource parses an annotation value into a restoreSource, rejecting +// anything missing the load-bearing fields so the controller never silently +// builds a half-configured restore pod that would boot an empty member. +func decodeRestoreSource(v string) (restoreSource, error) { + var src restoreSource + if err := json.Unmarshal([]byte(v), &src); err != nil { + return src, fmt.Errorf("decode restore source annotation: %w", err) + } + if src.Provider == "" || src.Bucket == "" || src.Key == "" || src.OperatorImage == "" { + return src, fmt.Errorf( + "restore source annotation incomplete (provider=%q bucket=%q key=%q operatorImage=%q)", + src.Provider, src.Bucket, src.Key, src.OperatorImage) + } + return src, nil +} + +// restoreSourceFromCluster returns the decoded restore source and true if the +// EtcdCluster carries the restore annotation, false otherwise. A present-but- +// malformed annotation surfaces as an error so the StatefulSet build fails +// loudly rather than emitting a data-less member that boots empty. +func restoreSourceFromCluster(ec *ecv1alpha1.EtcdCluster) (restoreSource, bool, error) { + v, ok := ec.Annotations[restoreSourceAnnotation] + if !ok || v == "" { + return restoreSource{}, false, nil + } + src, err := decodeRestoreSource(v) + if err != nil { + return restoreSource{}, true, err + } + return src, true, nil +} + +// credEnv maps the object-store creds Secret's keys onto the RESTORE_* env vars +// the restore-localize subcommand reads. Each entry is Optional so a key absent +// for the OTHER provider (e.g. serviceAccountJSON on an S3 restore) does not +// block the pod. When CredsSecretName is "", no credential env is added and the +// subcommand falls back to ambient/unauthenticated access. +func (src restoreSource) credEnv() []corev1.EnvVar { + if src.CredsSecretName == "" { + return nil + } + ref := func(key string) *corev1.EnvVarSource { + opt := true + return &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: src.CredsSecretName}, + Key: key, + Optional: &opt, + }, + } + } + return []corev1.EnvVar{ + {Name: "RESTORE_S3_ACCESS_KEY_ID", ValueFrom: ref("accessKeyID")}, + {Name: "RESTORE_S3_SECRET_ACCESS_KEY", ValueFrom: ref("secretAccessKey")}, + {Name: "RESTORE_S3_SESSION_TOKEN", ValueFrom: ref("sessionToken")}, + {Name: "RESTORE_GCS_SERVICE_ACCOUNT_JSON", ValueFrom: ref("serviceAccountJSON")}, + } +} + +// applyRestoreInitContainers mutates podSpec to add the volume(s) and the single +// restore init-container that bootstraps the genesis member from a snapshot. It +// is invoked from createOrPatchStatefulSet ONLY when the cluster carries a valid +// restore-source annotation, so a normal cluster's pod template is never touched. +// +// Data-dir sharing: the restore init-container must write into the SAME +// filesystem the etcd container reads as ETCD_DATA_DIR (/var/lib/etcd). When the +// cluster has a StorageSpec, the etcd container already mounts the PVC there +// (added by the caller); this function mirrors that mount onto the init- +// container. When there is NO StorageSpec, the etcd container has no data volume +// at all (it writes to its ephemeral container FS, which an init-container +// cannot share), so this function adds an emptyDir data volume and mounts it on +// BOTH the etcd container and the restore init-container. +// +// Member/ordinal scoping and idempotency are enforced inside the init-container +// (restore-localize): only ordinal 0 restores, and a per-generation marker makes +// a pod restart a no-op so a reschedule never re-wipes-and-restores. +func applyRestoreInitContainers( + podSpec *corev1.PodSpec, ec *ecv1alpha1.EtcdCluster, src restoreSource, +) { + dataMount := corev1.VolumeMount{ + Name: volumeName, + MountPath: etcdDataDir, + SubPathExpr: "$(POD_NAME)", + } + + // No StorageSpec: add a shared emptyDir data volume and mount it on the etcd + // container so it boots from the restored dir. + if ec.Spec.StorageSpec == nil { + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }) + podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, dataMount) + } + + // The restore init-container needs POD_NAME (ordinal guard + SubPathExpr) and + // POD_NAMESPACE, the snapshot addressing env, the restore identity env, and + // (optionally) the creds env. + env := []corev1.EnvVar{ + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, + {Name: "RESTORE_PROVIDER", Value: src.Provider}, + {Name: "RESTORE_BUCKET", Value: src.Bucket}, + {Name: "RESTORE_PREFIX", Value: src.Prefix}, + {Name: "RESTORE_KEY", Value: src.Key}, + {Name: "RESTORE_REGION", Value: src.Region}, + {Name: "RESTORE_ENDPOINT", Value: src.Endpoint}, + {Name: "RESTORE_FORCE_PATH_STYLE", Value: fmt.Sprintf("%t", src.ForcePathStyle)}, + {Name: "RESTORE_DATA_DIR", Value: etcdDataDir}, + {Name: "RESTORE_GENERATION", Value: src.Generation}, + // Member identity MUST match defaultArgs / the state ConfigMap so the + // restored genesis member advertises the same name + peer URL the etcd + // container boots with. + {Name: "RESTORE_MEMBER_NAME", Value: "$(POD_NAME)"}, + {Name: "RESTORE_PEER_URL", Value: fmt.Sprintf( + "http://$(POD_NAME).%s.$(POD_NAMESPACE).svc.cluster.local:2380", ec.Name)}, + } + env = append(env, src.credEnv()...) + + restore := corev1.Container{ + Name: restoreInitContainerName, + Image: src.OperatorImage, + Command: []string{"/manager", "restore-localize"}, + Env: env, + VolumeMounts: []corev1.VolumeMount{dataMount}, + } + + podSpec.InitContainers = append(podSpec.InitContainers, restore) +} diff --git a/internal/controller/restore_initcontainers_test.go b/internal/controller/restore_initcontainers_test.go new file mode 100644 index 00000000..506dfcb6 --- /dev/null +++ b/internal/controller/restore_initcontainers_test.go @@ -0,0 +1,163 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +func restoreTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, ecv1alpha1.AddToScheme(scheme)) + require.NoError(t, appsv1.AddToScheme(scheme)) + return scheme +} + +func newEtcdClusterFixture(name string, annotations map[string]string) *ecv1alpha1.EtcdCluster { + return &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Annotations: annotations}, + Spec: ecv1alpha1.EtcdClusterSpec{ + Size: 1, + Version: "v3.6.1", + ImageRegistry: "gcr.io/etcd-development/etcd", + }, + } +} + +func buildStatefulSetForCluster(t *testing.T, ec *ecv1alpha1.EtcdCluster) *appsv1.StatefulSet { + t.Helper() + ctx := context.Background() + scheme := restoreTestScheme(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + require.NoError(t, createOrPatchStatefulSet(ctx, log.FromContext(ctx), ec, fakeClient, 1, scheme)) + var sts appsv1.StatefulSet + require.NoError(t, fakeClient.Get(ctx, client.ObjectKey{Name: ec.Name, Namespace: ec.Namespace}, &sts)) + return &sts +} + +// TestStatefulSet_NormalCluster_NoRestoreInitContainers is the regression guard +// that protects the existing cluster e2e: a cluster WITHOUT the restore-source +// annotation must yield ZERO init-containers (a byte-identical pod template), so +// injecting the restore path never triggers a fleet-wide rolling restart. +func TestStatefulSet_NormalCluster_NoRestoreInitContainers(t *testing.T) { + ec := newEtcdClusterFixture("normal-cluster", nil) + sts := buildStatefulSetForCluster(t, ec) + assert.Empty(t, sts.Spec.Template.Spec.InitContainers, + "a non-restore cluster must have zero init-containers") + for _, v := range sts.Spec.Template.Spec.Volumes { + assert.NotEqual(t, volumeName, v.Name, "no data volume without StorageSpec on a normal cluster") + } +} + +// TestStatefulSet_RestoreCluster_InjectsOneInitContainer proves a cluster +// carrying a valid restore-source annotation gets exactly ONE restore +// init-container (operator image, restore-localize), a shared data emptyDir, and +// the data-dir mount on the etcd container. +func TestStatefulSet_RestoreCluster_InjectsOneInitContainer(t *testing.T) { + src := restoreSource{ + Provider: "s3", Bucket: "b", Key: "etcd-x/snap.db", Prefix: "e2e", + Region: "us-east-1", Endpoint: "http://minio:9000", ForcePathStyle: true, + CredsSecretName: "creds", OperatorImage: "op:img", Generation: "etcd-x/snap.db", + } + raw, err := encodeRestoreSource(src) + require.NoError(t, err) + ec := newEtcdClusterFixture("restore-cluster", map[string]string{restoreSourceAnnotation: raw}) + sts := buildStatefulSetForCluster(t, ec) + + require.Len(t, sts.Spec.Template.Spec.InitContainers, 1) + init := sts.Spec.Template.Spec.InitContainers[0] + assert.Equal(t, restoreInitContainerName, init.Name) + assert.Equal(t, "op:img", init.Image) + assert.Equal(t, []string{"/manager", "restore-localize"}, init.Command) + + var foundDataMount bool + for _, m := range init.VolumeMounts { + if m.Name == volumeName && m.MountPath == etcdDataDir { + foundDataMount = true + } + } + assert.True(t, foundDataMount, "restore init-container must mount the etcd data dir") + + var dataVol *corev1.Volume + for i := range sts.Spec.Template.Spec.Volumes { + if sts.Spec.Template.Spec.Volumes[i].Name == volumeName { + dataVol = &sts.Spec.Template.Spec.Volumes[i] + } + } + require.NotNil(t, dataVol, "a shared data volume must exist on a restore cluster") + assert.NotNil(t, dataVol.EmptyDir) + + var etcdHasDataMount bool + for _, m := range sts.Spec.Template.Spec.Containers[0].VolumeMounts { + if m.Name == volumeName && m.MountPath == etcdDataDir { + etcdHasDataMount = true + } + } + assert.True(t, etcdHasDataMount, "etcd container must mount the shared restored data dir") + + envByName := map[string]string{} + for _, e := range init.Env { + envByName[e.Name] = e.Value + } + assert.Equal(t, "s3", envByName["RESTORE_PROVIDER"]) + assert.Equal(t, "b", envByName["RESTORE_BUCKET"]) + assert.Equal(t, "etcd-x/snap.db", envByName["RESTORE_KEY"]) + assert.Equal(t, etcdDataDir, envByName["RESTORE_DATA_DIR"]) + assert.Equal(t, "$(POD_NAME)", envByName["RESTORE_MEMBER_NAME"]) + assert.Contains(t, envByName["RESTORE_PEER_URL"], "restore-cluster") + + // Credentials must be wired as optional secretKeyRefs scoped to this + // init-container, never inlined values. + var sawCredRef bool + for _, e := range init.Env { + if e.Name == "RESTORE_S3_ACCESS_KEY_ID" { + require.NotNil(t, e.ValueFrom) + require.NotNil(t, e.ValueFrom.SecretKeyRef) + assert.Equal(t, "creds", e.ValueFrom.SecretKeyRef.Name) + sawCredRef = true + } + } + assert.True(t, sawCredRef, "creds must reach the restore init-container via secretKeyRef") +} + +// TestStatefulSet_RestoreCluster_MalformedAnnotationErrors proves a present-but- +// malformed annotation fails the StatefulSet build loudly rather than emitting a +// data-less member. +func TestStatefulSet_RestoreCluster_MalformedAnnotationErrors(t *testing.T) { + ec := newEtcdClusterFixture("bad-restore", map[string]string{restoreSourceAnnotation: "{not json"}) + ctx := context.Background() + scheme := restoreTestScheme(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + err := createOrPatchStatefulSet(ctx, log.FromContext(ctx), ec, fakeClient, 1, scheme) + require.Error(t, err) + assert.Contains(t, err.Error(), "restore-source annotation") +} diff --git a/internal/controller/snapshotter.go b/internal/controller/snapshotter.go new file mode 100644 index 00000000..9cc69f3e --- /dev/null +++ b/internal/controller/snapshotter.go @@ -0,0 +1,140 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "io" + "time" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + + "go.etcd.io/etcd/client/pkg/v3/logutil" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// Snapshotter abstracts "produce an etcd snapshot stream from a running +// member". It exists as an interface so the backup controller's +// snapshot+upload orchestration can be unit-tested with a fake that returns +// canned bytes, with no live etcd required. +type Snapshotter interface { + // Snapshot writes a consistent point-in-time etcd snapshot of the cluster + // reachable via pod to w. It returns the number of bytes written. + // + // Implementations run the equivalent of `etcdctl snapshot save` against a + // healthy member and stream the resulting file to w. + Snapshot(ctx context.Context, pod types.NamespacedName, w io.Writer) (int64, error) +} + +// clientSnapshotter implements Snapshotter using the etcd v3 Maintenance +// Snapshot API directly from the operator process, with no in-pod exec. +// +// This is deliberately NOT exec-based: the etcd images the operator deploys +// (e.g. gcr.io/etcd-development/etcd:v3.6.x) are distroless and ship no shell +// or coreutils, so `kubectl exec -- sh -c "etcdctl snapshot save ... | cat"` +// fails with `exec: "sh": executable file not found in $PATH`. The Maintenance +// API streams the snapshot over the client port the operator can already reach +// in-cluster, so it works regardless of what binaries the member image carries. +type clientSnapshotter struct { + // endpointForPod resolves the etcd client URL for the target member pod. + // Parameterized so tests can point it at a local listener and so TLS/port + // specifics can evolve without touching the streaming logic. + endpointForPod func(pod types.NamespacedName) string + // dialTimeout bounds establishing the client connection. + dialTimeout time.Duration +} + +// newClientSnapshotter constructs the default, shell-free Snapshotter backed by +// the etcd v3 Maintenance Snapshot API. +func newClientSnapshotter() *clientSnapshotter { + return &clientSnapshotter{ + endpointForPod: clientURLForMemberPod, + dialTimeout: 10 * time.Second, + } +} + +// clientURLForMemberPod returns the in-cluster client URL of a member pod. +// StatefulSet pods are named - and addressable through the +// cluster's headless Service (named after the cluster) at +// ...svc.cluster.local:2379, which is exactly the +// --advertise-client-urls the member is started with (see utils.go). +func clientURLForMemberPod(pod types.NamespacedName) string { + cluster := clusterNameFromPod(pod.Name) + return fmt.Sprintf("http://%s.%s.%s.svc.cluster.local:2379", + pod.Name, cluster, pod.Namespace) +} + +// clusterNameFromPod strips the trailing - from a StatefulSet pod name +// to recover the cluster (and headless Service) name. +func clusterNameFromPod(podName string) string { + for i := len(podName) - 1; i >= 0; i-- { + if podName[i] == '-' { + return podName[:i] + } + if podName[i] < '0' || podName[i] > '9' { + break + } + } + return podName +} + +func (s *clientSnapshotter) Snapshot(ctx context.Context, pod types.NamespacedName, w io.Writer) (int64, error) { + lg, err := logutil.CreateDefaultZapLogger(zap.WarnLevel) + if err != nil { + lg = zap.NewNop() + } + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: []string{s.endpointForPod(pod)}, + DialTimeout: s.dialTimeout, + Context: ctx, + Logger: lg, + }) + if err != nil { + return 0, fmt.Errorf("snapshotter: new etcd client for %s: %w", pod, err) + } + defer func() { _ = cli.Close() }() + + rc, err := cli.Snapshot(ctx) + if err != nil { + return 0, fmt.Errorf("snapshotter: open snapshot stream for %s: %w", pod, err) + } + defer func() { _ = rc.Close() }() + + cw := &countingWriter{w: w} + if _, err := io.Copy(cw, rc); err != nil { + return cw.n, fmt.Errorf("snapshotter: stream snapshot from %s: %w", pod, err) + } + if cw.n == 0 { + return 0, fmt.Errorf("snapshotter: empty snapshot produced for %s", pod) + } + return cw.n, nil +} + +// countingWriter counts bytes passing through to the wrapped writer. +type countingWriter struct { + w io.Writer + n int64 +} + +func (c *countingWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += int64(n) + return n, err +} diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 3092c379..00c19ae8 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -310,6 +310,19 @@ func createOrPatchStatefulSet(ctx context.Context, logger logr.Logger, ec *ecv1a } } + // Restore-target clusters (marked by the EtcdRestore controller with the + // restore-source annotation) get a single restore init-container that + // bootstraps the genesis member's data dir from a snapshot before etcd + // starts. A normal cluster carries no such annotation and its pod template is + // left byte-identical (zero init-containers / no extra volumes), so the + // existing cluster reconcile/e2e is unaffected. + if src, ok, srcErr := restoreSourceFromCluster(ec); srcErr != nil { + return fmt.Errorf("invalid restore-source annotation on EtcdCluster %s/%s: %w", + ec.Namespace, ec.Name, srcErr) + } else if ok { + applyRestoreInitContainers(&stsSpec.Template.Spec, ec, src) + } + logger.Info("Now creating/updating statefulset", "name", ec.Name, "namespace", ec.Namespace, "replicas", replicas) _, err := controllerutil.CreateOrPatch(ctx, c, sts, func() error { // Define or update the desired spec diff --git a/pkg/objectstore/gcs.go b/pkg/objectstore/gcs.go new file mode 100644 index 00000000..ab67fbd9 --- /dev/null +++ b/pkg/objectstore/gcs.go @@ -0,0 +1,202 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "context" + "errors" + "fmt" + "io" + "sort" + + gcs "cloud.google.com/go/storage" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +func init() { + Register(ProviderGCS, newGCSStore) +} + +// gcsBucketHandle is the subset of *storage.BucketHandle the store uses, +// narrowed so the orchestration is testable with a fake (see gcs_test.go). +type gcsBucketHandle interface { + NewWriter(ctx context.Context, key string) io.WriteCloser + NewReader(ctx context.Context, key string) (io.ReadCloser, error) + Delete(ctx context.Context, key string) error + List(ctx context.Context, prefix string) ([]ObjectInfo, error) +} + +// gcsStore implements Store against Google Cloud Storage. +type gcsStore struct { + bucket gcsBucketHandle + name string + prefix string + closer func() error +} + +// newGCSStore builds a gcsStore. With an explicit service-account JSON it uses +// those credentials; otherwise it relies on Application Default Credentials +// (Workload Identity in-cluster), the recommended production posture. +func newGCSStore(ctx context.Context, dst Destination, creds Credentials) (Store, error) { + var opts []option.ClientOption + switch { + case dst.Endpoint != "": + // Emulator mode (fake-gcs-server / gcloud storage testbench): point the + // client at the emulator's JSON-API root and skip the token source. + // WithoutAuthentication is required because the emulator is + // unauthenticated — without it the client tries to mint a Google token + // at startup and fails. This mirrors the S3 endpoint override that lets + // the same code path target MinIO, and keeps emulator credentials out + // of the picture entirely (any secretRef is ignored in this mode). + opts = append(opts, + option.WithEndpoint(dst.Endpoint), + option.WithoutAuthentication(), + ) + case len(creds.ServiceAccountJSON) > 0: + // WithCredentialsJSON is the documented way to pass an explicit GCP + // service-account key. The deprecation note concerns the general risk + // of handling raw key material; here the key originates from a + // user-provided Secret and explicit credentials are an intentional, + // supported mode (the recommended path remains Workload Identity, i.e. + // no secret at all). + //nolint:staticcheck // SA1019: explicit-credentials mode is intentional + opts = append(opts, option.WithCredentialsJSON(creds.ServiceAccountJSON)) + } + + client, err := gcs.NewClient(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("objectstore/gcs: new client: %w", err) + } + + return &gcsStore{ + bucket: &realGCSBucket{handle: client.Bucket(dst.Bucket)}, + name: dst.Bucket, + prefix: dst.Prefix, + closer: client.Close, + }, nil +} + +func (g *gcsStore) Scheme() string { return "gs" } + +func (g *gcsStore) Upload(ctx context.Context, key string, r io.Reader, _ int64) (UploadResult, error) { + fullKey := JoinKey(g.prefix, key) + + w := g.bucket.NewWriter(ctx, fullKey) + n, err := io.Copy(w, r) + if err != nil { + // Close to release resources; ignore its error in favor of the copy error. + _ = w.Close() + return UploadResult{}, fmt.Errorf("objectstore/gcs: upload %s: %w", fullKey, err) + } + if err := w.Close(); err != nil { + return UploadResult{}, fmt.Errorf("objectstore/gcs: finalize %s: %w", fullKey, err) + } + + return UploadResult{ + URI: fmt.Sprintf("gs://%s/%s", g.name, fullKey), + Size: n, + }, nil +} + +// Download opens the object for streaming reads. The GCS reader streams the +// body, so the snapshot is never staged on the operator's disk on the read +// path. A missing object is mapped to ErrNotFound for the restore controller. +func (g *gcsStore) Download(ctx context.Context, key string) (io.ReadCloser, error) { + fullKey := JoinKey(g.prefix, key) + rc, err := g.bucket.NewReader(ctx, fullKey) + if err != nil { + if errors.Is(err, gcs.ErrObjectNotExist) { + return nil, fmt.Errorf("objectstore/gcs: get %s: %w", fullKey, ErrNotFound) + } + return nil, fmt.Errorf("objectstore/gcs: get %s: %w", fullKey, err) + } + return rc, nil +} + +func (g *gcsStore) List(ctx context.Context, keyPrefix string) ([]ObjectInfo, error) { + // Append a trailing slash so the prefix matches a directory boundary rather + // than a raw string prefix; otherwise listing "/etcd-a" also matches + // "/etcd-a-2" and retention would prune another cluster's snapshots. + // JoinKey trims trailing slashes, so the boundary is added here. An empty + // prefix lists the whole bucket and must stay empty. + listPrefix := JoinKey(g.prefix, keyPrefix) + if listPrefix != "" { + listPrefix += "/" + } + infos, err := g.bucket.List(ctx, listPrefix) + if err != nil { + return nil, err + } + // Stable sort with a key tiebreaker so an equal LastModified can never + // non-deterministically order the just-uploaded snapshot last (and into the + // retention deletion set). Keys embed a sortable UTC timestamp. + sort.SliceStable(infos, func(i, j int) bool { + if infos[i].LastModified.Equal(infos[j].LastModified) { + return infos[i].Key > infos[j].Key + } + return infos[i].LastModified.After(infos[j].LastModified) + }) + return infos, nil +} + +func (g *gcsStore) Delete(ctx context.Context, key string) error { + return g.bucket.Delete(ctx, JoinKey(g.prefix, key)) +} + +// realGCSBucket adapts the concrete *storage.BucketHandle to gcsBucketHandle. +type realGCSBucket struct { + handle *gcs.BucketHandle +} + +func (b *realGCSBucket) NewWriter(ctx context.Context, key string) io.WriteCloser { + return b.handle.Object(key).NewWriter(ctx) +} + +func (b *realGCSBucket) NewReader(ctx context.Context, key string) (io.ReadCloser, error) { + // The concrete *storage.Reader surfaces storage.ErrObjectNotExist for a + // missing object; pass it through unwrapped so gcsStore.Download can map it + // to ErrNotFound. + return b.handle.Object(key).NewReader(ctx) +} + +func (b *realGCSBucket) Delete(ctx context.Context, key string) error { + if err := b.handle.Object(key).Delete(ctx); err != nil { + return fmt.Errorf("objectstore/gcs: delete %s: %w", key, err) + } + return nil +} + +func (b *realGCSBucket) List(ctx context.Context, prefix string) ([]ObjectInfo, error) { + it := b.handle.Objects(ctx, &gcs.Query{Prefix: prefix}) + var out []ObjectInfo + for { + attrs, err := it.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return nil, fmt.Errorf("objectstore/gcs: list %s: %w", prefix, err) + } + out = append(out, ObjectInfo{ + Key: attrs.Name, + Size: attrs.Size, + LastModified: attrs.Updated, + }) + } + return out, nil +} diff --git a/pkg/objectstore/gcs_test.go b/pkg/objectstore/gcs_test.go new file mode 100644 index 00000000..ed10616a --- /dev/null +++ b/pkg/objectstore/gcs_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + "time" + + gcs "cloud.google.com/go/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeGCSBucket implements gcsBucketHandle in memory. +type fakeGCSBucket struct { + objects map[string][]byte + modTimes map[string]time.Time + clock time.Time + delCalls int +} + +func newFakeGCSBucket() *fakeGCSBucket { + return &fakeGCSBucket{ + objects: map[string][]byte{}, + modTimes: map[string]time.Time{}, + clock: time.Unix(2000, 0), + } +} + +type fakeGCSWriter struct { + bucket *fakeGCSBucket + key string + buf bytes.Buffer +} + +func (w *fakeGCSWriter) Write(p []byte) (int, error) { return w.buf.Write(p) } +func (w *fakeGCSWriter) Close() error { + w.bucket.objects[w.key] = append([]byte(nil), w.buf.Bytes()...) + w.bucket.clock = w.bucket.clock.Add(time.Second) + w.bucket.modTimes[w.key] = w.bucket.clock + return nil +} + +func (b *fakeGCSBucket) NewWriter(_ context.Context, key string) io.WriteCloser { + return &fakeGCSWriter{bucket: b, key: key} +} + +func (b *fakeGCSBucket) NewReader(_ context.Context, key string) (io.ReadCloser, error) { + data, ok := b.objects[key] + if !ok { + // Mirror the real client: a missing object surfaces ErrObjectNotExist. + return nil, gcs.ErrObjectNotExist + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (b *fakeGCSBucket) Delete(_ context.Context, key string) error { + b.delCalls++ + delete(b.objects, key) + return nil +} + +func (b *fakeGCSBucket) List(_ context.Context, prefix string) ([]ObjectInfo, error) { + var out []ObjectInfo + for k, v := range b.objects { + if strings.HasPrefix(k, prefix) { + out = append(out, ObjectInfo{Key: k, Size: int64(len(v)), LastModified: b.modTimes[k]}) + } + } + return out, nil +} + +func newTestGCSStore(bucket gcsBucketHandle, prefix string) *gcsStore { + return &gcsStore{bucket: bucket, name: "test-bucket", prefix: prefix, closer: func() error { return nil }} +} + +func TestGCSStore_UploadJoinsPrefixAndReportsURI(t *testing.T) { + b := newFakeGCSBucket() + store := newTestGCSStore(b, "etcd") + + res, err := store.Upload(context.Background(), "cluster/snap.db", strings.NewReader("payload"), -1) + require.NoError(t, err) + assert.Equal(t, "gs://test-bucket/etcd/cluster/snap.db", res.URI) + assert.Equal(t, int64(7), res.Size) + assert.Equal(t, []byte("payload"), b.objects["etcd/cluster/snap.db"]) +} + +func TestGCSStore_ListNewestFirst(t *testing.T) { + b := newFakeGCSBucket() + store := newTestGCSStore(b, "p") + for _, k := range []string{"c/a.db", "c/b.db", "c/c.db"} { + _, err := store.Upload(context.Background(), k, strings.NewReader(k), -1) + require.NoError(t, err) + } + infos, err := store.List(context.Background(), "c") + require.NoError(t, err) + require.Len(t, infos, 3) + assert.Equal(t, "p/c/c.db", infos[0].Key) +} + +func TestGCSStore_Delete(t *testing.T) { + b := newFakeGCSBucket() + store := newTestGCSStore(b, "p") + _, err := store.Upload(context.Background(), "c/a.db", strings.NewReader("x"), -1) + require.NoError(t, err) + require.NoError(t, store.Delete(context.Background(), "c/a.db")) + assert.Equal(t, 1, b.delCalls) + _, ok := b.objects["p/c/a.db"] + assert.False(t, ok) +} + +func TestGCSStore_DownloadJoinsPrefixAndStreamsBody(t *testing.T) { + b := newFakeGCSBucket() + store := newTestGCSStore(b, "etcd") + _, err := store.Upload(context.Background(), "cluster/snap.db", strings.NewReader("payload"), -1) + require.NoError(t, err) + + rc, err := store.Download(context.Background(), "cluster/snap.db") + require.NoError(t, err) + defer func() { _ = rc.Close() }() + got, err := io.ReadAll(rc) + require.NoError(t, err) + assert.Equal(t, "payload", string(got)) +} + +func TestGCSStore_DownloadMissingIsErrNotFound(t *testing.T) { + b := newFakeGCSBucket() + store := newTestGCSStore(b, "p") + _, err := store.Download(context.Background(), "c/absent.db") + require.Error(t, err) + assert.True(t, errors.Is(err, ErrNotFound), "missing object must map to ErrNotFound, got %v", err) +} + +func TestGCSStore_Scheme(t *testing.T) { + assert.Equal(t, "gs", newTestGCSStore(newFakeGCSBucket(), "").Scheme()) +} diff --git a/pkg/objectstore/interface.go b/pkg/objectstore/interface.go new file mode 100644 index 00000000..78b41ed0 --- /dev/null +++ b/pkg/objectstore/interface.go @@ -0,0 +1,234 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package objectstore defines a pluggable abstraction over object-storage +// backends (S3, GCS, ...) used to upload etcd snapshots. +// +// The package is deliberately decoupled from the Kubernetes API types and the +// controller: a Provider receives plain credentials and a Destination, and +// exposes a minimal Upload/List/Delete surface. This keeps the heavy cloud +// SDKs (aws-sdk-go-v2, cloud.google.com/go/storage) behind a single seam so +// that: +// +// - the rest of the operator can be unit-tested against a fake Provider +// without any cloud credentials, and +// - the implementation could be lifted into a standalone backup-manager +// binary later without touching its callers. +package objectstore + +import ( + "context" + "errors" + "fmt" + "io" + "time" +) + +// ErrNotFound is returned by Download when the requested object does not exist. +// Callers (e.g. the restore controller) use errors.Is to distinguish a missing +// snapshot from a transport error so they can report an actionable, terminal +// failure rather than retrying forever. +var ErrNotFound = errors.New("objectstore: object not found") + +// Provider is the name of an object-storage backend. It mirrors the +// v1alpha1.BackupProvider enum but is redeclared here so the package carries +// no dependency on the API types. +type Provider string + +const ( + // ProviderS3 is the AWS S3 (and S3-compatible) backend. + ProviderS3 Provider = "s3" + // ProviderGCS is the Google Cloud Storage backend. + ProviderGCS Provider = "gcs" +) + +// Credentials carries the secret material a provider needs to authenticate. +// All fields are optional; a provider falls back to its ambient credential +// chain (IRSA, Workload Identity, instance metadata, ...) when the relevant +// fields are empty. This lets operators run credential-free in production +// while still supporting explicit secrets for portability and tests. +type Credentials struct { + // S3 credentials. + AccessKeyID string + SecretAccessKey string + SessionToken string + + // GCS credentials: the raw JSON of a GCP service-account key. + ServiceAccountJSON []byte +} + +// Destination fully describes where an object should be written. It is the +// flattened, provider-agnostic union of the per-provider destination specs in +// the API. The factory validates that the fields relevant to Provider are set. +type Destination struct { + Provider Provider + Bucket string + // Prefix is an optional key prefix; it is joined with the object key. + Prefix string + + // S3-only knobs. + Region string + Endpoint string + ForcePathStyle bool +} + +// ObjectInfo describes a stored object, returned by List. +type ObjectInfo struct { + Key string + Size int64 + LastModified time.Time +} + +// UploadResult reports the outcome of a successful upload. +type UploadResult struct { + // URI is the canonical address of the uploaded object, e.g. + // "s3://bucket/key" or "gs://bucket/key". + URI string + // Size is the number of bytes written. + Size int64 +} + +// Store is the minimal object-storage surface the backup controller needs. +// Implementations must be safe for concurrent use by multiple goroutines. +type Store interface { + // Upload streams r to key under the configured bucket/prefix and returns + // the canonical URI and byte count. size may be -1 if unknown; providers + // that require a known length will buffer as needed. + Upload(ctx context.Context, key string, r io.Reader, size int64) (UploadResult, error) + + // Download opens the object at key (joined with the destination Prefix) for + // reading. The caller owns the returned ReadCloser and must Close it. It is + // the read-side counterpart of Upload, used by the restore path to stream a + // snapshot back out of object storage. Implementations stream the body + // rather than buffering it, so an arbitrarily large snapshot stays bounded + // to the consumer's read buffer. A missing object surfaces as ErrNotFound. + Download(ctx context.Context, key string) (io.ReadCloser, error) + + // List returns objects under the configured bucket/prefix whose key begins + // with keyPrefix (joined with the destination Prefix), most-recent first. + List(ctx context.Context, keyPrefix string) ([]ObjectInfo, error) + + // Delete removes the object at key (joined with the destination Prefix). + Delete(ctx context.Context, key string) error + + // Scheme returns the URI scheme for this store ("s3" or "gs"). + Scheme() string +} + +// Factory builds a Store for a given Destination and Credentials. It is the +// single dispatch point on Destination.Provider; new backends are added by +// extending this switch and dropping a new implementation file in the package. +type Factory func(ctx context.Context, dst Destination, creds Credentials) (Store, error) + +// registry holds the provider constructors. It is package-private and +// populated by each provider's init via Register; New dispatches through it so +// that tests can register fakes without importing cloud SDKs. +var registry = map[Provider]Factory{} + +// Register associates a Factory with a Provider. It is intended to be called +// from provider init functions (see s3.go, gcs.go). Calling it twice for the +// same provider panics, surfacing accidental duplicate registration at startup. +func Register(p Provider, f Factory) { + if _, exists := registry[p]; exists { + panic(fmt.Sprintf("objectstore: provider %q already registered", p)) + } + registry[p] = f +} + +// New constructs a Store for dst.Provider, validating that the destination is +// internally consistent before dispatching to the registered factory. +func New(ctx context.Context, dst Destination, creds Credentials) (Store, error) { + if err := dst.validate(); err != nil { + return nil, err + } + f, ok := registry[dst.Provider] + if !ok { + return nil, fmt.Errorf("objectstore: no provider registered for %q", dst.Provider) + } + return f(ctx, dst, creds) +} + +// isRegistered reports whether a provider has a factory registered. +func isRegistered(p Provider) bool { + _, ok := registry[p] + return ok +} + +// SupportedProviders returns the set of registered providers, sorted is not +// guaranteed; primarily useful for diagnostics and tests. +func SupportedProviders() []Provider { + out := make([]Provider, 0, len(registry)) + for p := range registry { + out = append(out, p) + } + return out +} + +// validate performs provider-agnostic and provider-specific sanity checks on a +// Destination. It is intentionally strict so that misconfiguration surfaces +// before any network I/O. +func (d Destination) validate() error { + if d.Bucket == "" { + return fmt.Errorf("objectstore: destination bucket must not be empty") + } + switch d.Provider { + case ProviderS3: + // Region/Endpoint are optional (endpoint covers S3-compatible stores). + case ProviderGCS: + // Endpoint is shared with S3 and carries the optional emulator URL + // (fake-gcs-server / testbench) for hermetic tests; it is honored by + // newGCSStore. ForcePathStyle remains S3-only addressing semantics and + // has no meaning for GCS, so it is still rejected. + if d.ForcePathStyle { + return fmt.Errorf("objectstore: forcePathStyle is s3-only and unsupported for gcs destination") + } + case "": + return fmt.Errorf("objectstore: destination provider must not be empty") + default: + // Unknown to the built-in switch: accept it only if a provider was + // registered for it (the package is intentionally pluggable), reject + // otherwise. + if !isRegistered(d.Provider) { + return fmt.Errorf("objectstore: unsupported provider %q", d.Provider) + } + } + return nil +} + +// JoinKey joins an optional prefix and a key into a normalized object key. +// Exported so providers and callers share identical key construction. +func JoinKey(prefix, key string) string { + prefix = trimSlashes(prefix) + key = trimSlashes(key) + switch { + case prefix == "": + return key + case key == "": + return prefix + default: + return prefix + "/" + key + } +} + +func trimSlashes(s string) string { + for len(s) > 0 && s[0] == '/' { + s = s[1:] + } + for len(s) > 0 && s[len(s)-1] == '/' { + s = s[:len(s)-1] + } + return s +} diff --git a/pkg/objectstore/memstore_test.go b/pkg/objectstore/memstore_test.go new file mode 100644 index 00000000..28ced44d --- /dev/null +++ b/pkg/objectstore/memstore_test.go @@ -0,0 +1,109 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "bytes" + "context" + "fmt" + "io" + "sort" + "sync" + "time" +) + +// memStore is an in-memory Store implementation used across objectstore tests. +// It models the prefix-joining contract so callers can assert end-to-end key +// behavior without any cloud SDK. +type memStore struct { + scheme string + prefix string + + mu sync.Mutex + objects map[string][]byte + times map[string]time.Time + clock time.Time +} + +func newMemStore(scheme string) *memStore { + return &memStore{ + scheme: scheme, + objects: map[string][]byte{}, + times: map[string]time.Time{}, + clock: time.Unix(0, 0), + } +} + +func (m *memStore) Scheme() string { return m.scheme } + +func (m *memStore) Upload(_ context.Context, key string, r io.Reader, _ int64) (UploadResult, error) { + full := JoinKey(m.prefix, key) + data, err := io.ReadAll(r) + if err != nil { + return UploadResult{}, err + } + m.mu.Lock() + defer m.mu.Unlock() + m.objects[full] = data + m.clock = m.clock.Add(time.Second) + m.times[full] = m.clock + return UploadResult{ + URI: fmt.Sprintf("%s://%s", m.scheme, full), + Size: int64(len(data)), + }, nil +} + +func (m *memStore) Download(_ context.Context, key string) (io.ReadCloser, error) { + full := JoinKey(m.prefix, key) + m.mu.Lock() + defer m.mu.Unlock() + data, ok := m.objects[full] + if !ok { + return nil, fmt.Errorf("memstore: download %q: %w", full, ErrNotFound) + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (m *memStore) List(_ context.Context, keyPrefix string) ([]ObjectInfo, error) { + full := JoinKey(m.prefix, keyPrefix) + m.mu.Lock() + defer m.mu.Unlock() + var out []ObjectInfo + for k, v := range m.objects { + if len(full) == 0 || hasPrefix(k, full) { + out = append(out, ObjectInfo{Key: k, Size: int64(len(v)), LastModified: m.times[k]}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].LastModified.After(out[j].LastModified) }) + return out, nil +} + +func (m *memStore) Delete(_ context.Context, key string) error { + full := JoinKey(m.prefix, key) + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.objects[full]; !ok { + return fmt.Errorf("memstore: object %q not found", full) + } + delete(m.objects, full) + delete(m.times, full) + return nil +} + +func hasPrefix(s, p string) bool { + return len(s) >= len(p) && s[:len(p)] == p +} diff --git a/pkg/objectstore/objectstore_test.go b/pkg/objectstore/objectstore_test.go new file mode 100644 index 00000000..7204aede --- /dev/null +++ b/pkg/objectstore/objectstore_test.go @@ -0,0 +1,128 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJoinKey(t *testing.T) { + cases := []struct { + prefix, key, want string + }{ + {"", "a.db", "a.db"}, + {"etcd", "a.db", "etcd/a.db"}, + {"etcd/", "/a.db", "etcd/a.db"}, + {"/etcd/backups/", "cluster/a.db", "etcd/backups/cluster/a.db"}, + {"etcd", "", "etcd"}, + {"", "", ""}, + } + for _, c := range cases { + assert.Equal(t, c.want, JoinKey(c.prefix, c.key), "JoinKey(%q,%q)", c.prefix, c.key) + } +} + +func TestDestinationValidate(t *testing.T) { + cases := []struct { + name string + dst Destination + wantErr bool + }{ + {"s3 ok", Destination{Provider: ProviderS3, Bucket: "b", Region: "us-east-1"}, false}, + { + "s3 compatible endpoint ok", + Destination{Provider: ProviderS3, Bucket: "b", Endpoint: "http://minio:9000", ForcePathStyle: true}, + false, + }, + {"gcs ok", Destination{Provider: ProviderGCS, Bucket: "b"}, false}, + { + "gcs emulator endpoint ok", + Destination{Provider: ProviderGCS, Bucket: "b", Endpoint: "http://fake-gcs:9000/storage/v1/"}, + false, + }, + {"empty bucket", Destination{Provider: ProviderS3}, true}, + {"empty provider", Destination{Bucket: "b"}, true}, + {"unknown provider", Destination{Provider: "azure", Bucket: "b"}, true}, + {"gcs with forcePathStyle rejected", Destination{Provider: ProviderGCS, Bucket: "b", ForcePathStyle: true}, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err := c.dst.validate() + if c.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +// capture records what the factory was handed, to assert dispatch wiring. +type capture struct { + dst Destination + creds Credentials +} + +func TestNew_DispatchAndValidation(t *testing.T) { + const fakeProvider Provider = "fake-test-provider" + + var captured *capture + Register(fakeProvider, func(_ context.Context, dst Destination, creds Credentials) (Store, error) { + captured = &capture{dst: dst, creds: creds} + return newMemStore("fake"), nil + }) + + t.Run("dispatches to registered factory", func(t *testing.T) { + s, err := New(context.Background(), Destination{ + Provider: fakeProvider, + Bucket: "bucket", + }, Credentials{AccessKeyID: "AK"}) + require.NoError(t, err) + require.NotNil(t, s) + require.NotNil(t, captured) + assert.Equal(t, "bucket", captured.dst.Bucket) + assert.Equal(t, "AK", captured.creds.AccessKeyID) + }) + + t.Run("validation runs before dispatch", func(t *testing.T) { + _, err := New(context.Background(), Destination{Provider: fakeProvider}, Credentials{}) + assert.Error(t, err, "empty bucket must fail validation") + }) + + t.Run("unregistered provider errors", func(t *testing.T) { + _, err := New(context.Background(), Destination{Provider: "nope", Bucket: "b"}, Credentials{}) + assert.Error(t, err) + }) +} + +func TestRegister_DuplicatePanics(t *testing.T) { + const p Provider = "dup-test-provider" + Register(p, func(context.Context, Destination, Credentials) (Store, error) { return nil, nil }) + assert.Panics(t, func() { + Register(p, func(context.Context, Destination, Credentials) (Store, error) { return nil, nil }) + }) +} + +func TestSupportedProviders_IncludesBuiltins(t *testing.T) { + got := SupportedProviders() + assert.Contains(t, got, ProviderS3) + assert.Contains(t, got, ProviderGCS) +} diff --git a/pkg/objectstore/s3.go b/pkg/objectstore/s3.go new file mode 100644 index 00000000..69c49a57 --- /dev/null +++ b/pkg/objectstore/s3.go @@ -0,0 +1,222 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "sort" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +func init() { + Register(ProviderS3, newS3Store) +} + +// s3API is the subset of the S3 client the store uses. Narrowing the surface +// keeps the store testable in isolation (see s3_test.go) without spinning up +// the real client. +type s3API interface { + PutObject( + ctx context.Context, in *s3.PutObjectInput, optFns ...func(*s3.Options), + ) (*s3.PutObjectOutput, error) + GetObject( + ctx context.Context, in *s3.GetObjectInput, optFns ...func(*s3.Options), + ) (*s3.GetObjectOutput, error) + ListObjectsV2( + ctx context.Context, in *s3.ListObjectsV2Input, optFns ...func(*s3.Options), + ) (*s3.ListObjectsV2Output, error) + DeleteObject( + ctx context.Context, in *s3.DeleteObjectInput, optFns ...func(*s3.Options), + ) (*s3.DeleteObjectOutput, error) +} + +// s3Store implements Store against AWS S3 (and S3-compatible endpoints). +type s3Store struct { + client s3API + bucket string + prefix string +} + +// newS3Store builds an s3Store from a Destination and Credentials. When +// explicit credentials are absent it relies on the default AWS credential +// chain (env, shared config, IRSA, instance profile), which is the +// recommended production posture. +func newS3Store(ctx context.Context, dst Destination, creds Credentials) (Store, error) { + loadOpts := []func(*awsconfig.LoadOptions) error{} + if dst.Region != "" { + loadOpts = append(loadOpts, awsconfig.WithRegion(dst.Region)) + } + if creds.AccessKeyID != "" && creds.SecretAccessKey != "" { + loadOpts = append(loadOpts, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider( + creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken), + )) + } + + cfg, err := awsconfig.LoadDefaultConfig(ctx, loadOpts...) + if err != nil { + return nil, fmt.Errorf("objectstore/s3: load aws config: %w", err) + } + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + if dst.Endpoint != "" { + o.BaseEndpoint = aws.String(dst.Endpoint) + } + o.UsePathStyle = dst.ForcePathStyle + }) + + return &s3Store{client: client, bucket: dst.Bucket, prefix: dst.Prefix}, nil +} + +func (s *s3Store) Scheme() string { return "s3" } + +// Upload writes r to the destination. S3 PutObject requires a seekable body +// with a known length; an etcd snapshot stream has neither, so it is spooled +// to an ephemeral temp file first (snapshots are bounded and the operator pod +// has scratch space). The temp file is removed before returning. +func (s *s3Store) Upload(ctx context.Context, key string, r io.Reader, _ int64) (UploadResult, error) { + fullKey := JoinKey(s.prefix, key) + + tmp, err := os.CreateTemp("", "etcd-snapshot-*.db") + if err != nil { + return UploadResult{}, fmt.Errorf("objectstore/s3: create temp file: %w", err) + } + defer func() { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + }() + + size, err := io.Copy(tmp, r) + if err != nil { + return UploadResult{}, fmt.Errorf("objectstore/s3: buffer snapshot: %w", err) + } + if _, err := tmp.Seek(0, io.SeekStart); err != nil { + return UploadResult{}, fmt.Errorf("objectstore/s3: rewind snapshot: %w", err) + } + + _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(fullKey), + Body: tmp, + ContentLength: aws.Int64(size), + }) + if err != nil { + return UploadResult{}, fmt.Errorf("objectstore/s3: put %s: %w", fullKey, err) + } + + return UploadResult{ + URI: fmt.Sprintf("s3://%s/%s", s.bucket, fullKey), + Size: size, + }, nil +} + +// Download opens the object for streaming reads. Unlike Upload (which must spool +// to a temp file because PutObject needs a known length), GetObject returns a +// streaming body, so the snapshot is never staged on the operator's disk on the +// read path. A NoSuchKey response is mapped to ErrNotFound so the restore +// controller can report a terminal "snapshot not found" rather than retrying. +func (s *s3Store) Download(ctx context.Context, key string) (io.ReadCloser, error) { + fullKey := JoinKey(s.prefix, key) + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(fullKey), + }) + if err != nil { + var nsk *s3types.NoSuchKey + if errors.As(err, &nsk) { + return nil, fmt.Errorf("objectstore/s3: get %s: %w", fullKey, ErrNotFound) + } + return nil, fmt.Errorf("objectstore/s3: get %s: %w", fullKey, err) + } + return out.Body, nil +} + +func (s *s3Store) List(ctx context.Context, keyPrefix string) ([]ObjectInfo, error) { + // Append a trailing slash so the prefix matches a directory boundary rather + // than a raw string prefix. Without it, listing "etcd/backups/etcd-a" also + // returns (and retention would delete) objects under "etcd/backups/etcd-a-2". + // JoinKey trims trailing slashes, so the boundary is added here after the + // join. An empty prefix lists the whole bucket and must stay empty. + fullPrefix := JoinKey(s.prefix, keyPrefix) + if fullPrefix != "" { + fullPrefix += "/" + } + return s.listExact(ctx, fullPrefix) +} + +func (s *s3Store) listExact(ctx context.Context, fullPrefix string) ([]ObjectInfo, error) { + var out []ObjectInfo + var token *string + for { + resp, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(s.bucket), + Prefix: aws.String(fullPrefix), + ContinuationToken: token, + }) + if err != nil { + return nil, fmt.Errorf("objectstore/s3: list %s: %w", fullPrefix, err) + } + for _, o := range resp.Contents { + info := ObjectInfo{Key: aws.ToString(o.Key)} + if o.Size != nil { + info.Size = *o.Size + } + if o.LastModified != nil { + info.LastModified = *o.LastModified + } + out = append(out, info) + } + if resp.IsTruncated == nil || !*resp.IsTruncated { + break + } + token = resp.NextContinuationToken + } + + // Most-recent first so callers can apply retention trivially. S3 + // LastModified has one-second granularity, so a stable sort with a + // deterministic tiebreaker on the key (which embeds a sortable UTC + // timestamp) is required; otherwise retention could non-deterministically + // delete the just-uploaded snapshot when its modtime ties an older one. + sort.SliceStable(out, func(i, j int) bool { + if out[i].LastModified.Equal(out[j].LastModified) { + return out[i].Key > out[j].Key + } + return out[i].LastModified.After(out[j].LastModified) + }) + return out, nil +} + +func (s *s3Store) Delete(ctx context.Context, key string) error { + fullKey := JoinKey(s.prefix, key) + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(fullKey), + }) + if err != nil { + return fmt.Errorf("objectstore/s3: delete %s: %w", fullKey, err) + } + return nil +} diff --git a/pkg/objectstore/s3_test.go b/pkg/objectstore/s3_test.go new file mode 100644 index 00000000..cca4eb74 --- /dev/null +++ b/pkg/objectstore/s3_test.go @@ -0,0 +1,232 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectstore + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeS3 implements s3API in memory so the s3Store logic (key joining, URI +// construction, list pagination, retention deletes) is testable without AWS. +type fakeS3 struct { + objects map[string][]byte + modTimes map[string]time.Time + clock time.Time + putCalls int + delCalls int + failPut bool +} + +func newFakeS3() *fakeS3 { + return &fakeS3{ + objects: map[string][]byte{}, + modTimes: map[string]time.Time{}, + clock: time.Unix(1000, 0), + } +} + +func (f *fakeS3) PutObject( + _ context.Context, in *s3.PutObjectInput, _ ...func(*s3.Options), +) (*s3.PutObjectOutput, error) { + f.putCalls++ + if f.failPut { + return nil, io.ErrClosedPipe + } + data, _ := io.ReadAll(in.Body) + key := aws.ToString(in.Key) + f.objects[key] = data + f.clock = f.clock.Add(time.Second) + f.modTimes[key] = f.clock + return &s3.PutObjectOutput{}, nil +} + +func (f *fakeS3) GetObject( + _ context.Context, in *s3.GetObjectInput, _ ...func(*s3.Options), +) (*s3.GetObjectOutput, error) { + key := aws.ToString(in.Key) + data, ok := f.objects[key] + if !ok { + // Mirror the real client: a missing object surfaces as *NoSuchKey. + return nil, &s3types.NoSuchKey{} + } + return &s3.GetObjectOutput{Body: io.NopCloser(bytes.NewReader(data))}, nil +} + +func (f *fakeS3) ListObjectsV2( + _ context.Context, in *s3.ListObjectsV2Input, _ ...func(*s3.Options), +) (*s3.ListObjectsV2Output, error) { + prefix := aws.ToString(in.Prefix) + var keys []string + for k := range f.objects { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + // deterministic order + for i := 0; i < len(keys); i++ { + for j := i + 1; j < len(keys); j++ { + if keys[j] < keys[i] { + keys[i], keys[j] = keys[j], keys[i] + } + } + } + out := &s3.ListObjectsV2Output{} + for _, k := range keys { + sz := int64(len(f.objects[k])) + mt := f.modTimes[k] + out.Contents = append(out.Contents, s3types.Object{ + Key: aws.String(k), + Size: &sz, + LastModified: &mt, + }) + } + return out, nil +} + +func (f *fakeS3) DeleteObject( + _ context.Context, in *s3.DeleteObjectInput, _ ...func(*s3.Options), +) (*s3.DeleteObjectOutput, error) { + f.delCalls++ + delete(f.objects, aws.ToString(in.Key)) + return &s3.DeleteObjectOutput{}, nil +} + +func newTestS3Store(api s3API, prefix string) *s3Store { + return &s3Store{client: api, bucket: "test-bucket", prefix: prefix} +} + +func TestS3Store_UploadJoinsPrefixAndReportsURI(t *testing.T) { + api := newFakeS3() + store := newTestS3Store(api, "etcd/backups") + + res, err := store.Upload(context.Background(), "cluster-a/snap.db", strings.NewReader("hello"), -1) + require.NoError(t, err) + assert.Equal(t, "s3://test-bucket/etcd/backups/cluster-a/snap.db", res.URI) + assert.Equal(t, int64(5), res.Size) + assert.Equal(t, 1, api.putCalls) + _, ok := api.objects["etcd/backups/cluster-a/snap.db"] + assert.True(t, ok, "object stored under joined key") +} + +func TestS3Store_UploadError(t *testing.T) { + api := newFakeS3() + api.failPut = true + store := newTestS3Store(api, "") + _, err := store.Upload(context.Background(), "x.db", strings.NewReader("data"), -1) + assert.Error(t, err) +} + +func TestS3Store_ListNewestFirst(t *testing.T) { + api := newFakeS3() + store := newTestS3Store(api, "p") + for _, k := range []string{"c/1.db", "c/2.db", "c/3.db"} { + _, err := store.Upload(context.Background(), k, strings.NewReader(k), -1) + require.NoError(t, err) + } + infos, err := store.List(context.Background(), "c") + require.NoError(t, err) + require.Len(t, infos, 3) + // modtime increases per upload; newest (3.db) must be first. + assert.Equal(t, "p/c/3.db", infos[0].Key) + assert.True(t, infos[0].LastModified.After(infos[2].LastModified)) +} + +func TestS3Store_ListPrefixIsDirectoryBoundary(t *testing.T) { + // Two clusters whose names share a string prefix ("etcd-a" is a prefix of + // "etcd-a-2") live under the same destination prefix. Listing one cluster's + // snapshots must not return the other's, or retention would delete a live + // cluster's valid backups. + api := newFakeS3() + store := newTestS3Store(api, "etcd/backups") + for _, k := range []string{"etcd-a/1.db", "etcd-a/2.db", "etcd-a-2/1.db"} { + _, err := store.Upload(context.Background(), k, strings.NewReader(k), -1) + require.NoError(t, err) + } + + infos, err := store.List(context.Background(), "etcd-a") + require.NoError(t, err) + require.Len(t, infos, 2, "only etcd-a/* must be listed, not etcd-a-2/*") + for _, info := range infos { + assert.True(t, strings.HasPrefix(info.Key, "etcd/backups/etcd-a/"), + "unexpected key crossed the directory boundary: %s", info.Key) + } +} + +func TestS3Store_ListStableOnEqualModTimes(t *testing.T) { + // When several objects share an identical LastModified (S3's 1s + // granularity), the order must be deterministic (key descending) so the + // newest key is first and never falls into a retention deletion set. + api := newFakeS3() + store := newTestS3Store(api, "p") + tied := time.Unix(2000, 0) + for _, k := range []string{"c/a-20260617T000001Z.db", "c/a-20260617T000003Z.db", "c/a-20260617T000002Z.db"} { + api.objects["p/"+k] = []byte(k) + api.modTimes["p/"+k] = tied + } + + infos, err := store.List(context.Background(), "c") + require.NoError(t, err) + require.Len(t, infos, 3) + // Newest timestamp (…000003Z) sorts first by key on the modtime tie. + assert.Equal(t, "p/c/a-20260617T000003Z.db", infos[0].Key) + assert.Equal(t, "p/c/a-20260617T000001Z.db", infos[2].Key) +} + +func TestS3Store_Delete(t *testing.T) { + api := newFakeS3() + store := newTestS3Store(api, "p") + _, err := store.Upload(context.Background(), "c/1.db", strings.NewReader("x"), -1) + require.NoError(t, err) + require.NoError(t, store.Delete(context.Background(), "c/1.db")) + assert.Equal(t, 1, api.delCalls) + _, ok := api.objects["p/c/1.db"] + assert.False(t, ok) +} + +func TestS3Store_DownloadJoinsPrefixAndStreamsBody(t *testing.T) { + api := newFakeS3() + store := newTestS3Store(api, "etcd/backups") + _, err := store.Upload(context.Background(), "cluster-a/snap.db", strings.NewReader("SNAPSHOT"), -1) + require.NoError(t, err) + + rc, err := store.Download(context.Background(), "cluster-a/snap.db") + require.NoError(t, err) + defer func() { _ = rc.Close() }() + got, err := io.ReadAll(rc) + require.NoError(t, err) + assert.Equal(t, "SNAPSHOT", string(got)) +} + +func TestS3Store_DownloadMissingIsErrNotFound(t *testing.T) { + api := newFakeS3() + store := newTestS3Store(api, "p") + _, err := store.Download(context.Background(), "c/absent.db") + require.Error(t, err) + assert.True(t, errors.Is(err, ErrNotFound), "missing object must map to ErrNotFound, got %v", err) +} diff --git a/test/e2e/backup_providers_test.go b/test/e2e/backup_providers_test.go new file mode 100644 index 00000000..d86badf9 --- /dev/null +++ b/test/e2e/backup_providers_test.go @@ -0,0 +1,455 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "strconv" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/envconf" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// Pinned images for the GCS emulator and the snapshot-inspection tooling. As +// with the MinIO/mc pins, fixed tags keep the gated e2e reproducible rather +// than tracking whatever the registry publishes that day. +const ( + // fakeGCSImage is the fsouza fake-gcs-server emulator. It speaks the GCS + // JSON API, so the operator's GCS provider (pointed at it via the new + // GCSDestinationSpec.Endpoint) exercises the real upload/list/download code + // path with no Google credentials. + fakeGCSImage = "fsouza/fake-gcs-server:1.52.2" + // etcdToolsImage carries `etcdutl`, used to independently validate that an + // uploaded object is a real, intact etcd backend snapshot (not merely a + // blob of the right size). It is the same distroless etcd image the + // operator deploys, so `etcdutl` is always present. + etcdToolsImage = "gcr.io/etcd-development/etcd:v3.6.1" + // curlImage is a tiny shell+curl image used to fetch a GCS object's bytes + // (and metadata) directly from the emulator, independent of the operator. + curlImage = "curlimages/curl:8.11.1" +) + +// richSeedKeyCount is the number of content-addressable keys seeded into the +// source cluster on top of the edge-case keys. Each value is a pure function of +// its key (hex(sha256(key))) so any corruption of the snapshot's bbolt pages is +// a guaranteed mismatch rather than a maybe. +const richSeedKeyCount = 50 + +// Edge-case and provenance keys. These are seeded alongside the content- +// addressable keyspace to exercise value shapes a naive round-trip mishandles, +// and to give the (gated) restore test point-in-time provenance controls. +const ( + edgeLargeKey = "cyc/edge/large" // ~100 KiB value: forces multi-page bbolt + edgeUTF8Key = "cyc/edge/utf8" // multibyte UTF-8 value + sentinelPre = "cyc/sentinel/before" + sentinelPost = "cyc/sentinel/after" + // largeValueBytes is kept well under the kernel's per-arg execve limit + // (MAX_ARG_STRLEN, ~128 KiB) because the value is passed to `etcdctl put` + // as an argv argument; 100 KiB still spans multiple bbolt pages and sits + // far above the 16 KiB snapshot-size floor the presence check applies. + largeValueBytes = 100 * 1024 +) + +// contentAddressableValue is the single source of truth mapping a seeded key to +// its value, shared by the seed and (gated) restore-verify paths so both agree +// on exactly what must be present. +func contentAddressableValue(key string) string { + sum := sha256.Sum256([]byte(key)) + return hex.EncodeToString(sum[:]) +} + +func richSeedKey(i int) string { return fmt.Sprintf("cyc/k-%04d", i) } + +// expectedSeedKeyCount is the total number of distinct keys the seed writes +// (content-addressable keys + the large + utf8 edges + the pre-backup +// sentinel). The post-backup sentinel is written AFTER the snapshot, so it is +// intentionally excluded from the in-snapshot count. +func expectedSeedKeyCount() int { + return richSeedKeyCount + 3 // large, utf8, sentinelPre +} + +// seedRichKeyspace writes the full pre-backup keyspace into the member via +// per-key `etcdctl put` execs (the distroless etcd image ships etcdctl but no +// shell, so each put is its own argv exec). It returns nothing; the count that +// must appear in the snapshot is expectedSeedKeyCount(). +func seedRichKeyspace(t *testing.T, cfg *envconf.Config, podName string) { + t.Helper() + put := func(k, v string) { + if _, stderr, err := backupExecInPod(t, cfg, podName, []string{"etcdctl", "put", k, v}); err != nil { + t.Fatalf("seed key %q: %v (stderr: %s)", k, err, stderr) + } + } + for i := 0; i < richSeedKeyCount; i++ { + k := richSeedKey(i) + put(k, contentAddressableValue(k)) + } + put(edgeLargeKey, strings.Repeat("L", largeValueBytes)) + put(edgeUTF8Key, "héllo·wörld·🔬·αβγ") + // sentinelPre is written BEFORE the backup and must survive a restore. + put(sentinelPre, contentAddressableValue(sentinelPre)) +} + +// --- provider abstraction -------------------------------------------------- + +// backupBackend abstracts an in-cluster object-storage backend (MinIO for S3, +// fake-gcs-server for GCS) so the full backup cycle can be driven once and run +// against both providers. Each method is independent so two providers can run +// side by side in the shared namespace without colliding. +type backupBackend interface { + // name is the backend's Deployment/Service name (also its in-cluster DNS). + name() string + // deploy stands up the backend Deployment+Service and waits for Available, + // failing fast (with logs) on a crash/pull back-off. + deploy(ctx context.Context, t *testing.T, cfg *envconf.Config) + // bootstrapBucket creates the destination bucket via a one-shot pod. + bootstrapBucket(ctx context.Context, t *testing.T, cfg *envconf.Config) + // destination builds the EtcdBackup destination targeting this backend. + destination(prefix string) ecv1alpha1.BackupDestination + // statObjectSize returns the stored object's size in bytes, fetched + // directly from the backend (not via the operator), failing if absent. + statObjectSize(t *testing.T, cfg *envconf.Config, key string) int64 + // downloadInitContainer returns an init container that fetches the object at + // key into /snap/s.db on a shared emptyDir, for the etcdutl validity check. + downloadInitContainer(key string) corev1.Container + // credsSecretName is the creds Secret the destination references, or "" when + // the backend is unauthenticated (GCS emulator). + credsSecretName() string + // cleanup best-effort removes the backend Deployment/Service. + cleanup(ctx context.Context, t *testing.T, cfg *envconf.Config) +} + +// --- S3 / MinIO backend ----------------------------------------------------- + +type minioBackend struct { + instance string + bucket string + accessKey string + secretKey string + creds string +} + +func newMinIOBackend(instance, bucket, credsSecret string) *minioBackend { + return &minioBackend{ + instance: instance, bucket: bucket, + accessKey: "minioadmin", secretKey: "minioadmin", creds: credsSecret, + } +} + +func (m *minioBackend) name() string { return m.instance } +func (m *minioBackend) credsSecretName() string { return m.creds } + +func (m *minioBackend) deploy(ctx context.Context, t *testing.T, cfg *envconf.Config) { + deployMinIO(ctx, t, cfg, m.instance, m.accessKey, m.secretKey) +} + +func (m *minioBackend) bootstrapBucket(ctx context.Context, t *testing.T, cfg *envconf.Config) { + createBucketPod(ctx, t, cfg, m.instance, m.bucket, m.accessKey, m.secretKey) + createBackupCredsSecret(ctx, t, cfg, m.creds, m.accessKey, m.secretKey) +} + +func (m *minioBackend) destination(prefix string) ecv1alpha1.BackupDestination { + return ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderS3, + Prefix: prefix, + SecretRef: &corev1.LocalObjectReference{Name: m.creds}, + S3: &ecv1alpha1.S3DestinationSpec{ + Bucket: m.bucket, + Region: "us-east-1", + Endpoint: "http://" + m.instance + "." + namespace + ".svc.cluster.local:9000", + ForcePathStyle: true, + }, + } +} + +func (m *minioBackend) statObjectSize(t *testing.T, cfg *envconf.Config, key string) int64 { + return statMinIOObjectSize(t, cfg, m.instance, m.bucket, key, m.accessKey, m.secretKey) +} + +func (m *minioBackend) downloadInitContainer(key string) corev1.Container { + script := "mc alias set m http://" + m.instance + ":9000 " + m.accessKey + " " + m.secretKey + + " && mc cp m/" + m.bucket + "/" + key + " /snap/s.db" + return corev1.Container{ + Name: "fetch", + Image: mcImage, + Command: []string{"sh", "-c", script}, + VolumeMounts: []corev1.VolumeMount{{Name: "snap", MountPath: "/snap"}}, + } +} + +func (m *minioBackend) cleanup(ctx context.Context, t *testing.T, cfg *envconf.Config) { + cleanupBackupWorkloads(ctx, t, cfg, m.instance, m.creds, nil) +} + +// --- GCS / fake-gcs-server backend ----------------------------------------- + +type fakeGCSBackend struct { + instance string + bucket string +} + +func newFakeGCSBackend(instance, bucket string) *fakeGCSBackend { + return &fakeGCSBackend{instance: instance, bucket: bucket} +} + +func (g *fakeGCSBackend) name() string { return g.instance } +func (g *fakeGCSBackend) credsSecretName() string { return "" } // unauthenticated emulator + +// gcsBaseURL is the in-cluster URL of the emulator, used both for the operator +// endpoint and for the independent stat/download checks. +func (g *fakeGCSBackend) gcsBaseURL() string { + return "http://" + g.instance + "." + namespace + ".svc.cluster.local:9000" +} + +func (g *fakeGCSBackend) deploy(ctx context.Context, t *testing.T, cfg *envconf.Config) { + t.Helper() + labels := map[string]string{"app": g.instance} + replicas := int32(1) + publicHost := g.instance + "." + namespace + ".svc.cluster.local:9000" + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: g.instance, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "fake-gcs", + Image: fakeGCSImage, + // -scheme http: the operator dials plain HTTP. -public-host + // / -external-url are set to the in-cluster Service DNS so + // the emulator's host/signed-URL rewrites match how the + // operator (and the stat/download pods) reach it. + Args: []string{ + "-scheme", "http", + "-host", "0.0.0.0", + "-port", "9000", + "-public-host", publicHost, + "-external-url", "http://" + publicHost, + }, + Ports: []corev1.ContainerPort{{ContainerPort: 9000}}, + }}, + }, + }, + }, + } + if err := cfg.Client().Resources().Create(ctx, deploy); err != nil { + t.Fatalf("create fake-gcs deployment: %v", err) + } + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: g.instance, Namespace: namespace}, + Spec: corev1.ServiceSpec{ + Selector: labels, + Ports: []corev1.ServicePort{{Port: 9000, TargetPort: intstr.FromInt(9000)}}, + }, + } + if err := cfg.Client().Resources().Create(ctx, svc); err != nil { + t.Fatalf("create fake-gcs service: %v", err) + } + if err := wait.For(conditions.New(cfg.Client().Resources()). + DeploymentConditionMatch(deploy, appsv1.DeploymentAvailable, corev1.ConditionTrue), + wait.WithTimeout(2*time.Minute), wait.WithInterval(3*time.Second)); err != nil { + dumpPodTrouble(ctx, t, cfg, labels) + t.Fatalf("fake-gcs %q not available: %v", g.instance, err) + } +} + +func (g *fakeGCSBackend) bootstrapBucket(ctx context.Context, t *testing.T, cfg *envconf.Config) { + t.Helper() + // fake-gcs-server creates buckets via POST /storage/v1/b?project=… . Run it + // in a one-shot curl pod, named per-instance so two GCS features in the + // shared namespace do not collide. + podName := "gcs-mkbucket-" + g.instance + body := `{"name":"` + g.bucket + `"}` + script := "curl -sf -X POST '" + g.gcsBaseURL() + "/storage/v1/b?project=e2e' " + + "-H 'Content-Type: application/json' -d '" + body + "'" + g.runCurlPod(ctx, t, cfg, podName, script) +} + +func (g *fakeGCSBackend) destination(prefix string) ecv1alpha1.BackupDestination { + return ecv1alpha1.BackupDestination{ + Provider: ecv1alpha1.BackupProviderGCS, + Prefix: prefix, + // No SecretRef: the emulator is unauthenticated, exercising the new + // WithEndpoint + WithoutAuthentication path in newGCSStore. + GCS: &ecv1alpha1.GCSDestinationSpec{ + Bucket: g.bucket, + Endpoint: g.gcsBaseURL() + "/storage/v1/", + }, + } +} + +func (g *fakeGCSBackend) statObjectSize(t *testing.T, cfg *envconf.Config, key string) int64 { + t.Helper() + // GCS JSON API: GET /storage/v1/b//o/ returns + // metadata with a "size" field (a *string* in the GCS API, unlike mc). + enc := url.PathEscape(key) + script := "curl -sf '" + g.gcsBaseURL() + "/storage/v1/b/" + g.bucket + "/o/" + enc + "'" + out := g.runCurlPod(t.Context(), t, cfg, "gcs-stat-"+g.instance, script) + size, ok := jsonInt64Field(out, "size") + if !ok { + t.Fatalf("could not read GCS object size for %s/%s; curl output: %s", g.bucket, key, out) + } + return size +} + +func (g *fakeGCSBackend) downloadInitContainer(key string) corev1.Container { + enc := url.PathEscape(key) + url := g.gcsBaseURL() + "/storage/v1/b/" + g.bucket + "/o/" + enc + "?alt=media" + return corev1.Container{ + Name: "fetch", + Image: curlImage, + Command: []string{"sh", "-c", "curl -sf -o /snap/s.db '" + url + "'"}, + VolumeMounts: []corev1.VolumeMount{{Name: "snap", MountPath: "/snap"}}, + } +} + +func (g *fakeGCSBackend) cleanup(ctx context.Context, t *testing.T, cfg *envconf.Config) { + res := cfg.Client().Resources() + _ = res.Delete(ctx, &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: g.instance, Namespace: namespace}}) + _ = res.Delete(ctx, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: g.instance, Namespace: namespace}}) + _ = res.Delete(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "gcs-mkbucket-" + g.instance, Namespace: namespace}}) +} + +// runCurlPod runs a one-shot curl pod with the given script and returns its +// stdout, failing — with the pod's logs — if it does not succeed. +func (g *fakeGCSBackend) runCurlPod(ctx context.Context, t *testing.T, cfg *envconf.Config, podBase, script string) string { + t.Helper() + podName := podBase + "-" + strconv.FormatInt(time.Now().UnixNano()%100000, 10) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: namespace}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{{ + Name: "curl", + Image: curlImage, + Command: []string{"sh", "-c", script}, + }}, + }, + } + if err := cfg.Client().Resources().Create(ctx, pod); err != nil { + t.Fatalf("create curl pod %s: %v", podName, err) + } + defer func() { _ = cfg.Client().Resources().Delete(ctx, pod) }() + if err := waitPodSucceeded(ctx, t, cfg, podName); err != nil { + t.Fatalf("curl pod %s did not succeed: %v", podName, err) + } + logs, err := podLogs(ctx, cfg, podName) + if err != nil { + t.Fatalf("read curl pod logs %s: %v", podName, err) + } + return logs +} + +// --- independent snapshot validity (etcdutl) -------------------------------- + +// etcdutlSnapshotStatus is the subset of `etcdutl snapshot status -w json` +// output the validity check asserts on. +type etcdutlSnapshotStatus struct { + Hash int64 `json:"hash"` + Revision int64 `json:"revision"` + TotalKey int64 `json:"totalKey"` + Version string `json:"version"` +} + +// assertSnapshotValid downloads the uploaded object directly from the backend +// (via an init container) and runs `etcdutl snapshot status` on it in a main +// container, proving — entirely independently of the operator's status — that +// the object is a real, intact etcd backend snapshot AND that the seeded key +// count actually made it into the snapshot. A corrupt/truncated object makes +// etcdutl exit non-zero; a snapshot missing keys fails the count assertion. +func assertSnapshotValid( + t *testing.T, cfg *envconf.Config, backend backupBackend, key string, wantKeys int64, +) { + t.Helper() + ctx := t.Context() + podName := "snap-status-" + backend.name() + "-" + strconv.FormatInt(time.Now().UnixNano()%100000, 10) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: namespace}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + InitContainers: []corev1.Container{backend.downloadInitContainer(key)}, + Containers: []corev1.Container{{ + Name: "etcdutl", + Image: etcdToolsImage, + Command: []string{"/usr/local/bin/etcdutl", "snapshot", "status", "/snap/s.db", "-w", "json"}, + VolumeMounts: []corev1.VolumeMount{{Name: "snap", MountPath: "/snap"}}, + }}, + Volumes: []corev1.Volume{{Name: "snap", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}, + }, + } + if err := cfg.Client().Resources().Create(ctx, pod); err != nil { + t.Fatalf("create snapshot-status pod %s: %v", podName, err) + } + defer func() { _ = cfg.Client().Resources().Delete(ctx, pod) }() + if err := waitPodSucceeded(ctx, t, cfg, podName); err != nil { + t.Fatalf("etcdutl snapshot status pod %s did not succeed (snapshot invalid/corrupt or fetch failed): %v", podName, err) + } + out, err := podLogs(ctx, cfg, podName) + if err != nil { + t.Fatalf("read etcdutl status logs %s: %v", podName, err) + } + status := parseEtcdutlStatus(t, out) + if status.TotalKey != wantKeys { + t.Errorf("snapshot totalKey=%d, want %d (the snapshot does not contain the seeded keyspace)", status.TotalKey, wantKeys) + } + if status.Revision <= 0 || status.Hash == 0 { + t.Errorf("snapshot status looks degenerate: %+v", status) + } + t.Logf("etcdutl snapshot status OK: totalKey=%d revision=%d version=%s", + status.TotalKey, status.Revision, status.Version) +} + +// parseEtcdutlStatus extracts the status fields from etcdutl's -w json line +// without a struct-tag JSON dependency on the whole pod-log blob (the line may +// be preceded by warnings on stderr, but stdout carries only the JSON). +func parseEtcdutlStatus(t *testing.T, out string) etcdutlSnapshotStatus { + t.Helper() + var s etcdutlSnapshotStatus + if v, ok := jsonInt64Field(out, "totalKey"); ok { + s.TotalKey = v + } else { + t.Fatalf("etcdutl status: no totalKey in output: %s", out) + } + if v, ok := jsonInt64Field(out, "revision"); ok { + s.Revision = v + } + if v, ok := jsonInt64Field(out, "hash"); ok { + s.Hash = v + } + if i := strings.Index(out, "\"version\":\""); i >= 0 { + rest := out[i+len("\"version\":\""):] + if j := strings.IndexByte(rest, '"'); j >= 0 { + s.Version = rest[:j] + } + } + return s +} diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go new file mode 100644 index 00000000..b61e18d1 --- /dev/null +++ b/test/e2e/backup_test.go @@ -0,0 +1,593 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "strconv" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/klient/wait/conditions" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// Pinned MinIO/mc images. Using :latest makes the e2e's pass/fail depend on +// whatever MinIO publishes that day (flag/alias semantics have churned +// historically); a gated "optional" e2e would rot silently. Pin to known-good +// RELEASE tags so the test is reproducible. +const ( + minioImage = "quay.io/minio/minio:RELEASE.2025-04-22T22-12-26Z" + mcImage = "quay.io/minio/mc:RELEASE.2025-04-16T18-13-26Z" +) + +// backupSeedKeyCount is the number of keys seeded into the source cluster. A +// single key is the weakest possible proof that a snapshot round-trips; seeding +// a whole keyspace lets the restore side assert the *entire* dataset survives +// (data-loss is the worst failure mode of a backup/restore feature). +const backupSeedKeyCount = 50 + +// TestEtcdBackupToS3 exercises the EtcdBackup snapshot+upload flow end-to-end +// against an in-cluster MinIO (S3-compatible) bucket, so it requires NO real +// cloud credentials. It is gated behind ETCD_E2E_BACKUP=true because it +// provisions extra workloads (MinIO) and is not part of the default e2e matrix. +func TestEtcdBackupToS3(t *testing.T) { + requireBackupE2E(t) + runFullBackupCycle(t, + newMinIOBackend("minio-backup-e2e", "etcd-backups", "backup-e2e-creds"), + "etcd-backup-s3", "etcd-backup-e2e", "etcd-backup-e2e-snap") +} + +// TestEtcdBackupToGCS exercises the identical snapshot+upload flow against an +// in-cluster fake-gcs-server (GCS JSON-API emulator), via the new +// GCSDestinationSpec.Endpoint + the unauthenticated WithEndpoint path in +// newGCSStore. It requires NO real Google credentials and runs the SAME +// present-and-valid assertion stack as the S3 cycle, so the operator's +// second advertised provider is exercised live rather than only with fakes. +func TestEtcdBackupToGCS(t *testing.T) { + requireBackupE2E(t) + runFullBackupCycle(t, + newFakeGCSBackend("fake-gcs-backup-e2e", "etcd-backups"), + "etcd-backup-gcs", "etcd-backup-gcs-e2e", "etcd-backup-gcs-e2e-snap") +} + +// requireBackupE2E skips unless the object-storage backup e2e is explicitly +// enabled; these tests provision extra in-cluster workloads (MinIO / fake-gcs) +// and are not part of the default matrix. +func requireBackupE2E(t *testing.T) { + t.Helper() + if os.Getenv("ETCD_E2E_BACKUP") != "true" { + t.Skip("set ETCD_E2E_BACKUP=true to run the object-storage backup e2e (provisions a backend)") + } +} + +// runFullBackupCycle drives one provider through the complete backup proof: +// +// 1. Stand up the object-storage backend + bucket (+ creds when authenticated). +// 2. Create a single-member EtcdCluster, wait for it ready, and seed a rich, +// content-addressable keyspace (50 keys whose values are hash(key)) plus +// edge values (a ~100 KiB large value, a multibyte UTF-8 value) and a +// pre-backup provenance sentinel. +// 3. Create an EtcdBackup at the backend and wait for phase Completed. +// 4. Assert the controller status is coherent (size>0, location, completion). +// 5. Independently of the controller, stat the object directly on the backend +// and assert its size equals status.SnapshotSizeBytes and clears the 16 KiB +// real-snapshot floor. +// 6. Strongest, controller-independent validity proof: download the object's +// bytes and run `etcdutl snapshot status` on them, asserting it is an +// intact etcd backend snapshot whose totalKey equals the seeded key count. +// A controller that set status without uploading, or uploaded a truncated +// blob, or uploaded a snapshot missing the seeded data, fails one of (5)/(6). +func runFullBackupCycle(t *testing.T, backend backupBackend, featureName, clusterName, backupName string) { + feature := features.New(featureName) + + feature.Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + backend.deploy(ctx, t, cfg) + backend.bootstrapBucket(ctx, t, cfg) + createBackupTestCluster(ctx, t, cfg, clusterName, 1) + waitStatefulSetReady(t, cfg, clusterName, 1) + seedRichKeyspace(t, cfg, clusterName+"-0") + + backup := &ecv1alpha1.EtcdBackup{ + ObjectMeta: metav1.ObjectMeta{Name: backupName, Namespace: namespace}, + Spec: ecv1alpha1.EtcdBackupSpec{ + ClusterRef: clusterName, + Destination: backend.destination("e2e"), + }, + } + if err := cfg.Client().Resources().Create(ctx, backup); err != nil { + t.Fatalf("create EtcdBackup: %v", err) + } + return ctx + }) + + feature.Assess("backup reaches Completed and the object is a valid snapshot containing the seeded keyspace", func( + ctx context.Context, t *testing.T, cfg *envconf.Config, + ) context.Context { + done := waitBackupCompleted(t, cfg, backupName) + + // (1) The controller's self-reported status must be coherent. + if done.Status.SnapshotSizeBytes <= 0 { + t.Errorf("expected positive snapshot size, got %d", done.Status.SnapshotSizeBytes) + } + if done.Status.SnapshotLocation == "" { + t.Errorf("expected snapshot location to be set") + } + if done.Status.CompletionTime == nil { + t.Errorf("expected completion time to be set") + } + + // (2) Independently verify the object is actually in object storage with + // the recorded size. A controller that set the status fields without + // uploading a byte would pass (1) but fail here. + objectKey := objectKeyFromLocation(t, done.Status.SnapshotLocation) + size := backend.statObjectSize(t, cfg, objectKey) + if size != done.Status.SnapshotSizeBytes { + t.Errorf("backend object size %d != status.SnapshotSizeBytes %d (key %q)", + size, done.Status.SnapshotSizeBytes, objectKey) + } + + // (3) Sanity floor: a real etcd backend snapshot is a bbolt DB and is + // never a handful of bytes. + if size < 16*1024 { + t.Errorf("snapshot object only %d bytes; too small to be a real etcd backend snapshot", size) + } + + // (4) Strongest check: download the bytes and prove via `etcdutl + // snapshot status` that it is an intact etcd snapshot whose key count + // equals the seeded keyspace — controller status is never consulted. + assertSnapshotValid(t, cfg, backend, objectKey, int64(expectedSeedKeyCount())) + + t.Logf("backup completed and verified: location=%s size=%d key=%s", + done.Status.SnapshotLocation, done.Status.SnapshotSizeBytes, objectKey) + return ctx + }) + + feature.Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + backend.cleanup(ctx, t, cfg) + _ = cfg.Client().Resources().Delete(ctx, &ecv1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{Name: clusterName, Namespace: namespace}}) + _ = cfg.Client().Resources().Delete(ctx, &ecv1alpha1.EtcdBackup{ + ObjectMeta: metav1.ObjectMeta{Name: backupName, Namespace: namespace}}) + return ctx + }) + + _ = testEnv.Test(t, feature.Feature()) +} + +// --- locally-scoped helpers (intentionally self-contained) ------------------ + +func backupExecInPod(t *testing.T, cfg *envconf.Config, podName string, command []string) (string, string, error) { + t.Helper() + var stdout, stderr bytes.Buffer + var pod corev1.Pod + if err := cfg.Client().Resources().Get(t.Context(), podName, namespace, &pod); err != nil { + return "", "", err + } + containerName := pod.Spec.Containers[0].Name + err := cfg.Client().Resources().ExecInPod(t.Context(), namespace, podName, containerName, command, &stdout, &stderr) + return stdout.String(), stderr.String(), err +} + +// seedKeyspace writes backupSeedKeyCount deterministic key/value pairs into the +// member via direct etcdctl exec (the etcd image ships etcdctl but no shell, so +// each put is a separate argv exec). Values are content-addressable so the +// restore side can assert exact round-trip, not mere presence. +func seedKeyspace(t *testing.T, cfg *envconf.Config, podName string, count int) { + t.Helper() + for i := 0; i < count; i++ { + k, v := seedKV(i) + if _, stderr, err := backupExecInPod(t, cfg, podName, + []string{"etcdctl", "put", k, v}); err != nil { + t.Fatalf("seed key %s: %v (stderr: %s)", k, err, stderr) + } + } +} + +// seedKV is the single source of truth for the seeded keyspace so backup and +// restore agree on exactly which keys/values must exist. +func seedKV(i int) (string, string) { + return fmt.Sprintf("restore-e2e/k-%03d", i), fmt.Sprintf("val-%03d-payload", i) +} + +func createBackupTestCluster(ctx context.Context, t *testing.T, cfg *envconf.Config, name string, size int) { + t.Helper() + cluster := &ecv1alpha1.EtcdCluster{ + TypeMeta: metav1.TypeMeta{APIVersion: "operator.etcd.io/v1alpha1", Kind: "EtcdCluster"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: ecv1alpha1.EtcdClusterSpec{Size: size, Version: "v3.6.1"}, + } + if err := cfg.Client().Resources().Create(ctx, cluster); err != nil { + t.Fatalf("create cluster %s: %v", name, err) + } +} + +func waitStatefulSetReady(t *testing.T, cfg *envconf.Config, name string, replicas int32) { + t.Helper() + err := wait.For(func(ctx context.Context) (bool, error) { + var sts appsv1.StatefulSet + if err := cfg.Client().Resources().Get(ctx, name, namespace, &sts); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return sts.Status.ReadyReplicas == replicas, nil + }, wait.WithTimeout(3*time.Minute), wait.WithInterval(3*time.Second)) + if err != nil { + t.Fatalf("statefulset %s not ready: %v", name, err) + } +} + +func createBackupCredsSecret( + ctx context.Context, t *testing.T, cfg *envconf.Config, name, accessKey, secretKey string, +) { + t.Helper() + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Data: map[string][]byte{ + "accessKeyID": []byte(accessKey), + "secretAccessKey": []byte(secretKey), + }, + } + if err := cfg.Client().Resources().Create(ctx, secret); err != nil { + t.Fatalf("create creds secret: %v", err) + } +} + +func deployMinIO(ctx context.Context, t *testing.T, cfg *envconf.Config, name, accessKey, secretKey string) { + t.Helper() + labels := map[string]string{"app": name} + replicas := int32(1) + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "minio", + Image: minioImage, + Args: []string{"server", "/data", "--address", ":9000"}, + Env: []corev1.EnvVar{ + {Name: "MINIO_ROOT_USER", Value: accessKey}, + {Name: "MINIO_ROOT_PASSWORD", Value: secretKey}, + }, + Ports: []corev1.ContainerPort{{ContainerPort: 9000}}, + }}, + }, + }, + }, + } + if err := cfg.Client().Resources().Create(ctx, deploy); err != nil { + t.Fatalf("create minio deployment: %v", err) + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: corev1.ServiceSpec{ + Selector: labels, + Ports: []corev1.ServicePort{{Port: 9000, TargetPort: intstr.FromInt(9000)}}, + }, + } + if err := cfg.Client().Resources().Create(ctx, svc); err != nil { + t.Fatalf("create minio service: %v", err) + } + + // Wait for Available, but fail fast (with logs) if the pod is wedged in a + // crash/pull-back-off rather than spinning until the deadline with no clue. + if err := wait.For(conditions.New(cfg.Client().Resources()). + DeploymentConditionMatch(deploy, appsv1.DeploymentAvailable, corev1.ConditionTrue), + wait.WithTimeout(2*time.Minute), wait.WithInterval(3*time.Second)); err != nil { + dumpPodTrouble(ctx, t, cfg, labels) + t.Fatalf("minio %q not available: %v", name, err) + } +} + +func createBucketPod( + ctx context.Context, t *testing.T, cfg *envconf.Config, minioName, bucket, accessKey, secretKey string, +) { + t.Helper() + // Name the bootstrap pod per-MinIO-instance so two backup/restore features in + // the shared namespace do not collide on a singleton "mc-mkbucket". + podName := "mc-mkbucket-" + minioName + script := "mc alias set m http://" + minioName + ":9000 " + accessKey + " " + secretKey + + " && mc mb --ignore-existing m/" + bucket + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: namespace}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + Containers: []corev1.Container{{ + Name: "mc", + Image: mcImage, + Command: []string{"sh", "-c", script}, + }}, + }, + } + _ = cfg.Client().Resources().Delete(ctx, pod) // tolerate a leftover from a killed run + if err := waitPodGone(ctx, cfg, podName); err != nil { + t.Fatalf("waiting for stale %s to clear: %v", podName, err) + } + if err := cfg.Client().Resources().Create(ctx, pod); err != nil { + t.Fatalf("create bucket-create pod: %v", err) + } + // Succeed OR fail loudly: a Failed pod (bad alias, MinIO down) must surface + // its logs immediately, not time out opaquely after two minutes. + if err := waitPodSucceeded(ctx, t, cfg, podName); err != nil { + t.Fatalf("bucket-create pod did not succeed: %v", err) + } +} + +// --- object-storage verification (mc one-shot pods) ------------------------- + +// objectKeyFromLocation extracts the bucket-relative object key from a snapshot +// location URI like "s3://etcd-backups/e2e/etcd-backup-e2e/...snap.db". The key +// is everything after the bucket segment. +func objectKeyFromLocation(t *testing.T, location string) string { + t.Helper() + rest := location + if i := strings.Index(rest, "://"); i >= 0 { + rest = rest[i+3:] + } + // rest is now "/"; drop the first path segment (the bucket). + if i := strings.Index(rest, "/"); i >= 0 { + return rest[i+1:] + } + t.Fatalf("cannot parse object key from snapshot location %q", location) + return "" +} + +// statMinIOObjectSize runs a one-shot mc pod to `mc stat` the object and returns +// its size in bytes, failing the test if the object is absent. This is the +// independent "is it really in the bucket?" check the status fields cannot give. +func statMinIOObjectSize( + t *testing.T, cfg *envconf.Config, minioName, bucket, key, accessKey, secretKey string, +) int64 { + t.Helper() + // `mc stat --json` emits a JSON line with a "size" field. Parse just that. + script := "mc alias set m http://" + minioName + ":9000 " + accessKey + " " + secretKey + + " >/dev/null && mc stat --json m/" + bucket + "/" + key + out := runMCJob(t, cfg, "mc-stat-"+minioName, script) + size, ok := jsonInt64Field(out, "size") + if !ok { + t.Fatalf("could not read object size for m/%s/%s; mc output: %s", bucket, key, out) + } + return size +} + +// jsonInt64Field extracts an integer field from mc's --json output without a +// full JSON parser dependency: finds `"":`. +func jsonInt64Field(jsonOut, field string) (int64, bool) { + needle := "\"" + field + "\":" + i := strings.Index(jsonOut, needle) + if i < 0 { + return 0, false + } + rest := jsonOut[i+len(needle):] + j := 0 + // Skip leading spaces and an optional opening quote: mc emits a bare number + // ("size":22) while the GCS JSON API quotes it ("size":"22"). Both must + // parse with the same extractor. + for j < len(rest) && (rest[j] == ' ' || rest[j] == '"') { + j++ + } + start := j + for j < len(rest) && rest[j] >= '0' && rest[j] <= '9' { + j++ + } + if start == j { + return 0, false + } + n, err := strconv.ParseInt(rest[start:j], 10, 64) + if err != nil { + return 0, false + } + return n, true +} + +// runMCJob runs a one-shot mc pod with the given shell script and returns its +// stdout (the pod logs), failing the test — with the pod's logs — if it does +// not succeed. +func runMCJob(t *testing.T, cfg *envconf.Config, podBase, script string) string { + t.Helper() + ctx := t.Context() + podName := podBase + "-" + strconv.FormatInt(time.Now().UnixNano()%100000, 10) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: namespace}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{{ + Name: "mc", + Image: mcImage, + Command: []string{"sh", "-c", script}, + }}, + }, + } + if err := cfg.Client().Resources().Create(ctx, pod); err != nil { + t.Fatalf("create mc job pod %s: %v", podName, err) + } + defer func() { _ = cfg.Client().Resources().Delete(ctx, pod) }() + if err := waitPodSucceeded(ctx, t, cfg, podName); err != nil { + t.Fatalf("mc job %s did not succeed: %v", podName, err) + } + logs, err := podLogs(ctx, cfg, podName) + if err != nil { + t.Fatalf("read mc job logs %s: %v", podName, err) + } + return logs +} + +// podLogs fetches a pod's first-container logs via the clientset (the e2e +// framework's resource client does not expose a logs subresource). +func podLogs(ctx context.Context, cfg *envconf.Config, podName string) (string, error) { + cs := kubernetes.NewForConfigOrDie(cfg.Client().RESTConfig()) + rc, err := cs.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{}).Stream(ctx) + if err != nil { + return "", err + } + defer func() { _ = rc.Close() }() + var buf bytes.Buffer + if _, err := io.Copy(&buf, rc); err != nil { + return "", err + } + return buf.String(), nil +} + +// --- pod-wait helpers that catch FAILURE, not just success ----------------- + +func waitPodSucceeded(ctx context.Context, t *testing.T, cfg *envconf.Config, podName string) error { + t.Helper() + return wait.For(func(ctx context.Context) (bool, error) { + var p corev1.Pod + if err := cfg.Client().Resources().Get(ctx, podName, namespace, &p); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + switch p.Status.Phase { + case corev1.PodSucceeded: + return true, nil + case corev1.PodFailed: + logs, _ := podLogs(ctx, cfg, podName) + return false, fmt.Errorf("pod %s entered Failed phase; logs:\n%s", podName, logs) + } + // Surface image-pull / crash-loop wedges instead of waiting blind. + if reason := badContainerReason(&p); reason != "" { + logs, _ := podLogs(ctx, cfg, podName) + return false, fmt.Errorf("pod %s stuck: %s; logs:\n%s", podName, reason, logs) + } + return false, nil + }, wait.WithTimeout(2*time.Minute), wait.WithInterval(2*time.Second)) +} + +// badContainerReason returns a non-empty reason if any container is wedged in a +// known-fatal waiting/terminated state (CrashLoopBackOff, ImagePullBackOff, +// ErrImagePull, etc.), so waiters fail fast with a diagnosis. +func badContainerReason(p *corev1.Pod) string { + for _, cs := range p.Status.ContainerStatuses { + if w := cs.State.Waiting; w != nil { + switch w.Reason { + case "CrashLoopBackOff", "ImagePullBackOff", "ErrImagePull", "CreateContainerError", + "CreateContainerConfigError", "InvalidImageName": + return cs.Name + ": " + w.Reason + " (" + w.Message + ")" + } + } + } + return "" +} + +func waitPodGone(ctx context.Context, cfg *envconf.Config, podName string) error { + return wait.For(func(ctx context.Context) (bool, error) { + var p corev1.Pod + err := cfg.Client().Resources().Get(ctx, podName, namespace, &p) + if apierrors.IsNotFound(err) { + return true, nil + } + return false, nil + }, wait.WithTimeout(1*time.Minute), wait.WithInterval(2*time.Second)) +} + +func dumpPodTrouble(ctx context.Context, t *testing.T, cfg *envconf.Config, labels map[string]string) { + t.Helper() + var pods corev1.PodList + if err := cfg.Client().Resources().List(ctx, &pods); err != nil { + return + } + for i := range pods.Items { + p := &pods.Items[i] + match := true + for k, v := range labels { + if p.Labels[k] != v { + match = false + break + } + } + if !match { + continue + } + if reason := badContainerReason(p); reason != "" || p.Status.Phase == corev1.PodPending { + logs, _ := podLogs(ctx, cfg, p.Name) + t.Logf("pod %s phase=%s reason=%q logs:\n%s", p.Name, p.Status.Phase, reason, logs) + } + } +} + +// --- shared backup status wait + cleanup ------------------------------------ + +// waitBackupCompleted blocks until the named EtcdBackup reaches Completed and +// returns it, failing the test on a Failed phase or timeout. It never calls +// t.Fatalf from inside the poll closure (that is racey from a poller goroutine); +// it returns an error and fails the test from the calling goroutine instead. +func waitBackupCompleted(t *testing.T, cfg *envconf.Config, name string) ecv1alpha1.EtcdBackup { + t.Helper() + err := wait.For(func(ctx context.Context) (bool, error) { + var got ecv1alpha1.EtcdBackup + if err := cfg.Client().Resources().Get(ctx, name, namespace, &got); err != nil { + return false, err + } + switch got.Status.Phase { + case ecv1alpha1.BackupPhaseCompleted: + return true, nil + case ecv1alpha1.BackupPhaseFailed: + return false, fmt.Errorf("backup %q failed: %+v", name, got.Status.Conditions) + } + return false, nil + }, wait.WithTimeout(3*time.Minute), wait.WithInterval(2*time.Second)) + if err != nil { + t.Fatalf("waiting for backup %q completion: %v", name, err) + } + var done ecv1alpha1.EtcdBackup + if err := cfg.Client().Resources().Get(t.Context(), name, namespace, &done); err != nil { + t.Fatalf("get completed backup %q: %v", name, err) + } + return done +} + +// cleanupBackupWorkloads best-effort deletes the MinIO deployment/service, the +// bucket-bootstrap pod, the creds secret, and the named EtcdClusters so a +// re-run (or a sibling feature in the shared namespace) starts clean. +func cleanupBackupWorkloads( + ctx context.Context, t *testing.T, cfg *envconf.Config, minioName, credsSecret string, clusters []string, +) { + t.Helper() + res := cfg.Client().Resources() + _ = res.Delete(ctx, &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: minioName, Namespace: namespace}}) + _ = res.Delete(ctx, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: minioName, Namespace: namespace}}) + _ = res.Delete(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "mc-mkbucket-" + minioName, Namespace: namespace}}) + _ = res.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: credsSecret, Namespace: namespace}}) + for _, c := range clusters { + _ = res.Delete(ctx, &ecv1alpha1.EtcdCluster{ObjectMeta: metav1.ObjectMeta{Name: c, Namespace: namespace}}) + } +} diff --git a/test/e2e/restore_negative_test.go b/test/e2e/restore_negative_test.go new file mode 100644 index 00000000..08d20ea4 --- /dev/null +++ b/test/e2e/restore_negative_test.go @@ -0,0 +1,340 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// These adversarial restore tests exercise the operator's SAFETY guards — the +// code paths whose silent regression is catastrophic (clobbering a live +// cluster, hanging forever on a missing object, restoring a corrupt snapshot). +// All three now run live and green under ETCD_E2E_BACKUP=true: the populated- +// target and missing-object guards fire in the controller before/independently +// of the data path, and corrupt-snapshot rejection is surfaced by the init- +// container restore (etcd's snapshot-restore library fails its integrity check +// on a corrupt snapshot and the init-container exits non-zero), so it is a true +// integrity-rejection proof rather than an artifact of a broken restorer. + +// TestEtcdRestoreRejectsPopulatedTarget proves the empty-target guarantee +// (ensureEmptyTargetCluster/assertClusterEmpty): a restore that targets a +// cluster which already has a ready member must FAIL terminally and must NOT +// clobber that cluster's existing data. This is the guard against a restore +// silently overwriting a live production cluster, and it currently has no e2e +// coverage. +func TestEtcdRestoreRejectsPopulatedTarget(t *testing.T) { + requireBackupE2E(t) + + const ( + sourceCluster = "etcd-neg-pop-src" + targetCluster = "etcd-neg-pop-tgt" + backupName = "etcd-neg-pop-snap" + restoreName = "etcd-neg-pop-restore" + markerKey = "neg-pop/precious" + markerVal = "do-not-clobber" + ) + backend := newMinIOBackend("minio-neg-pop", "etcd-backups", "neg-pop-creds") + + feature := features.New("etcd-restore-rejects-populated-target") + + feature.Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + backend.deploy(ctx, t, cfg) + backend.bootstrapBucket(ctx, t, cfg) + + // A real, completed backup so resolveSource succeeds and the restore + // reaches the empty-target guard rather than failing earlier. + createBackupTestCluster(ctx, t, cfg, sourceCluster, 1) + waitStatefulSetReady(t, cfg, sourceCluster, 1) + seedRichKeyspace(t, cfg, sourceCluster+"-0") + mkBackup(ctx, t, cfg, backupName, sourceCluster, backend.destination("e2e")) + waitBackupCompleted(t, cfg, backupName) + + // A POPULATED target cluster: ready member + a precious marker key. + createBackupTestCluster(ctx, t, cfg, targetCluster, 1) + waitStatefulSetReady(t, cfg, targetCluster, 1) + if _, stderr, err := backupExecInPod(t, cfg, targetCluster+"-0", + []string{"etcdctl", "put", markerKey, markerVal}); err != nil { + t.Fatalf("seed target marker: %v (stderr: %s)", err, stderr) + } + return ctx + }) + + feature.Assess("restore into a populated cluster is rejected and the data is preserved", func( + ctx context.Context, t *testing.T, cfg *envconf.Config, + ) context.Context { + restore := &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: restoreName, Namespace: namespace}, + Spec: ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{BackupRef: &ecv1alpha1.BackupReference{Name: backupName}}, + Target: ecv1alpha1.RestoreTarget{Name: targetCluster, Size: 1, Version: "v3.6.1"}, + }, + } + if err := cfg.Client().Resources().Create(ctx, restore); err != nil { + t.Fatalf("create EtcdRestore: %v", err) + } + + got := waitRestoreFailed(t, cfg, restoreName) + if msg := failedConditionMessage(got); !strings.Contains(strings.ToLower(msg), "non-empty") && + !strings.Contains(strings.ToLower(msg), "ready member") { + t.Errorf("expected a non-empty-target rejection reason, got: %q", msg) + } + + // The precious data must be intact — the rejected restore must not have + // clobbered the live cluster. + out, _, err := backupExecInPod(t, cfg, targetCluster+"-0", + []string{"etcdctl", "get", markerKey, "--print-value-only"}) + if err != nil { + t.Fatalf("re-read marker after rejected restore: %v", err) + } + if strings.TrimSpace(out) != markerVal { + t.Errorf("marker key clobbered: got %q want %q", strings.TrimSpace(out), markerVal) + } + t.Logf("populated-target restore correctly rejected; existing data preserved") + return ctx + }) + + feature.Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + backend.cleanup(ctx, t, cfg) + deleteClusters(ctx, cfg, sourceCluster, targetCluster) + _ = cfg.Client().Resources().Delete(ctx, &ecv1alpha1.EtcdBackup{ + ObjectMeta: metav1.ObjectMeta{Name: backupName, Namespace: namespace}}) + _ = cfg.Client().Resources().Delete(ctx, &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: restoreName, Namespace: namespace}}) + return ctx + }) + + _ = testEnv.Test(t, feature.Feature()) +} + +// TestEtcdRestoreMissingObjectFailsCleanly proves a restore that references an +// object which does not exist in the bucket fails TERMINALLY and PROMPTLY +// (ErrNotFound surfaced as a clean failure), rather than retrying or hanging +// until a timeout. The test's own bounded wait is the assertion: if the restore +// hung, waitRestoreFailed would time out and fail. This fires at the Download +// step, before the restore data-path, so it is live-green today. +func TestEtcdRestoreMissingObjectFailsCleanly(t *testing.T) { + requireBackupE2E(t) + + const ( + targetCluster = "etcd-neg-missing-tgt" + restoreName = "etcd-neg-missing-restore" + ) + backend := newMinIOBackend("minio-neg-missing", "etcd-backups", "neg-missing-creds") + + feature := features.New("etcd-restore-missing-object-clean-fail") + + feature.Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + backend.deploy(ctx, t, cfg) + backend.bootstrapBucket(ctx, t, cfg) + return ctx + }) + + feature.Assess("restore from a non-existent object fails cleanly within a bounded deadline", func( + ctx context.Context, t *testing.T, cfg *envconf.Config, + ) context.Context { + dst := backend.destination("e2e") + restore := &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: restoreName, Namespace: namespace}, + Spec: ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{ + Location: &ecv1alpha1.SnapshotLocation{ + Destination: dst, + Key: "this/object/does-not-exist.db", + }, + }, + Target: ecv1alpha1.RestoreTarget{Name: targetCluster, Size: 1, Version: "v3.6.1"}, + }, + } + if err := cfg.Client().Resources().Create(ctx, restore); err != nil { + t.Fatalf("create EtcdRestore: %v", err) + } + + got := waitRestoreFailed(t, cfg, restoreName) + if msg := failedConditionMessage(got); !strings.Contains(strings.ToLower(msg), "download") && + !strings.Contains(strings.ToLower(msg), "not found") { + t.Errorf("expected a download/not-found failure reason, got: %q", msg) + } + t.Logf("missing-object restore failed cleanly: %q", failedConditionMessage(got)) + return ctx + }) + + feature.Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + backend.cleanup(ctx, t, cfg) + deleteClusters(ctx, cfg, targetCluster) + _ = cfg.Client().Resources().Delete(ctx, &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: restoreName, Namespace: namespace}}) + return ctx + }) + + _ = testEnv.Test(t, feature.Feature()) +} + +// TestEtcdRestoreRejectsCorruptSnapshot uploads a deliberately corrupt object +// (64 KiB of non-bbolt bytes) and asserts the restore fails rather than +// bootstrapping a garbage cluster. +// +// With the init-container restore data-path, the restore init-container runs +// etcd's snapshot-restore library in-process, which fails its integrity (hash) +// check on a corrupt snapshot and exits non-zero; the controller observes the +// wedged init-container and marks the restore Failed. So this is now a true +// integrity-rejection proof and runs live under ETCD_E2E_BACKUP=true. +func TestEtcdRestoreRejectsCorruptSnapshot(t *testing.T) { + requireBackupE2E(t) + + const ( + targetCluster = "etcd-neg-corrupt-tgt" + restoreName = "etcd-neg-corrupt-restore" + corruptKey = "e2e/corrupt/garbage.db" + ) + backend := newMinIOBackend("minio-neg-corrupt", "etcd-backups", "neg-corrupt-creds") + + feature := features.New("etcd-restore-rejects-corrupt-snapshot") + + feature.Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + backend.deploy(ctx, t, cfg) + backend.bootstrapBucket(ctx, t, cfg) + // Upload 64 KiB of non-snapshot bytes under the corrupt key (above the + // 16 KiB floor, so size alone cannot save a naive check). + uploadCorruptMinIOObject(t, cfg, backend, corruptKey, 64*1024) + return ctx + }) + + feature.Assess("restore from a corrupt object is rejected", func( + ctx context.Context, t *testing.T, cfg *envconf.Config, + ) context.Context { + restore := &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: restoreName, Namespace: namespace}, + Spec: ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{ + Location: &ecv1alpha1.SnapshotLocation{ + Destination: backend.destination(""), + Key: corruptKey, + }, + }, + Target: ecv1alpha1.RestoreTarget{Name: targetCluster, Size: 1, Version: "v3.6.1"}, + }, + } + if err := cfg.Client().Resources().Create(ctx, restore); err != nil { + t.Fatalf("create EtcdRestore: %v", err) + } + got := waitRestoreFailed(t, cfg, restoreName) + t.Logf("corrupt-snapshot restore failed as required: %q", failedConditionMessage(got)) + return ctx + }) + + feature.Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + backend.cleanup(ctx, t, cfg) + deleteClusters(ctx, cfg, targetCluster) + _ = cfg.Client().Resources().Delete(ctx, &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: restoreName, Namespace: namespace}}) + return ctx + }) + + _ = testEnv.Test(t, feature.Feature()) +} + +// --- shared negative-path helpers ------------------------------------------ + +func mkBackup(ctx context.Context, t *testing.T, cfg *envconf.Config, name, clusterRef string, dst ecv1alpha1.BackupDestination) { + t.Helper() + backup := &ecv1alpha1.EtcdBackup{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: ecv1alpha1.EtcdBackupSpec{ClusterRef: clusterRef, Destination: dst}, + } + if err := cfg.Client().Resources().Create(ctx, backup); err != nil { + t.Fatalf("create EtcdBackup %q: %v", name, err) + } +} + +// waitRestoreFailed blocks until the named EtcdRestore reaches Failed and +// returns it. A restore that reaches Completed instead, or never reaches a +// terminal phase before the bounded deadline (e.g. it hung), fails the test — +// so this helper doubles as the "fails promptly, does not hang" assertion. +func waitRestoreFailed(t *testing.T, cfg *envconf.Config, name string) ecv1alpha1.EtcdRestore { + t.Helper() + err := wait.For(func(ctx context.Context) (bool, error) { + var got ecv1alpha1.EtcdRestore + if err := cfg.Client().Resources().Get(ctx, name, namespace, &got); err != nil { + return false, err + } + switch got.Status.Phase { + case ecv1alpha1.RestorePhaseFailed: + return true, nil + case ecv1alpha1.RestorePhaseCompleted: + return false, fmt.Errorf("restore %q unexpectedly Completed; it should have been rejected", name) + } + return false, nil + }, wait.WithTimeout(3*time.Minute), wait.WithInterval(2*time.Second)) + if err != nil { + t.Fatalf("waiting for restore %q to fail: %v", name, err) + } + var done ecv1alpha1.EtcdRestore + if err := cfg.Client().Resources().Get(t.Context(), name, namespace, &done); err != nil { + t.Fatalf("get failed restore %q: %v", name, err) + } + return done +} + +// failedConditionMessage returns the message of the Succeeded=False condition, +// where the controller records why a restore failed. +func failedConditionMessage(restore ecv1alpha1.EtcdRestore) string { + for _, c := range restore.Status.Conditions { + if c.Type == ecv1alpha1.RestoreConditionSucceeded { + return c.Message + } + } + return "" +} + +func deleteClusters(ctx context.Context, cfg *envconf.Config, names ...string) { + res := cfg.Client().Resources() + for _, n := range names { + _ = res.Delete(ctx, &ecv1alpha1.EtcdCluster{ObjectMeta: metav1.ObjectMeta{Name: n, Namespace: namespace}}) + } +} + +// uploadCorruptMinIOObject writes `size` bytes of non-snapshot data to the +// backend under key, via a one-shot mc pod, so a restore can be pointed at a +// deliberately corrupt object. +func uploadCorruptMinIOObject(t *testing.T, cfg *envconf.Config, backend *minioBackend, key string, size int) { + t.Helper() + // Generate `size` bytes from /dev/zero (the mc image has a shell + dd), then + // pipe into mc. The content is not a valid bbolt DB, so an integrity-aware + // restore must reject it. + script := fmt.Sprintf( + "mc alias set m http://%s:9000 %s %s >/dev/null && "+ + "head -c %d /dev/zero | mc pipe m/%s/%s", + backend.instance, backend.accessKey, backend.secretKey, + size, backend.bucket, key) + _ = runMCJob(t, cfg, "mc-corrupt-"+backend.instance, script) + // Sanity: confirm the object is present at the expected size, independent of + // the upload's own success report. + if got := backend.statObjectSize(t, cfg, key); got != int64(size) { + t.Fatalf("corrupt object size %d != expected %d", got, size) + } +} diff --git a/test/e2e/restore_test.go b/test/e2e/restore_test.go new file mode 100644 index 00000000..25883678 --- /dev/null +++ b/test/e2e/restore_test.go @@ -0,0 +1,187 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/e2e-framework/klient/wait" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + ecv1alpha1 "go.etcd.io/etcd-operator/api/v1alpha1" +) + +// TestEtcdRestoreFromBackupS3 / ...GCS exercise the FULL backup -> restore round +// trip end-to-end against an in-cluster object store (MinIO for S3, fake-gcs- +// server for GCS), so they require NO real cloud credentials. +// +// They are gated ONLY behind ETCD_E2E_BACKUP=true (which provisions the +// backends). The restore data-path is now a real init-container restore (the +// operator image's `restore-localize` subcommand downloads the snapshot and runs +// etcd's snapshot-restore library in-process into the genesis member's data dir +// before etcd starts), so the round-trip runs live in the default backup matrix. +// +// Flow: +// 1. Deploy the backend + bucket (+ creds for S3). +// 2. Create a source EtcdCluster, seed a distinctive keyspace, back it up; wait +// for the EtcdBackup to reach Completed. +// 3. Create an EtcdRestore referencing that backup, targeting a NEW cluster. +// 4. Wait for the EtcdRestore to reach Completed, then assert the ENTIRE seeded +// keyspace is readable from the restored member with exact values, and a key +// that was never in the snapshot is absent (negative control). +func TestEtcdRestoreFromBackupS3(t *testing.T) { + requireBackupE2E(t) + runRestoreRoundTrip(t, + newMinIOBackend("minio-restore-e2e", "etcd-restores", "restore-e2e-creds"), + "etcd-restore-s3", + "etcd-restore-src-e2e", "etcd-restore-dst-e2e", "etcd-restore-e2e-snap", "etcd-restore-e2e") +} + +func TestEtcdRestoreFromBackupGCS(t *testing.T) { + requireBackupE2E(t) + runRestoreRoundTrip(t, + newFakeGCSBackend("fake-gcs-restore-e2e", "etcd-restores"), + "etcd-restore-gcs", + "etcd-restore-gcs-src", "etcd-restore-gcs-dst", "etcd-restore-gcs-snap", "etcd-restore-gcs") +} + +// runRestoreRoundTrip drives one provider through the full backup->restore->byte- +// identical-readback proof. +func runRestoreRoundTrip( + t *testing.T, backend backupBackend, featureName, sourceCluster, restoredCluster, backupName, restoreName string, +) { + const absentKey = "restore-e2e/never-in-snapshot" + + feature := features.New(featureName) + + feature.Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + backend.deploy(ctx, t, cfg) + backend.bootstrapBucket(ctx, t, cfg) + createBackupTestCluster(ctx, t, cfg, sourceCluster, 1) + waitStatefulSetReady(t, cfg, sourceCluster, 1) + seedKeyspace(t, cfg, sourceCluster+"-0", backupSeedKeyCount) + + backup := &ecv1alpha1.EtcdBackup{ + ObjectMeta: metav1.ObjectMeta{Name: backupName, Namespace: namespace}, + Spec: ecv1alpha1.EtcdBackupSpec{ + ClusterRef: sourceCluster, + Destination: backend.destination("e2e"), + }, + } + if err := cfg.Client().Resources().Create(ctx, backup); err != nil { + t.Fatalf("create EtcdBackup: %v", err) + } + waitBackupCompleted(t, cfg, backupName) + return ctx + }) + + feature.Assess("restore reaches Completed and the full keyspace round-trips", func( + ctx context.Context, t *testing.T, cfg *envconf.Config, + ) context.Context { + restore := &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: restoreName, Namespace: namespace}, + Spec: ecv1alpha1.EtcdRestoreSpec{ + Source: ecv1alpha1.SnapshotSource{ + BackupRef: &ecv1alpha1.BackupReference{Name: backupName}, + }, + Target: ecv1alpha1.RestoreTarget{ + Name: restoredCluster, + Size: 1, + Version: "v3.6.1", + }, + }, + } + if err := cfg.Client().Resources().Create(ctx, restore); err != nil { + t.Fatalf("create EtcdRestore: %v", err) + } + + // Wait for terminal restore phase without t.Fatalf inside the poller. + err := wait.For(func(ctx context.Context) (bool, error) { + var got ecv1alpha1.EtcdRestore + if err := cfg.Client().Resources().Get(ctx, restoreName, namespace, &got); err != nil { + return false, err + } + switch got.Status.Phase { + case ecv1alpha1.RestorePhaseCompleted: + return true, nil + case ecv1alpha1.RestorePhaseFailed: + return false, fmt.Errorf("restore failed: %+v", got.Status.Conditions) + } + return false, nil + }, wait.WithTimeout(3*time.Minute), wait.WithInterval(2*time.Second)) + if err != nil { + t.Fatalf("waiting for restore completion: %v", err) + } + + waitStatefulSetReady(t, cfg, restoredCluster, 1) + + // Positive proof: the ENTIRE seeded keyspace must be present with exact + // values. Poll, so a member that boots and then catches up does not race + // a single-shot read; a never-restored member fails with "data never + // appeared" rather than an opaque mismatch. + pod := restoredCluster + "-0" + if err := wait.For(func(ctx context.Context) (bool, error) { + for i := 0; i < backupSeedKeyCount; i++ { + k, want := seedKV(i) + out, _, err := backupExecInPod(t, cfg, pod, + []string{"etcdctl", "get", k, "--print-value-only"}) + if err != nil { + return false, nil // member may still be coming up + } + if strings.TrimSpace(out) != want { + return false, nil + } + } + return true, nil + }, wait.WithTimeout(2*time.Minute), wait.WithInterval(3*time.Second)); err != nil { + t.Fatalf("restored cluster did not serve the full seeded keyspace: %v", err) + } + + // Negative control: a key that was never written must be absent. This + // rules out the vacuous pass where the member booted an empty dir and the + // "presence" check matched diagnostic output. + out, _, err := backupExecInPod(t, cfg, pod, + []string{"etcdctl", "get", absentKey, "--print-value-only"}) + if err != nil { + t.Fatalf("negative-control read failed: %v", err) + } + if strings.TrimSpace(out) != "" { + t.Errorf("negative control: key %q should be absent but returned %q", absentKey, out) + } + + t.Logf("restore verified: %d keys round-tripped, negative control absent", backupSeedKeyCount) + return ctx + }) + + feature.Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + backend.cleanup(ctx, t, cfg) + deleteClusters(ctx, cfg, sourceCluster, restoredCluster) + _ = cfg.Client().Resources().Delete(ctx, &ecv1alpha1.EtcdBackup{ + ObjectMeta: metav1.ObjectMeta{Name: backupName, Namespace: namespace}}) + _ = cfg.Client().Resources().Delete(ctx, &ecv1alpha1.EtcdRestore{ + ObjectMeta: metav1.ObjectMeta{Name: restoreName, Namespace: namespace}}) + return ctx + }) + + _ = testEnv.Test(t, feature.Feature()) +}