K8SPG-1012 stanza timing fix

e2e-tests/run-pr.csv

@@ -33,3 +33,6 @@ telemetry-transfer
 upgrade-consistency
 upgrade-minor
 users
+migration-from-crunchy-standby
+migration-from-crunchy-pv
+migration-from-crunchy-backup-restore

e2e-tests/tests/migration-from-crunchy-backup-restore/10-assert.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+commands:
+  - script: |-
+      set -o errexit
+      kubectl get configmap -n "${NAMESPACE}" 10-second-batch-written

e2e-tests/tests/migration-from-crunchy-backup-restore/10-write-more-data.yaml

@@ -0,0 +1,38 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      primary=$(kubectl get pod -n "${NAMESPACE}" \
+        -l postgres-operator.crunchydata.com/cluster=percona-restored,postgres-operator.crunchydata.com/role=primary \
+        -o jsonpath='{.items[0].metadata.name}')
+
+      if [ -z "${primary}" ]; then
+        echo "ERROR: primary pod not found"
+        exit 1
+      fi
+
+      # Insert the second batch of rows that must survive the upcoming PITR.
+      # The PITR target is captured in step 12 AFTER the step 11 backup
+      # completes, so the backup's start time is guaranteed to be before the
+      # target and pgBackRest can use it for the restore.
+      kubectl exec -n "${NAMESPACE}" "${primary}" -c database -- \
+        psql -d migrationtest -c "
+          INSERT INTO migration_data VALUES
+            (5, 'second-batch-one'),
+            (6, 'second-batch-two'),
+            (7, 'second-batch-three');
+        "
+
+      # Force a WAL switch so the inserted rows reach the archive before the
+      # step 11 backup starts.
+      kubectl exec -n "${NAMESPACE}" "${primary}" -c database -- \
+        psql -q -c "SELECT pg_switch_wal();"
+
+      kubectl create configmap -n "${NAMESPACE}" 10-second-batch-written \
+        --from-literal=rows="5,6,7"
+    timeout: 120

e2e-tests/tests/migration-from-crunchy-backup-restore/11-assert.yaml

@@ -0,0 +1,29 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 560
+---
+kind: Job
+apiVersion: batch/v1
+metadata:
+  annotations:
+    postgres-operator.crunchydata.com/pgbackrest-backup: second-post-restore-backup
+  labels:
+    postgres-operator.crunchydata.com/pgbackrest-backup: manual
+    postgres-operator.crunchydata.com/pgbackrest-repo: repo1
+  ownerReferences:
+    - apiVersion: pgv2.percona.com/v2
+      kind: PerconaPGBackup
+      controller: true
+      blockOwnerDeletion: true
+status:
+  succeeded: 1
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  name: second-post-restore-backup
+spec:
+  pgCluster: percona-restored
+  repoName: repo1
+status:
+  state: Succeeded

e2e-tests/tests/migration-from-crunchy-backup-restore/11-backup.yaml

@@ -0,0 +1,9 @@
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGBackup
+metadata:
+  name: second-post-restore-backup
+spec:
+  pgCluster: percona-restored
+  repoName: repo1
+  options:
+    - --type=full

e2e-tests/tests/migration-from-crunchy-backup-restore/12-assert.yaml

@@ -0,0 +1,60 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGRestore
+metadata:
+  name: second-pitr-restore
+status:
+  state: Succeeded
+---
+# One StatefulSet per pod; readyReplicas=1 each.  Aggregate validated below.
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: percona-restored
+    postgres-operator.crunchydata.com/data: postgres
+    postgres-operator.crunchydata.com/instance-set: instance1
+  ownerReferences:
+    - apiVersion: upstream.pgv2.percona.com/v1beta1
+      kind: PostgresCluster
+      name: percona-restored
+      controller: true
+      blockOwnerDeletion: true
+status:
+  availableReplicas: 1
+  readyReplicas: 1
+  replicas: 1
+  updatedReplicas: 1
+---
+apiVersion: upstream.pgv2.percona.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: percona-restored
+status:
+  instances:
+    - name: instance1
+      readyReplicas: 3
+      replicas: 3
+      updatedReplicas: 3
+  pgbackrest:
+    restore:
+      finished: true
+      id: second-pitr-restore
+      succeeded: 1
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: percona-restored
+status:
+  postgres:
+    instances:
+      - name: instance1
+        ready: 3
+        size: 3
+    ready: 3
+    size: 3
+  state: ready

e2e-tests/tests/migration-from-crunchy-backup-restore/12-restore.yaml

@@ -0,0 +1,73 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      # No scale-down needed: the cluster is on TL3 and the stanza existed
+      # before this timeline.  When postgres promotes TL3→TL4 after the PITR,
+      # 00000004.history is pushed by the async archiver immediately (stanza
+      # exists), so pg_rewind on the two replicas can trace the ancestry and
+      # rejoin without the scale-to-1 workaround required for step 08.
+
+      primary=$(kubectl get pod -n "${NAMESPACE}" \
+        -l postgres-operator.crunchydata.com/cluster=percona-restored,postgres-operator.crunchydata.com/role=primary \
+        -o jsonpath='{.items[0].metadata.name}')
+
+      if [ -z "${primary}" ]; then
+        echo "ERROR: primary pod not found"
+        exit 1
+      fi
+
+      # Capture the PITR target NOW — after the step 11 backup has completed.
+      # This guarantees the backup start time < target < after-target rows,
+      # which is the only valid ordering for a time-based pgBackRest restore.
+      pitr_target=$(kubectl exec -n "${NAMESPACE}" "${primary}" -c database -- \
+        psql -q -t -c "SELECT to_char(clock_timestamp(), 'YYYY-MM-DD HH24:MI:SS')" \
+        | xargs)
+      echo "PITR target: ${pitr_target}"
+
+      # Force a WAL switch so the LSN at pitr_target is flushed to the archive
+      # before we write the rows that must be absent after restore.
+      kubectl exec -n "${NAMESPACE}" "${primary}" -c database -- \
+        psql -q -c "SELECT pg_switch_wal();"
+
+      # Write rows that must be absent after the PITR restore.
+      kubectl exec -n "${NAMESPACE}" "${primary}" -c database -- \
+        psql -d migrationtest -c "
+          INSERT INTO migration_data VALUES
+            (8, 'after-pitr-target-one'),
+            (9, 'after-pitr-target-two');
+        "
+
+      # Resolve the latest full backup label (step 11's second-post-restore-
+      # backup).  Because that backup completed before pitr_target was
+      # captured above, --set is safe: backup-start < pitr_target.
+      backup_label=$(kubectl -n "${NAMESPACE}" exec "${primary}" -- \
+        pgbackrest info --output json --log-level-console=info --stanza=db \
+        | jq -r '[.[] | .backup[] | select(.type == "full") | select(.database.["repo-key"] == 1)][-1].label')
+
+      if [ -z "${backup_label}" ] || [ "${backup_label}" = "null" ]; then
+        echo "ERROR: could not determine latest full backup label"
+        exit 1
+      fi
+      echo "Restoring from backup: ${backup_label}"
+
+      cat <<EOF | kubectl -n "${NAMESPACE}" apply -f -
+      apiVersion: pgv2.percona.com/v2
+      kind: PerconaPGRestore
+      metadata:
+        name: second-pitr-restore
+      spec:
+        pgCluster: percona-restored
+        repoName: repo1
+        options:
+        - --set=${backup_label}
+        - --type=time
+        - --target="${pitr_target}"
+        - --target-timeline=current
+      EOF
+    timeout: 300

e2e-tests/tests/migration-from-crunchy-backup-restore/13-assert.yaml

@@ -0,0 +1,26 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 30
+commands:
+  - script: |-
+      set -o errexit
+
+      data=$(kubectl get configmap 13-restored-data -n "${NAMESPACE}" \
+        -o jsonpath='{.data.data}')
+
+      for expected in "row-one" "row-two" "row-three" "written-after-restore" \
+                      "second-batch-one" "second-batch-two" "second-batch-three"; do
+        echo "${data}" | grep -q "${expected}" || {
+          echo "ERROR: '${expected}' missing after second PITR restore. Got: ${data}"
+          exit 1
+        }
+      done
+
+      for absent in "after-pitr-target-one" "after-pitr-target-two"; do
+        if echo "${data}" | grep -q "${absent}"; then
+          echo "ERROR: '${absent}' present after second PITR restore (should be absent). Got: ${data}"
+          exit 1
+        fi
+      done
+
+      echo "All expected rows present, post-target rows correctly absent"

e2e-tests/tests/migration-from-crunchy-backup-restore/13-verify-after-restore.yaml

@@ -0,0 +1,46 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      primary=$(kubectl get pod -n "${NAMESPACE}" \
+        -l postgres-operator.crunchydata.com/cluster=percona-restored,postgres-operator.crunchydata.com/role=primary \
+        -o jsonpath='{.items[0].metadata.name}')
+
+      if [ -z "${primary}" ]; then
+        echo "ERROR: primary pod not found after second PITR restore"
+        exit 1
+      fi
+
+      data=$(kubectl exec -n "${NAMESPACE}" "${primary}" -c database -- bash -c \
+        "psql -q -t -d migrationtest -c 'SELECT id, value FROM migration_data ORDER BY id;'")
+
+      echo "Data after second PITR restore:"
+      echo "${data}"
+
+      kubectl create configmap -n "${NAMESPACE}" 13-restored-data \
+        --from-literal=data="${data}"
+
+      # Original rows plus the second batch must all be present.
+      for expected in "row-one" "row-two" "row-three" "written-after-restore" \
+                      "second-batch-one" "second-batch-two" "second-batch-three"; do
+        echo "${data}" | grep -q "${expected}" || {
+          echo "ERROR: '${expected}' missing after second PITR restore"
+          exit 1
+        }
+      done
+
+      # Rows written after the PITR target must be absent.
+      for absent in "after-pitr-target-one" "after-pitr-target-two"; do
+        if echo "${data}" | grep -q "${absent}"; then
+          echo "ERROR: '${absent}' present after second PITR restore (should be absent)"
+          exit 1
+        fi
+      done
+
+      echo "PASS: all expected rows present, post-target rows correctly absent"
+    timeout: 120

e2e-tests/tests/upgrade-minor/99-remove-cluster-gracefully.yaml

@@ -12,6 +12,8 @@ commands:
 
       source ../../functions
 
+      kubectl -n ${NAMESPACE} patch postgrescluster upgrade-minor --type=merge -p='{"metadata":{"finalizers":[]}}' 2>/dev/null || true
+      kubectl -n ${NAMESPACE} delete postgrescluster upgrade-minor --ignore-not-found
       remove_all_finalizers
       check_operator_panic
       destroy_operator

internal/controller/postgrescluster/instance.go

@@ -1100,7 +1100,7 @@ func (r *Reconciler) scaleUpInstances(
 		next := naming.GenerateInstance(cluster, set)
 		// if there are any available instance names (as determined by observing any PVCs for the
 		// instance set that are not currently associated with an instance, e.g. in the event the
-		// instance STS was deleted), then reuse them instead of generating a new name
+		// instance STS was deleted), then reuse them instead of generating a new name.
 		if len(availableInstanceNames) > 0 {
 			next.Name = availableInstanceNames[0]
 			availableInstanceNames = availableInstanceNames[1:]

internal/controller/postgrescluster/pgbackrest.go

@@ -3037,6 +3037,21 @@ func (r *Reconciler) reconcileStanzaCreate(ctx context.Context,
 	r.Recorder.Event(postgresCluster, corev1.EventTypeNormal, EventStanzasCreated,
 		"pgBackRest stanza creation completed successfully")
 
+	// Re-push any timeline history files stranded by the async-archiver race:
+	// postgres archives 00000002.history during bootstrap promotion before the
+	// stanza exists; pgBackRest drops it silently (error 103) and postgres
+	// never retries.  Without it pg_rewind fails on replicas after PITR.
+	log := logging.FromContext(ctx)
+	historyOut, historyErr := pgbackrest.Executor(exec).ArchivePushHistoryFiles(ctx)
+	if historyErr != nil {
+		r.Recorder.Event(postgresCluster, corev1.EventTypeWarning,
+			"ArchivePushHistoryFilesFailed", historyErr.Error())
+		log.Error(historyErr, "timeline history file recovery failed",
+			"pod", writableInstanceName, "output", historyOut)
+	} else if historyOut != "" {
+		log.Info("timeline history file recovery", "output", historyOut)
+	}
+
 	// if no errors then stanza(s) created successfully
 	for i := range postgresCluster.Status.PGBackRest.Repos {
 		postgresCluster.Status.PGBackRest.Repos[i].StanzaCreated = true