From c0aa2dcaeb683c3827806c351a6fedc30ad809dc Mon Sep 17 00:00:00 2001 From: Mazdak Badakhshan Date: Thu, 23 Jul 2026 12:10:51 +0200 Subject: [PATCH] feat(datastore,postgres): add logical logical replication based watch api for Postgres --- CHANGELOG.md | 1 + docs/spicedb.md | 3 + go.mod | 1 + go.sum | 2 + internal/datastore/postgres/gc.go | 32 + internal/datastore/postgres/lsn_ledger.go | 1715 ++++++++ .../datastore/postgres/lsn_ledger_test.go | 344 ++ ...zz_migration.0026_add_commit_lsn_ledger.go | 87 + internal/datastore/postgres/options.go | 131 + internal/datastore/postgres/pgoutput_row.go | 123 + .../datastore/postgres/pgoutput_row_test.go | 231 ++ internal/datastore/postgres/postgres.go | 69 +- .../postgres/postgres_shared_test.go | 5 + internal/datastore/postgres/revisions.go | 127 +- .../postgres/revisions_position_test.go | 217 + internal/datastore/postgres/revisions_test.go | 180 + internal/datastore/postgres/schema/schema.go | 16 + internal/datastore/postgres/snapshot.go | 10 +- internal/datastore/postgres/watch.go | 4 + internal/datastore/postgres/watch_cursor.go | 1011 +++++ .../datastore/postgres/watch_cursor_test.go | 3606 +++++++++++++++++ .../config/postgres-logical-replication.conf | 10 + internal/testserver/datastore/postgres.go | 40 +- pkg/cmd/datastore/datastore.go | 10 +- pkg/cmd/datastore/zz_generated.options.go | 9 + 25 files changed, 7965 insertions(+), 19 deletions(-) create mode 100644 internal/datastore/postgres/lsn_ledger.go create mode 100644 internal/datastore/postgres/lsn_ledger_test.go create mode 100644 internal/datastore/postgres/migrations/zz_migration.0026_add_commit_lsn_ledger.go create mode 100644 internal/datastore/postgres/pgoutput_row.go create mode 100644 internal/datastore/postgres/pgoutput_row_test.go create mode 100644 internal/datastore/postgres/revisions_position_test.go create mode 100644 internal/datastore/postgres/watch_cursor.go create mode 100644 internal/datastore/postgres/watch_cursor_test.go create mode 100644 internal/testserver/datastore/config/postgres-logical-replication.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index e1adddb94c..79a1053bcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - New metric `check_permissionship_total` for CheckPermission and CheckBulkPermissions that counts the number of requests that returned HAS_PERMISSION. Also, `write_relationships_updates` also includes BulkImport calls (https://github.com/authzed/spicedb/pull/3240) +- Postgres: opt-in logical-replication-based Watch API implementation (`--datastore-watch-logical-replication-enabled`) that streams changes from the WAL in exact commit order through a temporary replication slot, instead of polling the transactions table. Watch revisions gain a byte-sortable commit-LSN component (embedded alongside the usual snapshot, so ZedTokens remain fully usable across the rest of the API), watch no longer requires `track_commit_timestamp=on` in this mode, and `EmitImmediatelyStrategy` is now supported on Postgres. Requires `wal_level=logical` and the `REPLICATION` privilege; the polling watch remains the default (https://github.com/authzed/spicedb/pull/3245) ### Changed - Schema: reads inside write transactions now use a cheap hash-only lookup (`schema_revision`) to check the cache before loading the full schema blob, reducing DB round-trips on cache hits (https://github.com/authzed/spicedb/pull/3160) diff --git a/docs/spicedb.md b/docs/spicedb.md index 3efc6bd7c5..7dd34ff676 100644 --- a/docs/spicedb.md +++ b/docs/spicedb.md @@ -96,6 +96,7 @@ spicedb datastore gc [flags] --datastore-gc-max-operation-time duration maximum amount of time a garbage collection pass can operate before timing out (Postgres driver only) (default 1m0s) --datastore-gc-window duration amount of time before revisions are garbage collected (default 24h0m0s) --datastore-include-query-parameters-in-traces include query parameters in traces (Postgres and CockroachDB drivers only) + --datastore-logical-replication-watch serve the Watch API in true commit order, keyed by commit LSNs recorded from a logical replication slot; requires wal_level=logical and the REPLICATION privilege (Postgres driver only) --datastore-max-tx-retries int number of times a retriable transaction should be retried (default 10) --datastore-migration-phase string datastore-specific flag that should be used to signal to a datastore which phase of a multi-step migration it is in --datastore-mysql-table-prefix string prefix to add to the name of all SpiceDB database tables @@ -241,6 +242,7 @@ spicedb datastore repair [flags] --datastore-gc-max-operation-time duration maximum amount of time a garbage collection pass can operate before timing out (Postgres driver only) (default 1m0s) --datastore-gc-window duration amount of time before revisions are garbage collected (default 24h0m0s) --datastore-include-query-parameters-in-traces include query parameters in traces (Postgres and CockroachDB drivers only) + --datastore-logical-replication-watch serve the Watch API in true commit order, keyed by commit LSNs recorded from a logical replication slot; requires wal_level=logical and the REPLICATION privilege (Postgres driver only) --datastore-max-tx-retries int number of times a retriable transaction should be retried (default 10) --datastore-migration-phase string datastore-specific flag that should be used to signal to a datastore which phase of a multi-step migration it is in --datastore-mysql-table-prefix string prefix to add to the name of all SpiceDB database tables @@ -419,6 +421,7 @@ spicedb serve [flags] --datastore-gc-max-operation-time duration maximum amount of time a garbage collection pass can operate before timing out (Postgres driver only) (default 1m0s) --datastore-gc-window duration amount of time before revisions are garbage collected (default 24h0m0s) --datastore-include-query-parameters-in-traces include query parameters in traces (Postgres and CockroachDB drivers only) + --datastore-logical-replication-watch serve the Watch API in true commit order, keyed by commit LSNs recorded from a logical replication slot; requires wal_level=logical and the REPLICATION privilege (Postgres driver only) --datastore-max-tx-retries int number of times a retriable transaction should be retried (default 10) --datastore-migration-phase string datastore-specific flag that should be used to signal to a datastore which phase of a multi-step migration it is in --datastore-mysql-table-prefix string prefix to add to the name of all SpiceDB database tables diff --git a/go.mod b/go.mod index 92e382c31a..d29dee1619 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 github.com/hashicorp/go-memdb v1.3.5 github.com/jackc/pgio v1.0.0 + github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb github.com/jackc/pgx/v5 v5.9.2 github.com/jeroenrinzema/psql-wire v0.17.0 diff --git a/go.sum b/go.sum index 82263f5a9f..5b06cd9d5b 100644 --- a/go.sum +++ b/go.sum @@ -303,6 +303,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 h1:+PJCokZ2BhyDKlncScmiNzBwqOx+yH1i8xRlWN/wn6A= +github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510/go.mod h1:UzTJ5Jjuf4O9hYWW+HYVwVldYz9J7CaePW0iuNJkrPQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= diff --git a/internal/datastore/postgres/gc.go b/internal/datastore/postgres/gc.go index 5512ecefcf..2d4be6b69e 100644 --- a/internal/datastore/postgres/gc.go +++ b/internal/datastore/postgres/gc.go @@ -12,6 +12,7 @@ import ( "github.com/authzed/spicedb/internal/datastore/common" "github.com/authzed/spicedb/internal/datastore/postgres/schema" + log "github.com/authzed/spicedb/internal/logging" "github.com/authzed/spicedb/pkg/datastore" "github.com/authzed/spicedb/pkg/spiceerrors" ) @@ -183,6 +184,37 @@ func (pgg *pgGarbageCollector) deleteBeforeTx(ctx context.Context, conn exec, tx return removed, fmt.Errorf("failed to GC transactions table: %w", err) } + // Then the commit positions of those same transactions, if the commit LSN + // ledger is in use. + // + // This must run *after* the transactions above, and the order is not + // cosmetic. These are separate statements, so there is always a moment + // between them, and a crash can make that moment permanent. Deleting + // positions first would leave transaction rows that appear to have no + // recorded position, which is exactly how a transaction lost to a + // replication slot recreation appears — a watch reading them would refuse to + // deliver and tell its consumer to re-bootstrap. Deleting them second leaves + // the opposite: positions whose transaction is gone, which every reader + // joins away. + // + // ledger_gap is deliberately not pruned. It gains one row per slot + // invalidation, which should be rare enough to alert on, and a gap row stops + // matching watches on its own once every consumer's cursor is above it. + positionsRemoved, err := pgg.batchDelete( + ctx, + conn, + schema.TableLedgerXidLSN, + gcPKCols, + sq.Lt{schema.ColXID: minTxAlive}, + nil, + ) + if err != nil { + return removed, fmt.Errorf("failed to GC commit positions table: %w", err) + } + if positionsRemoved > 0 { + log.Ctx(ctx).Trace().Int64("commit-positions", positionsRemoved).Msg("deleted recorded commit positions") + } + // Delete any namespace rows with deleted_transaction <= the transaction ID. removed.Namespaces, err = pgg.batchDelete( ctx, diff --git a/internal/datastore/postgres/lsn_ledger.go b/internal/datastore/postgres/lsn_ledger.go new file mode 100644 index 0000000000..32d04718b8 --- /dev/null +++ b/internal/datastore/postgres/lsn_ledger.go @@ -0,0 +1,1715 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "hash/fnv" + "strconv" + "strings" + "time" + + "github.com/ccoveille/go-safecast/v2" + "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgproto3" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/prometheus/client_golang/prometheus" + + "github.com/authzed/spicedb/internal/datastore/postgres/schema" + log "github.com/authzed/spicedb/internal/logging" +) + +// The commit LSN ledger records, for every SpiceDB write transaction, the WAL +// position at which that transaction committed. +// +// A transaction cannot record its own commit LSN, because writing it would +// create a new commit. PostgreSQL keeps no commit-order record either: xids are +// assigned when a transaction begins, the transaction row's timestamp is taken +// at statement time, and pg_xact_commit_timestamp is optional and tie-prone. So +// the only durable source of commit order is the WAL itself. +// +// The ledger reads it. One instance in the cluster tails a durable logical +// replication slot publishing relation_tuple_transaction inserts, and appends +// each transaction's commit LSN to the ledger_xid_lsn table. The slot's +// confirmed_flush_lsn is only advanced after those appends commit, which makes +// it a completeness frontier: every transaction that committed at or below it +// has its commit LSN recorded. +// +// Recording is an append rather than an update of the transaction row, so a +// SpiceDB write costs one narrow row here instead of a second version of its +// transaction row plus an entry in each of that table's four indexes. The +// transaction table stays insert/delete-only. +// +// This is what lets the logical watch position a replayed transaction at its +// real commit LSN instead of one fabricated per Watch call. A token is then a +// function of the transaction it points at, identical no matter which Watch call +// delivered it, and comparable against every other token ever emitted. +// +// Exactly one instance may consume a replication slot at a time: PostgreSQL +// fails START_REPLICATION on a slot that is already attached. That error is the +// cluster's leader election. Every instance runs the ledger, the losers stand by +// and retry, and a leader's death is noticed within one retry interval. +const ( + // pgOutputPlugin is the logical decoding plugin the ledger consumes, + // PostgreSQL's built-in one. + pgOutputPlugin = "pgoutput" + + // pgOutputProtoVersion is the pgoutput protocol version requested at + // START_REPLICATION. + pgOutputProtoVersion = "1" + + // minimumStatusInterval floors how often the ledger sends standby status + // updates so the server can release WAL. + minimumStatusInterval = time.Second + + // pgDuplicateObjectErr is returned when creating an object that already + // exists, which is how creation races between instances are detected. + pgDuplicateObjectErr = "42710" + + // ledgerIdleFlushDelay bounds how long recorded commit positions wait when + // the stream has gone quiet. Under load, batches fill and flush by size + // instead. This is the dominant term in how long a starting Watch call waits + // for the frontier to reach its marker, so it is deliberately short. + ledgerIdleFlushDelay = 10 * time.Millisecond + + // minimumLedgerRetryInterval floors the wait between attempts to attach to + // the ledger slot, so a misconfigured interval cannot spin. + minimumLedgerRetryInterval = 100 * time.Millisecond + + // pgObjectInUseErr is returned by START_REPLICATION when another session + // already holds the replication slot. + pgObjectInUseErr = "55006" + + // walStatusLost marks a replication slot whose WAL has been removed. Such a + // slot can never be resumed and must be recreated. + walStatusLost = "lost" + + // ledgerGapPendingToLSN is the to_lsn of a ledger_gap row for a slot + // recreation that has not completed. It is the maximum pg_lsn, so the row + // fails every watch positioned below it until the recreation finishes and + // bounds the gap at the new slot's first position. A crash mid-recreation + // therefore leaves a loud, over-wide gap rather than an unrecorded one. + ledgerGapPendingToLSN = "FFFFFFFF/FFFFFFFF" +) + +var ( + // ledgerPublicationTables is only the transaction table: one row per + // SpiceDB write, carrying the xid8 the commit LSN is recorded against. + ledgerPublicationTables = []string{schema.TableTransaction} + + // recordCommitLSNsQuery appends a batch of decoded commit positions. The + // positions travel as parallel text arrays, so the statement has the same + // shape for every batch size, and are zipped by unnest. + // + // This is the ledger's only write, and it is an append: the transaction rows + // it describes are never touched, so recording a position costs one narrow + // row and one entry in each of this table's two indexes, rather than + // rewriting a transaction row and all four of its indexes. + // + // A batch replays after a crash between writing it and confirming the slot, + // which re-appends identical rows; conflicting on the transaction id makes + // that a no-op. The conflict is on xid rather than the position because + // "this transaction is already recorded" is the condition being tolerated. + recordCommitLSNsQuery = fmt.Sprintf(` + INSERT INTO %[1]s (%[2]s, %[3]s) + SELECT recorded.xid::xid8, recorded.commit_lsn::pg_lsn + FROM unnest($1::text[], $2::text[]) AS recorded(xid, commit_lsn) + ON CONFLICT (%[2]s) DO NOTHING;`, + schema.TableLedgerXidLSN, schema.ColXID, schema.ColCommitLSN) + + // commitLSNForXidQuery reads one transaction's recorded commit position. It + // returns no row when the ledger has not reached that transaction yet, which + // is how a starting watch waits for its marker. pgx has no pg_lsn codec, so + // the value is rendered as text and parsed with pglogrepl. + commitLSNForXidQuery = fmt.Sprintf( + `SELECT %[1]s::text FROM %[2]s WHERE %[3]s = $1;`, + schema.ColCommitLSN, schema.TableLedgerXidLSN, schema.ColXID) + + // insertLedgerGenesisQuery records the snapshot taken when the ledger slot + // was first created. It is never overwritten: it is the only way to tell a + // transaction that committed before the ledger existed from one lost to a + // slot recreation. + // + // ledger_state holds at most one row, enforced by its singleton column: a + // boolean primary key constrained to be true. Recording the genesis snapshot + // is therefore an insert that conflicts with itself on every attempt after + // the first, which makes it exactly-once across every instance without any + // coordination between them. The conflict target is named rather than left + // implicit so that a violation of some other constraint is reported instead + // of being silently read as "already recorded". + insertLedgerGenesisQuery = fmt.Sprintf( + `INSERT INTO %[1]s (%[3]s, %[2]s, %[4]s) VALUES (true, pg_current_snapshot(), $1::pg_lsn) + ON CONFLICT (%[3]s) DO NOTHING;`, + schema.TableLedgerState, schema.ColLedgerGenesisSnapshot, schema.ColLedgerSingleton, + schema.ColLedgerGenesisLSN) + + // selectLedgerBackfillStateQuery reads what the pre-ledger backfill needs to + // hand out its next batch: the snapshot that identifies pre-ledger history, + // the position everything it assigns must stay below, and how many positions + // it has already handed out below that. + selectLedgerBackfillStateQuery = fmt.Sprintf( + `SELECT %[2]s, %[3]s::text, %[4]s, %[5]s FROM %[1]s LIMIT 1;`, + schema.TableLedgerState, schema.ColLedgerGenesisSnapshot, schema.ColLedgerGenesisLSN, + schema.ColLedgerBackfillOffset, schema.ColLedgerBackfillComplete) + + // preLedgerTransactionsQuery returns the next batch of history to position, + // newest first. + // + // A transaction qualifies when it predates the ledger (visible in the genesis + // snapshot), has no position yet, and is still inside the collection window. + // The window bound is what makes the work finite: a revision older than it is + // already refused as stale, so positioning it would serve nobody. + // + // Newest first is deliberate. Positions are handed out counting down from the + // genesis position, so collection pruning the old end never shifts a position + // already assigned, and a partially finished backfill always leaves the + // *oldest* slice unpositioned, which is exactly where the catch-up query's + // NULLS FIRST puts it. + preLedgerTransactionsQuery = fmt.Sprintf(` + SELECT t.%[1]s + FROM %[2]s t + LEFT JOIN %[3]s p ON p.%[1]s = t.%[1]s + WHERE p.%[1]s IS NULL + AND pg_visible_in_snapshot(t.%[1]s, $1) + AND t.%[4]s >= $2 + AND pg_xact_commit_timestamp(t.%[1]s::xid) IS NOT NULL + ORDER BY pg_xact_commit_timestamp(t.%[1]s::xid) DESC, t.%[1]s DESC + LIMIT $3;`, + schema.ColXID, schema.TableTransaction, schema.TableLedgerXidLSN, schema.ColTimestamp) + + // advanceLedgerBackfillQuery records how far the backfill has counted down, + // and marks it finished when a batch comes back short. + advanceLedgerBackfillQuery = fmt.Sprintf( + `UPDATE %[1]s SET %[2]s = $1, %[3]s = $2, %[4]s = (NOW() AT TIME ZONE 'utc');`, + schema.TableLedgerState, schema.ColLedgerBackfillOffset, + schema.ColLedgerBackfillComplete, schema.ColLedgerUpdatedAt) + + // selectLedgerGenesisQuery reads the genesis snapshot, if one was ever recorded. + selectLedgerGenesisQuery = fmt.Sprintf( + `SELECT %[1]s FROM %[2]s LIMIT 1;`, + schema.ColLedgerGenesisSnapshot, schema.TableLedgerState) + + // selectSlotRecreationsQuery reads how many times the ledger slot has been + // recreated, which is reported as a gauge for alerting. + selectSlotRecreationsQuery = fmt.Sprintf( + `SELECT COALESCE(MAX(%[1]s), 0) FROM %[2]s;`, + schema.ColLedgerSlotRecreations, schema.TableLedgerState) + + // recordSlotRecreationQuery counts recreations of an invalidated slot, which + // is the operator-visible signal that a gap in recorded commit positions exists. + recordSlotRecreationQuery = fmt.Sprintf( + `UPDATE %[1]s SET %[2]s = %[2]s + 1, %[3]s = (NOW() AT TIME ZONE 'utc');`, + schema.TableLedgerState, schema.ColLedgerSlotRecreations, schema.ColLedgerUpdatedAt) + + // selectSlotStateQuery reports whether the ledger slot exists, whether it is + // attached, how far it has durably confirmed, whether its WAL still exists, + // and which database it is bound to. + selectSlotStateQuery = ` + SELECT confirmed_flush_lsn::text, active, COALESCE(wal_status, ''), COALESCE(database, '') + FROM pg_replication_slots WHERE slot_name = $1;` + + // createLedgerSlotQuery creates the ledger's durable slot and returns its + // consistent point: the position from which the new slot decodes, which is + // where a recreation's gap in recorded positions ends. + createLedgerSlotQuery = `SELECT lsn::text FROM pg_create_logical_replication_slot($1, $2);` + dropLedgerSlotQuery = `SELECT pg_drop_replication_slot($1);` + + // insertLedgerGapQuery opens a gap: an interval of WAL the ledger will never + // decode. It is written with the pending sentinel as its end position before + // the slot is touched, so that no crash can leave the gap unrecorded. + insertLedgerGapQuery = fmt.Sprintf( + `INSERT INTO %[1]s (%[2]s, %[3]s) VALUES ($1::pg_lsn, $2::pg_lsn) ON CONFLICT DO NOTHING;`, + schema.TableLedgerGap, schema.ColGapFromLSN, schema.ColGapToLSN) + + // closeLedgerGapsQuery bounds every pending gap at the given position, which + // is the first position the recreated slot decodes from. Replacing the rows + // rather than updating them in place keeps the operation atomic under the + // table's (from_lsn, to_lsn) primary key. + closeLedgerGapsQuery = fmt.Sprintf(` + WITH pending AS ( + DELETE FROM %[1]s WHERE %[3]s = '%[4]s'::pg_lsn AND %[2]s < $1::pg_lsn RETURNING %[2]s + ) + INSERT INTO %[1]s (%[2]s, %[3]s) SELECT %[2]s, $1::pg_lsn FROM pending ON CONFLICT DO NOTHING;`, + schema.TableLedgerGap, schema.ColGapFromLSN, schema.ColGapToLSN, ledgerGapPendingToLSN) + + // firstLedgerGapAboveQuery finds the lowest recorded gap that a cursor at the + // given position has not passed. A match means transactions above the cursor + // were never recorded and delivery from it cannot be complete. The detection + // time comes along so the watch can report *when* the gap opened, which is + // what lets an operator line it up against the rest of an incident. + firstLedgerGapAboveQuery = fmt.Sprintf( + `SELECT %[2]s::text, %[3]s::text, %[4]s::text FROM %[1]s + WHERE %[3]s > $1::pg_lsn ORDER BY %[2]s LIMIT 1;`, + schema.TableLedgerGap, schema.ColGapFromLSN, schema.ColGapToLSN, schema.ColGapDetectedAt) + + // selectLedgerGapsQuery lists every recorded gap. Healing replays a gap's + // transactions out of the tables, and a transaction carries no evidence of + // which gap swallowed it, so a database holding more than one gap is left to + // the loud failure path rather than guessing an interval. + selectLedgerGapsQuery = fmt.Sprintf( + `SELECT %[2]s::text, %[3]s::text, %[4]s::text FROM %[1]s ORDER BY %[2]s;`, + schema.TableLedgerGap, schema.ColGapFromLSN, schema.ColGapToLSN, schema.ColGapDetectedAt) + + // ledgerHealTargetQuery samples the position the ledger must confirm past + // before a gap can be healed, together with the snapshot that bounds the + // window at the same instant. A transaction visible in that snapshot + // committed before the sample, so once the ledger has confirmed past the + // sampled position, such a transaction is either recorded or was swallowed + // by the gap; nothing that is merely still in flight can be mistaken for a + // victim. + ledgerHealTargetQuery = `SELECT pg_current_wal_lsn()::text, pg_current_snapshot();` + + // gapWindowTransactionsQuery returns the transactions a gap swallowed, in + // true commit order. + // + // A transaction belongs to the window when it postdates the ledger's genesis + // snapshot (so it is not pre-ledger history, which never had a position and + // is delivered unpositioned), committed before the window was bounded, and + // either has no recorded position or already carries one an earlier healing + // attempt assigned. Including the already-assigned ones is what keeps the + // rank below stable: a retry that finds part of its own work committed still + // ranks the same set and so reproduces the same positions. + // + // pg_xact_commit_timestamp is the commit order the lost WAL would have given. + // Commit timestamps are only microsecond resolution, so the transaction id + // breaks ties, which also makes the order total and the rank deterministic. + gapWindowTransactionsQuery = fmt.Sprintf(` + SELECT t.%[1]s, t.%[4]s + FROM %[2]s t + LEFT JOIN %[3]s p ON p.%[1]s = t.%[1]s + WHERE NOT pg_visible_in_snapshot(t.%[1]s, $1) + AND pg_visible_in_snapshot(t.%[1]s, $2) + AND (p.%[1]s IS NULL OR (p.%[5]s > $3::pg_lsn AND p.%[5]s < $4::pg_lsn)) + ORDER BY pg_xact_commit_timestamp(t.%[1]s::xid), t.%[1]s;`, + schema.ColXID, schema.TableTransaction, schema.TableLedgerXidLSN, + schema.ColTimestamp, schema.ColCommitLSN) + + // deleteLedgerGapQuery retires a gap whose transactions have all been + // replayed, which is what lets watches positioned below it resume. + deleteLedgerGapQuery = fmt.Sprintf( + `DELETE FROM %[1]s WHERE %[2]s = $1::pg_lsn AND %[3]s = $2::pg_lsn;`, + schema.TableLedgerGap, schema.ColGapFromLSN, schema.ColGapToLSN) + + // maxRecordedCommitLSNQuery reports the highest commit position the ledger + // has recorded, or NULL when none remain. Everything at or below it is + // recorded, which makes it the tightest sound start for a gap whose true + // start died with an operator-dropped slot. + maxRecordedCommitLSNQuery = fmt.Sprintf( + `SELECT max(%[1]s)::text FROM %[2]s;`, + schema.ColCommitLSN, schema.TableLedgerXidLSN) + + // errLedgerGenesisMissing indicates no genesis snapshot has been recorded, + // which means the ledger has never been provisioned against this database. + errLedgerGenesisMissing = errors.New("the commit LSN ledger has not been initialized") + + // errLedgerReprovisioned indicates the ledger's slot was missing or + // invalidated and has been re-created, so the stream has to be established + // against the new slot. It is a step in recovering, not a failure, and in + // particular it is not the clean end of the stream that shutdown produces. + errLedgerReprovisioned = errors.New("the commit LSN ledger slot was re-provisioned") +) + +var ( + ledgerActiveGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "ledger_active", + Help: "Whether this instance is the one recording commit LSNs for the logical watch.", + }) + + ledgerBackfilledCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "ledger_backfilled_transactions_total", + Help: "The number of transactions whose commit LSN has been recorded.", + }) + + ledgerFlushDurationHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "ledger_flush_duration_seconds", + Help: "The duration of writing a batch of recorded commit LSNs.", + Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5}, + }) + + ledgerLagBytesGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "ledger_lag_bytes", + Help: "WAL bytes between the server's current position and the commit LSN ledger's confirmed frontier.", + }) + + ledgerAttachmentsCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "ledger_attachments_total", + Help: "The number of times this instance has taken over recording commit LSNs.", + }) + + ledgerFailureCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "ledger_failure_total", + Help: "The number of times the commit LSN ledger disconnected with an error.", + }) + + ledgerSlotRecreationsGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "ledger_slot_recreations", + Help: "The number of times the commit LSN ledger's replication slot has been recreated, each of which leaves a gap in recorded commit positions.", + }) + + ledgerGapsHealedCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "ledger_gaps_healed_total", + Help: "Gaps in recorded commit positions replayed from the transaction tables, after which watches positioned below them resume instead of failing.", + }) + + ledgerPreLedgerBackfilledCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "ledger_pre_ledger_backfilled_total", + Help: "Transactions predating the commit LSN ledger given a reconstructed commit position.", + }) + + ledgerGapTransactionsHealedCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "ledger_gap_transactions_healed_total", + Help: "Transactions given a replayed commit position while healing a gap.", + }) + + watchLedgerWaitHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "watch_ledger_wait_seconds", + Help: "How long a starting logical watch waited for the commit LSN ledger to record its marker transaction.", + Buckets: []float64{0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 30}, + }) +) + +// ledgerSlotState is what pg_replication_slots reports about the ledger's slot. +type ledgerSlotState struct { + confirmed pglogrepl.LSN + active bool + walStatus string + database string + exists bool +} + +// ledgerPosition is one transaction's commit position, as read from the WAL. +type ledgerPosition struct { + xid xid8 + commitLSN pglogrepl.LSN +} + +// ledgerGap is a recorded interval of WAL the ledger never decoded, left behind +// when its replication slot was invalidated or dropped and had to be recreated. +// Transactions that committed inside it have no recorded commit position. +type ledgerGap struct { + // from is the last position known complete before the gap, to the first + // position complete after it. + from, to pglogrepl.LSN + + // detectedAt is when the recreation was noticed, rendered as text: it is + // reported to operators rather than compared, so it is never parsed. + detectedAt string +} + +// ledgerBatch accumulates commit positions until they are worth a write, and +// tracks the stream position those writes let the slot confirm. It is the +// ledger's flush policy, kept separate from the replication session. +type ledgerBatch struct { + maxSize int + maxDelay time.Duration + + positions []ledgerPosition + + // confirmLSN is the end position of the last transaction added, which + // becomes the slot's frontier once the batch is durable. + confirmLSN pglogrepl.LSN + + // startedAt is when the oldest un-flushed position was added. + startedAt time.Time +} + +// ledgerTransaction is one WAL transaction, reduced to what the ledger records. +type ledgerTransaction struct { + commitLSN pglogrepl.LSN + endLSN pglogrepl.LSN + + // hasTransactionRow reports whether the transaction inserted a + // relation_tuple_transaction row. Transactions without one have no commit + // position to record: the ledger's own writes are the common case. + hasTransactionRow bool + xid xid8 +} + +// ledgerDecoder is a minimal pgoutput state machine that extracts the xid8 of +// each committed transaction. It decodes only the transaction table's xid +// column, deliberately sharing none of the watch's semantic decoding. +type ledgerDecoder struct { + relations map[uint32]*pglogrepl.RelationMessage + typeMap *pgtype.Map + current *ledgerTransaction +} + +// prepareCommitLSNLedger provisions the ledger's publication and durable slot. +// It runs on every instance at construction; the database objects are shared and +// created at most once. +func (pgd *pgDatastore) prepareCommitLSNLedger(ctx context.Context) error { + if err := pgd.resolveLedgerSlotName(ctx); err != nil { + return err + } + + if err := pgd.ensurePublication(ctx, pgd.logicalWatchLedgerPublicationName, ledgerPublicationTables, "insert"); err != nil { + return err + } + + pgd.warnAboutLedgerSlotLimits(ctx) + + return pgd.ensureCommitLSNLedgerSlot(ctx) +} + +// warnAboutLedgerSlotLimits reports the two server settings that decide what +// happens when the ledger falls behind or cannot get a slot. Neither is fatal and +// neither can be set from here: they are cluster-wide, and on managed PostgreSQL +// they live in a parameter group. +func (pgd *pgDatastore) warnAboutLedgerSlotLimits(ctx context.Context) { + var maxSlots, usedSlots int + if err := pgd.writePool.QueryRow(ctx, + "SELECT current_setting('max_replication_slots')::int, (SELECT count(*) FROM pg_replication_slots);", + ).Scan(&maxSlots, &usedSlots); err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("unable to check replication slot headroom for the commit LSN ledger") + } else if usedSlots >= maxSlots { + log.Ctx(ctx).Warn().Int("max_replication_slots", maxSlots).Int("in_use", usedSlots). + Msg("no replication slot headroom: the commit LSN ledger needs one durable slot for this database, so raise max_replication_slots") + } + + // Unlimited retention means a stalled ledger grows pg_wal until the disk + // fills, taking the database down with it. A bounded setting makes PostgreSQL + // invalidate the ledger's slot instead, which is recoverable: the watch + // reports the resulting gap and consumers restart from a current revision. + var keepSize string + if err := pgd.writePool.QueryRow(ctx, "SHOW max_slot_wal_keep_size;").Scan(&keepSize); err != nil { + return + } + if keepSize == "-1" { + log.Ctx(ctx).Warn(). + Msg("max_slot_wal_keep_size is unlimited, so a stalled commit LSN ledger will retain WAL until the disk fills; bound it so PostgreSQL invalidates the slot instead, and alert on spicedb_datastore_postgres_ledger_lag_bytes") + } +} + +// warnIfAbandonedLedgerSlot reports a leftover ledger slot on a database where +// the feature it serves has been switched off. Preflight does not run in that +// configuration, so nothing else would notice a durable slot that no longer has +// a consumer and retains WAL indefinitely. +func (pgd *pgDatastore) warnIfAbandonedLedgerSlot(ctx context.Context) { + if err := pgd.resolveLedgerSlotName(ctx); err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("unable to check for an abandoned commit LSN ledger replication slot") + return + } + + state, err := pgd.readLedgerSlotState(ctx) + if err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("unable to check for an abandoned commit LSN ledger replication slot") + return + } + + // An attached slot is another instance's, still running with the feature + // enabled; only an unattached one is abandoned. + if state.exists && !state.active { + log.Ctx(ctx).Warn().Str("slot", pgd.ledgerSlotName).Str("wal_status", state.walStatus). + Msgf("a commit LSN ledger replication slot exists but the watch it serves is disabled; nothing consumes it, so it will retain WAL indefinitely. If the feature is staying off, drop it: SELECT pg_drop_replication_slot('%s');", pgd.ledgerSlotName) + } +} + +// resolveLedgerSlotName qualifies the configured slot name with a digest of the +// database name. +// +// Replication slot names are cluster-global, but a logical slot decodes exactly +// one database and the ledger is per-database. Two SpiceDB databases in one +// PostgreSQL cluster would otherwise fight over a single slot: the first would +// create it and the second would find it already present but bound elsewhere, +// leaving it unable to record anything. Qualifying the name gives each database +// its own slot with no configuration. +func (pgd *pgDatastore) resolveLedgerSlotName(ctx context.Context) error { + var database string + if err := pgd.writePool.QueryRow(ctx, "SELECT current_database();").Scan(&database); err != nil { + return fmt.Errorf("unable to determine the current database for the commit LSN ledger: %w", err) + } + + digest := fnv.New32a() + if _, err := digest.Write([]byte(database)); err != nil { + return fmt.Errorf("unable to derive the commit LSN ledger slot name: %w", err) + } + + pgd.ledgerSlotName = fmt.Sprintf("%s_%08x", pgd.logicalWatchLedgerSlotName, digest.Sum32()) + pgd.ledgerDatabase = database + return nil +} + +// ensureCommitLSNLedgerSlot creates the ledger's replication slot if it is +// missing, recreates it if its WAL has been lost, and records the genesis +// snapshot the watch uses to interpret transactions with no recorded position. +// +// Every recreation leaves an interval of WAL the ledger never decoded, and +// after it the slot's frontier jumps past that interval, so nothing downstream +// can detect it. The interval is therefore recorded as a ledger_gap row, and +// recorded *before* the slot is touched: the gap opens with a pending sentinel +// end position and is bounded at the new slot's consistent point once creation +// succeeds. A crash between the two leaves a loud over-wide gap instead of a +// silent one. +func (pgd *pgDatastore) ensureCommitLSNLedgerSlot(ctx context.Context) error { + slotName := pgd.ledgerSlotName + + state, err := pgd.readLedgerSlotState(ctx) + if err != nil { + return err + } + slotPreExisted := state.exists + + if state.exists && state.database != pgd.ledgerDatabase { + return fmt.Errorf( + "the commit LSN ledger's replication slot %q decodes database %q rather than %q; configure a distinct slot name for this datastore", + slotName, state.database, pgd.ledgerDatabase) + } + + // Whether the ledger has ever been provisioned against this database + // decides whether a missing slot is a first creation or a recreation that + // skipped WAL. + provisioned := true + if _, err := pgd.ledgerGenesisSnapshot(ctx); err != nil { + if !errors.Is(err, errLedgerGenesisMissing) { + return err + } + provisioned = false + } + + if state.exists && state.walStatus == walStatusLost { + // The slot's WAL is gone, so the transactions that committed while it was + // unattended can never have their commit positions recovered. Recreating + // it restores the ledger going forward; the genesis snapshot stays as it + // was, which is what lets the watch tell these transactions apart from + // ones that predate the ledger. The gap opens at the slot's last + // confirmed position: everything at or below it is recorded, nothing + // above it was decoded. + log.Ctx(ctx).Error().Str("slot", slotName). + Msg("the commit LSN ledger's replication slot was invalidated; recreating it. Transactions that committed while it was invalid have no recorded commit position, and watches positioned below the recorded gap will fail until their consumers restart from a current revision") + + if err := pgd.openLedgerGap(ctx, state.confirmed); err != nil { + return err + } + if _, err := pgd.writePool.Exec(ctx, dropLedgerSlotQuery, slotName); err != nil { + return fmt.Errorf("unable to drop the invalidated commit LSN ledger slot %s: %w", slotName, err) + } + state = ledgerSlotState{} + } + + recreated := !state.exists && provisioned + + if !state.exists && provisioned { + // The slot is gone without the ledger having invalidation-dropped it, + // which means an operator dropped it, and its last confirmed position + // died with it. The highest recorded commit position is a sound lower + // bound for the gap: everything at or below it is recorded. + log.Ctx(ctx).Error().Str("slot", slotName). + Msg("the commit LSN ledger's replication slot is missing on a database where the ledger was provisioned; recreating it and recording the gap. Transactions that committed while it was gone have no recorded commit position") + + var maxRecordedText *string + if err := pgd.writePool.QueryRow(ctx, maxRecordedCommitLSNQuery).Scan(&maxRecordedText); err != nil { + return fmt.Errorf("unable to bound the commit LSN ledger gap: %w", err) + } + var gapFrom pglogrepl.LSN + if maxRecordedText != nil { + gapFrom, err = pglogrepl.ParseLSN(*maxRecordedText) + if err != nil { + return fmt.Errorf("unable to parse the highest recorded commit position: %w", err) + } + } + if err := pgd.openLedgerGap(ctx, gapFrom); err != nil { + return err + } + } + + // The slot's consistent point is the position everything predating the ledger + // committed below, which is where the pre-ledger backfill counts down from. + var consistentPointText string + + if !state.exists { + if err := pgd.writePool.QueryRow(ctx, createLedgerSlotQuery, slotName, pgOutputPlugin).Scan(&consistentPointText); err != nil { + // Another instance may have raced us to create the slot; its slot + // state read below bounds any pending gap all the same. + if pgerr, ok := errors.AsType[*pgconn.PgError](err); !ok || pgerr.Code != pgDuplicateObjectErr { + return fmt.Errorf("unable to create the commit LSN ledger slot %s: %w", slotName, err) + } + } + + state, err = pgd.readLedgerSlotState(ctx) + if err != nil { + return err + } + if !state.exists { + return fmt.Errorf("the commit LSN ledger slot %s disappeared while being provisioned", slotName) + } + } + + // Bound every pending gap at the slot's first decoded position. After a + // crash mid-recreation the slot may have confirmed further by now, so this + // can over-approximate the gap's end, which fails a few more watches than + // strictly necessary but never hides a skipped interval. + if err := pgd.closePendingLedgerGaps(ctx, state.confirmed); err != nil { + return err + } + + if recreated { + if _, err := pgd.writePool.Exec(ctx, recordSlotRecreationQuery); err != nil { + return fmt.Errorf("unable to record the commit LSN ledger slot recreation: %w", err) + } + } + + // The genesis snapshot must be taken at or after the slot's creation: a + // transaction invisible in it commits after the slot exists, and so is + // guaranteed to reach the ledger. Transactions that commit between the + // slot's creation and this snapshot are visible in it *and* reach the + // ledger, which is harmless because the watch only consults the genesis + // snapshot for rows that still have no recorded position after the frontier + // has passed them. + // A slot this process did not create reports no consistent point, so its + // confirmed position stands in: everything predating the ledger committed + // below that too. + genesisLSN := consistentPointText + if genesisLSN == "" { + genesisLSN = state.confirmed.String() + } + + tag, err := pgd.writePool.Exec(ctx, insertLedgerGenesisQuery, genesisLSN) + if err != nil { + return fmt.Errorf("unable to record the commit LSN ledger genesis snapshot: %w", err) + } + if tag.RowsAffected() > 0 && slotPreExisted { + log.Ctx(ctx).Warn().Str("slot", slotName). + Msg("recorded a commit LSN ledger genesis snapshot for a pre-existing replication slot; transactions that committed while that slot was unattended may be treated as predating the ledger") + } + + var recreations int64 + if err := pgd.writePool.QueryRow(ctx, selectSlotRecreationsQuery).Scan(&recreations); err == nil { + ledgerSlotRecreationsGauge.Set(float64(recreations)) + } + + return nil +} + +// openLedgerGap records the start of a WAL interval the ledger is about to +// lose, with the pending sentinel as its end. It runs before the slot surgery +// it protects, so no crash between the two can leave the interval unrecorded. +func (pgd *pgDatastore) openLedgerGap(ctx context.Context, fromLSN pglogrepl.LSN) error { + if _, err := pgd.writePool.Exec(ctx, insertLedgerGapQuery, fromLSN.String(), ledgerGapPendingToLSN); err != nil { + return fmt.Errorf("unable to record the commit LSN ledger gap: %w", err) + } + return nil +} + +// closePendingLedgerGaps bounds every pending gap at the given position, the +// first position the recreated slot decodes from. +func (pgd *pgDatastore) closePendingLedgerGaps(ctx context.Context, toLSN pglogrepl.LSN) error { + if _, err := pgd.writePool.Exec(ctx, closeLedgerGapsQuery, toLSN.String()); err != nil { + return fmt.Errorf("unable to bound the commit LSN ledger gap: %w", err) + } + return nil +} + +// pendingGapHeal is a recorded gap the ledger may be able to replay, together +// with the state that bounds the replay: the position the ledger must confirm +// past first, and the snapshot taken at the same instant that decides which +// transactions the gap is answerable for. +type pendingGapHeal struct { + gap ledgerGap + target pglogrepl.LSN + boundary pgSnapshot +} + +// planGapHeal reports the gap this attachment should try to replay, if any. +// +// It returns nothing unless exactly one bounded gap exists: replaying works +// backwards from the transaction tables, where a transaction carries no +// evidence of which gap swallowed it, so attributing transactions among several +// gaps would be guesswork. A still-pending gap is skipped because its interval +// has no end yet. +func (pgd *pgDatastore) planGapHeal(ctx context.Context) (*pendingGapHeal, error) { + rows, err := pgd.readPool.Query(ctx, selectLedgerGapsQuery) + if err != nil { + return nil, fmt.Errorf("unable to list commit LSN ledger gaps: %w", err) + } + defer rows.Close() + + var gaps []ledgerGap + for rows.Next() { + var fromText, toText, detectedAt string + if err := rows.Scan(&fromText, &toText, &detectedAt); err != nil { + return nil, fmt.Errorf("unable to decode a recorded ledger gap: %w", err) + } + from, err := pglogrepl.ParseLSN(fromText) + if err != nil { + return nil, fmt.Errorf("unable to parse a recorded ledger gap: %w", err) + } + to, err := pglogrepl.ParseLSN(toText) + if err != nil { + return nil, fmt.Errorf("unable to parse a recorded ledger gap: %w", err) + } + gaps = append(gaps, ledgerGap{from: from, to: to, detectedAt: detectedAt}) + } + if rows.Err() != nil { + return nil, fmt.Errorf("unable to list commit LSN ledger gaps: %w", rows.Err()) + } + + if len(gaps) != 1 { + return nil, nil + } + gap := gaps[0] + + pending, err := pglogrepl.ParseLSN(ledgerGapPendingToLSN) + if err != nil { + return nil, fmt.Errorf("unable to parse the pending gap sentinel: %w", err) + } + if gap.to >= pending { + return nil, nil + } + + var targetText string + var boundary pgSnapshot + if err := pgd.readPool.QueryRow(ctx, ledgerHealTargetQuery).Scan(&targetText, &boundary); err != nil { + return nil, fmt.Errorf("unable to sample the commit LSN ledger heal target: %w", err) + } + target, err := pglogrepl.ParseLSN(targetText) + if err != nil { + return nil, fmt.Errorf("unable to parse the commit LSN ledger heal target: %w", err) + } + + return &pendingGapHeal{gap: gap, target: target, boundary: boundary}, nil +} + +// healLedgerGap replays a gap out of the transaction tables instead of leaving +// every watch below it to fail. +// +// A gap loses commit positions, not changes. The transactions it swallowed +// still have their relation_tuple_transaction rows, and the relationships they +// wrote are still in relation_tuple, removals included, because a removal is an +// MVCC soft delete that leaves the row in place until garbage collection. +// Giving each of those transactions a position inside the gap's interval +// therefore restores delivery of both the adds and the removes through the +// watch's ordinary query, with no special case on the read side. +// +// Positions are assigned in true commit order and land strictly inside the +// interval, so they sort above everything recorded before the gap and below +// everything the recreated slot recorded after it. They are stable: the window +// is closed, the order is total, and the rank covers positions an earlier +// attempt already assigned, so a retry reproduces them exactly and the insert +// ignores the conflicts. +// +// Replaying is only sound while the window is newer than the garbage collection +// horizon, because a removal can only be replayed while its soft-deleted row +// survives. An older window keeps the loud failure, which is the one case no +// stream can recover. +func (pgd *pgDatastore) healLedgerGap(ctx context.Context, heal *pendingGapHeal) error { + genesis, err := pgd.ledgerGenesisSnapshot(ctx) + if err != nil { + return err + } + + rows, err := pgd.readPool.Query(ctx, gapWindowTransactionsQuery, + genesis, heal.boundary, heal.gap.from.String(), heal.gap.to.String()) + if err != nil { + return fmt.Errorf("unable to load the transactions a commit LSN ledger gap swallowed: %w", err) + } + defer rows.Close() + + var xids []xid8 + var oldest time.Time + for rows.Next() { + var txid xid8 + var timestamp time.Time + if err := rows.Scan(&txid, ×tamp); err != nil { + return fmt.Errorf("unable to decode a gap transaction: %w", err) + } + if oldest.IsZero() || timestamp.Before(oldest) { + oldest = timestamp + } + xids = append(xids, txid) + } + if rows.Err() != nil { + return fmt.Errorf("unable to load the transactions a commit LSN ledger gap swallowed: %w", rows.Err()) + } + + if len(xids) > 0 { + var now time.Time + if err := pgd.readPool.QueryRow(ctx, "SELECT NOW() AT TIME ZONE 'utc';").Scan(&now); err != nil { + return fmt.Errorf("unable to determine the database time: %w", err) + } + if oldest.Before(now.Add(-pgd.gcWindow)) { + log.Ctx(ctx).Warn().Str("slot", pgd.ledgerSlotName). + Str("from", heal.gap.from.String()).Str("to", heal.gap.to.String()). + Msg("a commit LSN ledger gap is older than the garbage collection window, so the relationships it removed may already be collected and it cannot be replayed; watches below it will continue to fail") + return nil + } + + // Each transaction needs its own position strictly inside the interval. + // A commit occupies far more WAL than the one byte a position consumes + // here, so this only fails if the interval was bounded far too tightly. + if uint64(len(xids)) >= uint64(heal.gap.to-heal.gap.from) { + log.Ctx(ctx).Warn().Str("slot", pgd.ledgerSlotName).Int("transactions", len(xids)). + Str("from", heal.gap.from.String()).Str("to", heal.gap.to.String()). + Msg("a commit LSN ledger gap holds more transactions than its interval has positions, so it cannot be replayed") + return nil + } + + positions := make([]ledgerPosition, 0, len(xids)) + for i, txid := range xids { + positions = append(positions, ledgerPosition{ + xid: txid, + commitLSN: heal.gap.from + pglogrepl.LSN(i) + 1, + }) + } + if err := pgd.recordCommitLSNs(ctx, positions); err != nil { + return err + } + ledgerGapTransactionsHealedCounter.Add(float64(len(positions))) + } + + if _, err := pgd.writePool.Exec(ctx, deleteLedgerGapQuery, + heal.gap.from.String(), heal.gap.to.String()); err != nil { + return fmt.Errorf("unable to retire a replayed commit LSN ledger gap: %w", err) + } + ledgerGapsHealedCounter.Inc() + + log.Ctx(ctx).Info().Str("slot", pgd.ledgerSlotName).Int("transactions", len(xids)). + Str("from", heal.gap.from.String()).Str("to", heal.gap.to.String()). + Msg("replayed a commit LSN ledger gap from the transaction tables; watches positioned below it can resume") + + return nil +} + +// backfillPreLedgerPositions gives one batch of pre-ledger history a position, +// so that a consumer resuming across the upgrade compares tokens instead of +// meeting an unpositioned prefix. It reports whether there is more to do. +// +// History predating the ledger has no recoverable commit LSN: it committed +// before anything was decoding WAL. What it does still have, inside the +// collection window, is a commit timestamp, and that is enough to reconstruct +// its order. Positions are handed out counting down from the ledger's genesis +// position, so they sort below everything the ledger records for itself while +// preserving commit order among themselves. +// +// Counting down is what makes the work resumable. Collection prunes the old end, +// so a position already assigned never shifts, and the slice left unpositioned +// by a partial pass is always the oldest one, which the catch-up query already +// orders first. Every intermediate state is therefore correct, and the batch can +// stop and resume freely. +// +// It runs one batch per flush: recording live positions is the ledger's real +// job, and the backfill is the background tenant that fills in behind it. +func (pgd *pgDatastore) backfillPreLedgerPositions(ctx context.Context) (bool, error) { + var genesis pgSnapshot + var genesisLSNText *string + var offset int64 + var complete bool + if err := pgd.readPool.QueryRow(ctx, selectLedgerBackfillStateQuery). + Scan(&genesis, &genesisLSNText, &offset, &complete); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + return false, fmt.Errorf("unable to read the pre-ledger backfill state: %w", err) + } + + // Nothing to anchor against, so pre-ledger history keeps the unpositioned + // delivery it has always had. + if complete || genesisLSNText == nil { + return false, nil + } + + genesisLSN, err := pglogrepl.ParseLSN(*genesisLSNText) + if err != nil { + return false, fmt.Errorf("unable to parse the commit LSN ledger genesis position: %w", err) + } + + var horizon time.Time + if err := pgd.readPool.QueryRow(ctx, "SELECT (NOW() AT TIME ZONE 'utc') - $1::interval;", + pgd.gcWindow.String()).Scan(&horizon); err != nil { + return false, fmt.Errorf("unable to determine the collection horizon: %w", err) + } + + batchSize := max(pgd.logicalWatchLedgerBatchSize, 1) + rows, err := pgd.readPool.Query(ctx, preLedgerTransactionsQuery, genesis, horizon, batchSize) + if err != nil { + return false, fmt.Errorf("unable to load pre-ledger history: %w", err) + } + defer rows.Close() + + var xids []xid8 + for rows.Next() { + var txid xid8 + if err := rows.Scan(&txid); err != nil { + return false, fmt.Errorf("unable to decode a pre-ledger transaction: %w", err) + } + xids = append(xids, txid) + } + if rows.Err() != nil { + return false, fmt.Errorf("unable to load pre-ledger history: %w", rows.Err()) + } + + // Guarded rather than converted blindly: a negative offset could only come + // from a corrupted row, and the unsigned arithmetic below would turn it into + // a position far above the genesis rather than below it. + handedOut, err := safecast.Convert[uint64](offset) + if err != nil { + return false, fmt.Errorf("unable to use the recorded pre-ledger backfill offset %d: %w", offset, err) + } + + // Each position must stay strictly inside (0, genesis). Every commit writes + // WAL, so the genesis position dwarfs the number of transactions below it; + // running out means the anchor is wrong, and stopping is the safe answer. + if handedOut+uint64(len(xids)) >= uint64(genesisLSN) { + log.Ctx(ctx).Warn().Str("slot", pgd.ledgerSlotName). + Msg("pre-ledger history does not fit below the ledger's genesis position; leaving the remainder unpositioned") + return false, pgd.finishPreLedgerBackfill(ctx, offset) + } + + // Newest first, walking down from the genesis position. + next := genesisLSN - pglogrepl.LSN(handedOut) - 1 + positions := make([]ledgerPosition, 0, len(xids)) + for _, txid := range xids { + positions = append(positions, ledgerPosition{xid: txid, commitLSN: next}) + next-- + } + if len(positions) > 0 { + if err := pgd.recordCommitLSNs(ctx, positions); err != nil { + return false, err + } + ledgerPreLedgerBackfilledCounter.Add(float64(len(positions))) + } + + // A short batch means the reachable history is exhausted. + if len(xids) < batchSize { + return false, pgd.finishPreLedgerBackfill(ctx, offset+int64(len(xids))) + } + + if _, err := pgd.writePool.Exec(ctx, advanceLedgerBackfillQuery, offset+int64(len(xids)), false); err != nil { + return false, fmt.Errorf("unable to advance the pre-ledger backfill: %w", err) + } + + return true, nil +} + +// finishPreLedgerBackfill marks the backfill done so later flushes stop looking. +func (pgd *pgDatastore) finishPreLedgerBackfill(ctx context.Context, offset int64) error { + if _, err := pgd.writePool.Exec(ctx, advanceLedgerBackfillQuery, offset, true); err != nil { + return fmt.Errorf("unable to complete the pre-ledger backfill: %w", err) + } + return nil +} + +// firstLedgerGapAbove returns the lowest recorded gap not entirely at or below +// the given position, if any. A watch whose cursor is below a gap cannot prove +// completeness and must fail rather than skip the gap's transactions silently. +func (pgd *pgDatastore) firstLedgerGapAbove(ctx context.Context, position pglogrepl.LSN) (ledgerGap, bool, error) { + var fromText, toText, detectedAt string + if err := pgd.readPool.QueryRow(ctx, firstLedgerGapAboveQuery, position.String()).Scan(&fromText, &toText, &detectedAt); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ledgerGap{}, false, nil + } + return ledgerGap{}, false, fmt.Errorf("unable to check for commit LSN ledger gaps: %w", err) + } + + from, err := pglogrepl.ParseLSN(fromText) + if err != nil { + return ledgerGap{}, false, fmt.Errorf("unable to parse a recorded ledger gap: %w", err) + } + to, err := pglogrepl.ParseLSN(toText) + if err != nil { + return ledgerGap{}, false, fmt.Errorf("unable to parse a recorded ledger gap: %w", err) + } + + return ledgerGap{from: from, to: to, detectedAt: detectedAt}, true, nil +} + +// readLedgerSlotState reports the ledger slot's durable frontier, whether it is +// currently attached, its WAL status, the database it decodes, and whether it +// exists at all. +// +// It reads through the read pool: every caller runs on a primary, where both +// pools address the same server, and the cursor watch calls this once per poll +// per watcher, which does not belong on the write pool. That per-watcher traffic +// is why a single shared frontier poller per datastore is the noted refinement if +// watcher counts ever grow. +func (pgd *pgDatastore) readLedgerSlotState(ctx context.Context) (ledgerSlotState, error) { + var state ledgerSlotState + + var confirmedText *string + row := pgd.readPool.QueryRow(ctx, selectSlotStateQuery, pgd.ledgerSlotName) + if err := row.Scan(&confirmedText, &state.active, &state.walStatus, &state.database); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ledgerSlotState{}, nil + } + return ledgerSlotState{}, fmt.Errorf("unable to inspect the commit LSN ledger slot: %w", err) + } + state.exists = true + + if confirmedText != nil { + confirmed, err := pglogrepl.ParseLSN(*confirmedText) + if err != nil { + return state, fmt.Errorf("unable to parse the commit LSN ledger slot's confirmed position: %w", err) + } + state.confirmed = confirmed + } + + return state, nil +} + +// ledgerGenesisSnapshot returns the snapshot taken when the ledger slot was +// first created. Transactions visible in it that still have no recorded commit +// position predate the ledger; transactions invisible in it that have none are a +// gap left by a slot recreation. +func (pgd *pgDatastore) ledgerGenesisSnapshot(ctx context.Context) (pgSnapshot, error) { + var genesis pgSnapshot + if err := pgd.readPool.QueryRow(ctx, selectLedgerGenesisQuery).Scan(&genesis); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return genesis, errLedgerGenesisMissing + } + return genesis, fmt.Errorf("unable to read the commit LSN ledger genesis snapshot: %w", err) + } + return genesis, nil +} + +// runCommitLSNLedger records commit positions for as long as the datastore +// lives. Only one instance can hold the slot, so the others stand by and retry; +// the loop therefore only returns when the datastore shuts down. +func (pgd *pgDatastore) runCommitLSNLedger(ctx context.Context) error { + retryInterval := max(pgd.logicalWatchLedgerRetryInterval, minimumLedgerRetryInterval) + + for { + if ctx.Err() != nil { + return nil + } + + err := pgd.recordCommitLSNsFromWAL(ctx) + + reattachNow := false + switch { + case ctx.Err() != nil: + return nil + + case err == nil: + // The stream ended without an error, which only happens on shutdown. + return nil + + case errors.Is(err, errLedgerReprovisioned): + // The slot was just created, so there is nothing to wait for: the + // sooner recording resumes, the smaller the gap that was recorded. + log.Ctx(ctx).Info().Str("slot", pgd.ledgerSlotName). + Msg("re-provisioned the commit LSN ledger slot; resuming recording") + reattachNow = true + + case isReplicationSlotInUseError(err): + log.Ctx(ctx).Trace().Str("slot", pgd.ledgerSlotName). + Msg("the commit LSN ledger slot is held by another instance; standing by") + + default: + ledgerFailureCounter.Inc() + log.Ctx(ctx).Warn().Err(err).Str("slot", pgd.ledgerSlotName). + Msg("the commit LSN ledger disconnected; retrying") + } + + if reattachNow { + continue + } + + select { + case <-ctx.Done(): + return nil + case <-time.After(retryInterval): + } + } +} + +// recordCommitLSNsFromWAL attaches to the ledger slot and records commit +// positions until the stream fails or the context is cancelled. +func (pgd *pgDatastore) recordCommitLSNsFromWAL(ctx context.Context) error { + // The slot's own confirmed position is where recording resumes, and is read + // before attaching because the query cannot run on a replication connection. + state, err := pgd.readLedgerSlotState(ctx) + if err != nil { + return err + } + if !state.exists || state.walStatus == walStatusLost { + // A slot dropped or invalidated underneath a running instance: + // re-provision it, which records the resulting gap, and report that so + // the caller reattaches rather than reading the successful + // re-provisioning as the stream having ended. + if err := pgd.ensureCommitLSNLedgerSlot(ctx); err != nil { + return err + } + return errLedgerReprovisioned + } + + // Planned before attaching, on the same connection kind as the rest of the + // ledger's bookkeeping, and acted on once the stream has confirmed past the + // sampled position. Only the instance that wins the slot reaches the stream, + // so the replay has a single writer without any further coordination. + heal, err := pgd.planGapHeal(ctx) + if err != nil { + return err + } + + conn, err := pgd.connectLogicalReplication(ctx) + if err != nil { + return fmt.Errorf("unable to establish the commit LSN ledger's replication connection: %w", err) + } + defer func() { + closeCtx, cancelClose := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelClose() + _ = conn.Close(closeCtx) + }() + + // The ledger only reads xid8 values, but the walsender renders every value + // using this session's settings, so they are pinned exactly as the watch + // pins them. + if _, err := conn.Exec(ctx, "SET TIME ZONE 'UTC'; SET datestyle TO ISO; SET bytea_output = 'hex';").ReadAll(); err != nil { + return fmt.Errorf("unable to configure the commit LSN ledger's replication session: %w", err) + } + + publicationOption := "publication_names " + quotePGOutputStringOption(quotePGOutputIdentifier(pgd.logicalWatchLedgerPublicationName)) + if err := pglogrepl.StartReplication(ctx, conn, pgd.ledgerSlotName, state.confirmed, pglogrepl.StartReplicationOptions{ + Mode: pglogrepl.LogicalReplication, + PluginArgs: []string{ + fmt.Sprintf("proto_version '%s'", pgOutputProtoVersion), + publicationOption, + }, + }); err != nil { + return fmt.Errorf("unable to start the commit LSN ledger's replication stream: %w", err) + } + + log.Ctx(ctx).Info().Str("slot", pgd.ledgerSlotName).Stringer("from", state.confirmed). + Msg("recording commit LSNs for the logical watch") + ledgerAttachmentsCounter.Inc() + ledgerActiveGauge.Set(1) + defer ledgerActiveGauge.Set(0) + + return pgd.consumeLedgerStream(ctx, conn, state.confirmed, heal) +} + +// consumeLedgerStream reads the ledger's replication stream, writing each +// transaction's commit position back to its row and confirming the slot only +// once those writes are durable. +func (pgd *pgDatastore) consumeLedgerStream(ctx context.Context, conn *pgconn.PgConn, confirmedLSN pglogrepl.LSN, heal *pendingGapHeal) error { + statusInterval := max(pgd.logicalWatchStatusInterval, minimumStatusInterval) + nextStatusDeadline := time.Now().Add(statusInterval) + var nextLagSample time.Time + + decoder := newLedgerDecoder() + batch := newLedgerBatch(pgd.logicalWatchLedgerBatchSize, pgd.logicalWatchLedgerFlushMaxDelay) + + // Attempted until a pass reports the reachable history exhausted; the state + // it reads is persisted, so a reattach resumes rather than restarts. + backfilling := true + + sendStatus := func() error { + if err := pglogrepl.SendStandbyStatusUpdate(ctx, conn, pglogrepl.StandbyStatusUpdate{ + WALWritePosition: confirmedLSN, + WALFlushPosition: confirmedLSN, + WALApplyPosition: confirmedLSN, + ClientTime: time.Now(), + }); err != nil { + return fmt.Errorf("unable to send the commit LSN ledger's standby status update: %w", err) + } + nextStatusDeadline = time.Now().Add(statusInterval) + return nil + } + + // flush is the only place the frontier moves, and it moves strictly after + // the recorded positions are committed. A crash in between replays the batch + // on the next attach, which rewrites the same values. + flush := func() error { + positions, confirmTo := batch.take() + if len(positions) == 0 { + return nil + } + + startedAt := time.Now() + if err := pgd.recordCommitLSNs(ctx, positions); err != nil { + return err + } + ledgerFlushDurationHistogram.Observe(time.Since(startedAt).Seconds()) + ledgerBackfilledCounter.Add(float64(len(positions))) + + confirmedLSN = confirmTo + + // Replaying a gap is only sound once the ledger has confirmed past the + // position sampled when the replay was planned: until then a transaction + // that has merely not been decoded yet is indistinguishable from one the + // gap swallowed. It is attempted once per attachment; a replay left + // undone (an interval older than the collection window, say) stays + // recorded and keeps failing watches, which is the safe outcome. + if heal != nil && confirmedLSN >= heal.target { + if err := pgd.healLedgerGap(ctx, heal); err != nil { + return err + } + heal = nil + } + + // One batch of pre-ledger history per flush, so filling in behind the + // ledger never competes with recording in front of it. + if backfilling { + more, err := pgd.backfillPreLedgerPositions(ctx) + if err != nil { + return err + } + backfilling = more + } + + // Sampling the lag costs a round-trip, and a trickle of writes flushes as + // often as every ledgerIdleFlushDelay, so it is sampled on the same + // cadence as the status updates rather than per flush. + if time.Now().After(nextLagSample) { + pgd.observeLedgerLag(ctx, confirmedLSN) + nextLagSample = time.Now().Add(statusInterval) + } + + return sendStatus() + } + + for { + if ctx.Err() != nil { + return nil + } + + if time.Now().After(nextStatusDeadline) { + if err := sendStatus(); err != nil { + return err + } + } + + deadline := nextStatusDeadline + if flushDeadline, pending := batch.flushDeadline(); pending && flushDeadline.Before(deadline) { + deadline = flushDeadline + } + + receiveCtx, cancelReceive := context.WithDeadline(ctx, deadline) + rawMsg, err := conn.ReceiveMessage(receiveCtx) + cancelReceive() + if err != nil { + if ctx.Err() != nil { + return nil + } + if pgconn.Timeout(err) { + // Either the batch is due or the status update is; both are + // handled at the top of the loop, so flush here and continue. + if err := flush(); err != nil { + return err + } + continue + } + return fmt.Errorf("the commit LSN ledger's replication stream failed: %w", err) + } + + switch msg := rawMsg.(type) { + case *pgproto3.CopyData: + switch msg.Data[0] { + case pglogrepl.PrimaryKeepaliveMessageByteID: + keepalive, err := pglogrepl.ParsePrimaryKeepaliveMessage(msg.Data[1:]) + if err != nil { + return fmt.Errorf("unable to parse the commit LSN ledger's keepalive message: %w", err) + } + // A keepalive's ServerWALEnd may be confirmed when, and only + // when, the ledger is idle (see canConfirmLedgerIdle): the + // walsender streams WAL in order and reports positions it has + // fully sent, so with nothing pending and no transaction open, + // everything at or below that position is decoded and recorded. + // Without this, a quiet stream pins WAL forever and the + // frontier never crosses stretches with nothing to record. + if canConfirmLedgerIdle(batch, decoder) && keepalive.ServerWALEnd > confirmedLSN { + confirmedLSN = keepalive.ServerWALEnd + } + if keepalive.ReplyRequested { + if err := flush(); err != nil { + return err + } + if err := sendStatus(); err != nil { + return err + } + } + + case pglogrepl.XLogDataByteID: + xld, err := pglogrepl.ParseXLogData(msg.Data[1:]) + if err != nil { + return fmt.Errorf("unable to parse the commit LSN ledger's replication data: %w", err) + } + + logicalMsg, err := pglogrepl.Parse(xld.WALData) + if err != nil { + return fmt.Errorf("unable to parse the commit LSN ledger's pgoutput message: %w", err) + } + + committed, err := decoder.handleMessage(logicalMsg) + if err != nil { + return err + } + if committed == nil { + continue + } + + if !committed.hasTransactionRow { + // Nothing to record: the ledger's own writes and any other + // transaction that touched no transaction row land here. + // Their WAL is confirmable immediately when the ledger is + // otherwise idle, and is covered by a later batch when not. + if canConfirmLedgerIdle(batch, decoder) && committed.endLSN > confirmedLSN { + confirmedLSN = committed.endLSN + } + continue + } + + batch.add(ledgerPosition{xid: committed.xid, commitLSN: committed.commitLSN}, committed.endLSN, time.Now()) + if batch.full() { + if err := flush(); err != nil { + return err + } + } + } + + case *pgproto3.ErrorResponse: + return fmt.Errorf("the commit LSN ledger's replication stream errored: %s", msg.Message) + + default: + // Other protocol messages carry nothing the ledger records. + } + } +} + +// recordCommitLSNs appends a batch of commit positions in a single transaction. +// A position for a transaction that has already been garbage collected is +// appended all the same and simply never read: it sits below every consumer's +// horizon, and the next collection pass removes it. +func (pgd *pgDatastore) recordCommitLSNs(ctx context.Context, positions []ledgerPosition) error { + xids := make([]string, 0, len(positions)) + commitLSNs := make([]string, 0, len(positions)) + for _, position := range positions { + xids = append(xids, strconv.FormatUint(position.xid.Uint64, 10)) + commitLSNs = append(commitLSNs, position.commitLSN.String()) + } + + if _, err := pgd.writePool.Exec(ctx, recordCommitLSNsQuery, xids, commitLSNs); err != nil { + return fmt.Errorf("unable to record commit LSNs: %w", err) + } + + return nil +} + +// observeLedgerLag reports how far the confirmed frontier trails the server's +// current WAL position, which is the WAL the slot is holding on to. +func (pgd *pgDatastore) observeLedgerLag(ctx context.Context, confirmedLSN pglogrepl.LSN) { + var currentText string + if err := pgd.writePool.QueryRow(ctx, "SELECT pg_current_wal_lsn()::text;").Scan(¤tText); err != nil { + return + } + + current, err := pglogrepl.ParseLSN(currentText) + if err != nil || current < confirmedLSN { + return + } + + ledgerLagBytesGauge.Set(float64(current - confirmedLSN)) +} + +// canConfirmLedgerIdle reports whether it is safe to confirm a stream position +// the ledger has reached but recorded nothing at. +// +// Confirming a position asserts "everything at or below it is recorded", so it +// is only safe when nothing contradicts that assertion. Two things do: pending +// positions, which sit below the stream position and are not yet durable, and +// an open transaction in the decoder, whose records have been seen but not yet +// turned into a pending position. Confirming past either silently loses the +// events between the confirmed position and what was actually recorded. +func canConfirmLedgerIdle(batch *ledgerBatch, decoder *ledgerDecoder) bool { + return !batch.pending() && !decoder.inTransaction() +} + +func newLedgerBatch(maxSize int, maxDelay time.Duration) *ledgerBatch { + return &ledgerBatch{ + maxSize: max(maxSize, 1), + maxDelay: max(maxDelay, ledgerIdleFlushDelay), + } +} + +// add records one transaction's commit position, along with the stream position +// that becomes confirmable once the batch is written. +func (b *ledgerBatch) add(position ledgerPosition, endLSN pglogrepl.LSN, now time.Time) { + if len(b.positions) == 0 { + b.startedAt = now + } + + b.positions = append(b.positions, position) + if endLSN > b.confirmLSN { + b.confirmLSN = endLSN + } +} + +// full reports whether the batch is worth writing on size alone. +func (b *ledgerBatch) full() bool { + return len(b.positions) >= b.maxSize +} + +// pending reports whether any recorded positions await a flush. +func (b *ledgerBatch) pending() bool { + return len(b.positions) > 0 +} + +// flushDeadline returns when the pending positions must be written, and whether +// any are pending at all. A quiet stream flushes promptly so that a Watch call +// waiting on the frontier is not held up; a busy one is bounded by maxDelay. +func (b *ledgerBatch) flushDeadline() (time.Time, bool) { + if len(b.positions) == 0 { + return time.Time{}, false + } + + idle := time.Now().Add(ledgerIdleFlushDelay) + if maximum := b.startedAt.Add(b.maxDelay); maximum.Before(idle) { + return maximum, true + } + return idle, true +} + +// take drains the batch, returning its positions and the position the slot may +// confirm once they are durable. +func (b *ledgerBatch) take() ([]ledgerPosition, pglogrepl.LSN) { + positions, confirmLSN := b.positions, b.confirmLSN + b.positions = nil + b.startedAt = time.Time{} + return positions, confirmLSN +} + +// inTransaction reports whether the decoder is between a BEGIN and its COMMIT. +func (d *ledgerDecoder) inTransaction() bool { + return d.current != nil +} + +func newLedgerDecoder() *ledgerDecoder { + typeMap := pgtype.NewMap() + RegisterTypes(typeMap) + + return &ledgerDecoder{ + relations: make(map[uint32]*pglogrepl.RelationMessage), + typeMap: typeMap, + } +} + +// handleMessage processes one decoded pgoutput message, returning a transaction +// when a COMMIT completes it. +func (d *ledgerDecoder) handleMessage(msg pglogrepl.Message) (*ledgerTransaction, error) { + switch m := msg.(type) { + case *pglogrepl.RelationMessage: + d.relations[m.RelationID] = m + + case *pglogrepl.BeginMessage: + d.current = &ledgerTransaction{commitLSN: m.FinalLSN} + + case *pglogrepl.InsertMessage: + if d.current == nil { + return nil, errors.New("the commit LSN ledger received an INSERT outside of a transaction") + } + relation, ok := d.relations[m.RelationID] + if !ok { + return nil, fmt.Errorf("the commit LSN ledger received an INSERT for unknown relation OID %d", m.RelationID) + } + if relation.RelationName != schema.TableTransaction { + // The publication only publishes the transaction table, so this can + // only happen against a publication someone else has widened. + return nil, nil + } + + row, err := decodeLogicalRow(d.typeMap, relation, m.Tuple, nil) + if err != nil { + return nil, fmt.Errorf("the commit LSN ledger could not decode a transaction row: %w", err) + } + xid, err := row.xid8Column(schema.ColXID) + if err != nil { + return nil, err + } + + d.current.hasTransactionRow = true + d.current.xid = xid + + case *pglogrepl.CommitMessage: + committed := d.current + if committed == nil { + return nil, errors.New("the commit LSN ledger received a COMMIT outside of a transaction") + } + d.current = nil + + // BEGIN carries the commit position the watch stamps its live revisions + // with, and COMMIT carries it again. They must agree, or the two paths a + // transaction reaches a consumer by would position it differently. + if committed.commitLSN != m.CommitLSN { + return nil, fmt.Errorf( + "the commit LSN ledger observed disagreeing commit positions for one transaction: BEGIN reported %s and COMMIT reported %s", + committed.commitLSN, m.CommitLSN) + } + committed.endLSN = m.TransactionEndLSN + return committed, nil + + default: + // Updates, deletes, truncations, origins and type messages carry no + // commit position to record. + } + + return nil, nil +} + +// isReplicationSlotInUseError reports whether the error is PostgreSQL refusing a +// second consumer for a replication slot, which is how the ledger elects its +// single writer. +func isReplicationSlotInUseError(err error) bool { + pgerr, ok := errors.AsType[*pgconn.PgError](err) + return ok && pgerr.Code == pgObjectInUseErr +} + +// connectLogicalReplication opens a pgconn connection in logical replication +// mode (replication=database) using the datastore's connection configuration. +func (pgd *pgDatastore) connectLogicalReplication(ctx context.Context) (*pgconn.PgConn, error) { + poolConfig, err := pgxpool.ParseConfig(pgd.dburl) + if err != nil { + return nil, err + } + + connConfig := poolConfig.ConnConfig.Copy() + connConfig.RuntimeParams["replication"] = "database" + + if pgd.credentialsProvider != nil { + connConfig.User, connConfig.Password, err = pgd.credentialsProvider.Get(ctx, fmt.Sprintf("%s:%d", connConfig.Host, connConfig.Port), connConfig.User) + if err != nil { + return nil, err + } + } + + return pgconn.ConnectConfig(ctx, &connConfig.Config) +} + +// ensurePublication creates the named publication over exactly the given tables +// if it does not exist, or verifies (and repairs) its table list if it does. +// publishOperations is the value of the publication's `publish` parameter, which +// is only applied at creation: an operator who has narrowed an existing +// publication is not overruled. +func (pgd *pgDatastore) ensurePublication(ctx context.Context, publicationName string, publicationTables []string, publishOperations string) error { + var exists bool + if err := pgd.writePool.QueryRow( + ctx, + "SELECT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = $1);", publicationName, + ).Scan(&exists); err != nil { + return fmt.Errorf("unable to check for publication %s: %w", publicationName, err) + } + + if !exists { + tables := make([]string, 0, len(publicationTables)) + for _, table := range publicationTables { + tables = append(tables, pgx.Identifier{table}.Sanitize()) + } + + createPublication := fmt.Sprintf( + "CREATE PUBLICATION %s FOR TABLE %s WITH (publish = '%s');", + pgx.Identifier{publicationName}.Sanitize(), + strings.Join(tables, ", "), + publishOperations, + ) + if _, err := pgd.writePool.Exec(ctx, createPublication); err != nil { + // Another instance may have raced us to create the publication. + if pgerr, ok := errors.AsType[*pgconn.PgError](err); ok && pgerr.Code == pgDuplicateObjectErr { + return nil + } + return fmt.Errorf("unable to create publication %s: %w", publicationName, err) + } + return nil + } + + rows, err := pgd.writePool.Query(ctx, + "SELECT tablename FROM pg_publication_tables WHERE pubname = $1;", publicationName) + if err != nil { + return fmt.Errorf("unable to list tables for publication %s: %w", publicationName, err) + } + defer rows.Close() + + published := make(map[string]struct{}) + for rows.Next() { + var tableName string + if err := rows.Scan(&tableName); err != nil { + return fmt.Errorf("unable to list tables for publication %s: %w", publicationName, err) + } + published[tableName] = struct{}{} + } + if rows.Err() != nil { + return fmt.Errorf("unable to list tables for publication %s: %w", publicationName, rows.Err()) + } + + for _, table := range publicationTables { + if _, ok := published[table]; !ok { + log.Ctx(ctx).Info().Str("table", table).Str("publication", publicationName).Msg("adding missing table to publication") + alterPublication := fmt.Sprintf( + "ALTER PUBLICATION %s ADD TABLE %s;", + pgx.Identifier{publicationName}.Sanitize(), + pgx.Identifier{table}.Sanitize(), + ) + if _, err := pgd.writePool.Exec(ctx, alterPublication); err != nil { + return fmt.Errorf("unable to add table %s to publication %s: %w", table, publicationName, err) + } + } + } + + return nil +} + +// quotePGOutputIdentifier double-quotes an identifier for use inside the value +// of a pgoutput option such as publication_names. +func quotePGOutputIdentifier(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} + +// quotePGOutputStringOption single-quotes a string value for use in a +// START_REPLICATION plugin option list. +func quotePGOutputStringOption(value string) string { + return "'" + strings.ReplaceAll(value, "'", "''") + "'" +} + +// registerLedgerMetrics registers the commit LSN ledger's and the cursor +// watch's metrics and returns them so they can be unregistered when the +// datastore closes. +func registerLedgerMetrics() ([]prometheus.Collector, error) { + collectors := []prometheus.Collector{ + ledgerActiveGauge, + ledgerBackfilledCounter, + ledgerFlushDurationHistogram, + ledgerLagBytesGauge, + ledgerAttachmentsCounter, + ledgerFailureCounter, + ledgerSlotRecreationsGauge, + ledgerGapsHealedCounter, + ledgerGapTransactionsHealedCounter, + ledgerPreLedgerBackfilledCounter, + watchLedgerWaitHistogram, + watchFrontierLagGauge, + watchPollDurationHistogram, + watchBatchTransactionsHistogram, + watchStaleRevisionCounter, + watchGapRejectionsCounter, + } + + for _, collector := range collectors { + if err := prometheus.Register(collector); err != nil { + return nil, fmt.Errorf("failed to register commit LSN ledger metric: %w", err) + } + } + + return collectors, nil +} diff --git a/internal/datastore/postgres/lsn_ledger_test.go b/internal/datastore/postgres/lsn_ledger_test.go new file mode 100644 index 0000000000..2f79dfdfdd --- /dev/null +++ b/internal/datastore/postgres/lsn_ledger_test.go @@ -0,0 +1,344 @@ +//go:build datastore && postgres + +package postgres + +import ( + "testing" + "time" + + "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5/pgconn" + "github.com/stretchr/testify/require" +) + +// TestLedgerBatchFlushPolicy covers when the commit LSN ledger decides a batch of +// recorded positions is worth writing, and what the slot may confirm once it is. +func TestLedgerBatchFlushPolicy(t *testing.T) { + position := func(xid uint64, commitLSN uint64) ledgerPosition { + return ledgerPosition{xid: NewXid8(xid), commitLSN: pglogrepl.LSN(commitLSN)} + } + + t.Run("size triggers a flush", func(t *testing.T) { + batch := newLedgerBatch(2, time.Second) + now := time.Now() + + require.False(t, batch.full(), "an empty batch is never full") + + batch.add(position(100, 0x100), 0x108, now) + require.False(t, batch.full()) + + batch.add(position(101, 0x110), 0x118, now) + require.True(t, batch.full(), "a batch at its size limit must flush") + }) + + t.Run("non-positive limits are clamped", func(t *testing.T) { + batch := newLedgerBatch(0, 0) + require.Equal(t, 1, batch.maxSize, "a batch must hold at least one position") + require.Equal(t, ledgerIdleFlushDelay, batch.maxDelay, "the maximum delay cannot be shorter than the idle flush delay") + + batch.add(position(100, 0x100), 0x108, time.Now()) + require.True(t, batch.full()) + }) + + t.Run("no flush deadline while the batch is empty", func(t *testing.T) { + batch := newLedgerBatch(8, time.Second) + + _, pending := batch.flushDeadline() + require.False(t, pending, "an empty batch has nothing to flush") + }) + + t.Run("a quiet stream flushes promptly, a busy one within the maximum delay", func(t *testing.T) { + const maxDelay = 50 * time.Millisecond + batch := newLedgerBatch(1024, maxDelay) + + startedAt := time.Now() + batch.add(position(100, 0x100), 0x108, startedAt) + + deadline, pending := batch.flushDeadline() + require.True(t, pending) + require.False(t, deadline.After(startedAt.Add(maxDelay)), + "the deadline must never exceed the age the batch is allowed to reach") + + // A batch whose allowance has already elapsed is due immediately rather + // than waiting out another idle delay. + stale := newLedgerBatch(1024, maxDelay) + stale.add(position(101, 0x110), 0x118, startedAt.Add(-time.Second)) + staleDeadline, pending := stale.flushDeadline() + require.True(t, pending) + require.True(t, staleDeadline.Before(time.Now()), "an overdue batch must be due immediately") + }) + + t.Run("take drains the batch and reports the position the slot may confirm", func(t *testing.T) { + batch := newLedgerBatch(8, time.Second) + now := time.Now() + + batch.add(position(100, 0x100), 0x108, now) + batch.add(position(101, 0x110), 0x118, now) + + positions, confirmTo := batch.take() + require.Len(t, positions, 2) + require.Equal(t, NewXid8(100), positions[0].xid) + require.Equal(t, pglogrepl.LSN(0x110), positions[1].commitLSN) + require.Equal(t, pglogrepl.LSN(0x118), confirmTo, "the slot may confirm through the last transaction's end position") + + drained, _ := batch.take() + require.Empty(t, drained, "a drained batch must not hand out its positions twice") + + _, pending := batch.flushDeadline() + require.False(t, pending, "a drained batch has nothing left to flush") + }) + + t.Run("the confirmable position never moves backwards", func(t *testing.T) { + batch := newLedgerBatch(8, time.Second) + now := time.Now() + + batch.add(position(100, 0x200), 0x208, now) + // A transaction whose WAL ends earlier must not pull the frontier back. + batch.add(position(101, 0x100), 0x108, now) + + _, confirmTo := batch.take() + require.Equal(t, pglogrepl.LSN(0x208), confirmTo) + }) +} + +// TestLedgerDecoding covers the minimal pgoutput decoding the ledger performs: the +// xid8 of each committed transaction, the agreement between the two commit +// positions pgoutput reports, and rejection of malformed message sequences. +func TestLedgerDecoding(t *testing.T) { + const ( + commitLSN = pglogrepl.LSN(0x1A00) + endLSN = pglogrepl.LSN(0x1A40) + ) + + transactionRelation := testTransactionRelation() + tupleRelation := testTupleRelation() + + begin := &pglogrepl.BeginMessage{Xid: 900, FinalLSN: commitLSN} + commit := &pglogrepl.CommitMessage{CommitLSN: commitLSN, TransactionEndLSN: endLSN} + + transactionInsert := &pglogrepl.InsertMessage{ + RelationID: transactionRelation.RelationID, + Tuple: transactionRowTuple("900"), + } + + testCases := []struct { + name string + messages []pglogrepl.Message + wantXid uint64 + wantNoTxRow bool + errContains string + }{ + { + name: "a transaction row insert yields its transaction ID", + messages: []pglogrepl.Message{transactionRelation, begin, transactionInsert, commit}, + wantXid: 900, + }, + { + // The ledger's own writes are updates, not inserts, so they reach it as + // transactions with nothing to record. + name: "a transaction with no transaction row has nothing to record", + messages: []pglogrepl.Message{transactionRelation, begin, commit}, + wantNoTxRow: true, + }, + { + // Only the transaction table is published, so anything else can only + // come from a publication someone widened; it is ignored rather than + // misread. + name: "inserts on other tables are ignored", + messages: []pglogrepl.Message{ + transactionRelation, tupleRelation, begin, + &pglogrepl.InsertMessage{RelationID: tupleRelation.RelationID, Tuple: relationshipTuple("subject", nullValue{}, testLiveSentinel)}, + transactionInsert, + commit, + }, + wantXid: 900, + }, + { + // The watch stamps live revisions with the position BEGIN reports, so + // a disagreement between the two would position the same transaction + // differently depending on which phase delivered it. + name: "disagreeing commit positions are rejected", + messages: []pglogrepl.Message{ + transactionRelation, begin, transactionInsert, + &pglogrepl.CommitMessage{CommitLSN: commitLSN + 8, TransactionEndLSN: endLSN}, + }, + errContains: "disagreeing commit positions", + }, + { + name: "an insert outside a transaction is rejected", + messages: []pglogrepl.Message{transactionRelation, transactionInsert}, + errContains: "INSERT outside of a transaction", + }, + { + name: "a commit outside a transaction is rejected", + messages: []pglogrepl.Message{commit}, + errContains: "COMMIT outside of a transaction", + }, + { + name: "an insert for an unknown relation is rejected", + messages: []pglogrepl.Message{begin, transactionInsert}, + errContains: "unknown relation OID", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + decoder := newLedgerDecoder() + + var committed *ledgerTransaction + var err error + for _, message := range tc.messages { + committed, err = decoder.handleMessage(message) + if err != nil { + break + } + } + + if tc.errContains != "" { + require.ErrorContains(t, err, tc.errContains) + return + } + require.NoError(t, err) + require.NotNil(t, committed, "the commit must complete a transaction") + + require.Equal(t, commitLSN, committed.commitLSN) + require.Equal(t, endLSN, committed.endLSN) + if tc.wantNoTxRow { + require.False(t, committed.hasTransactionRow) + return + } + require.True(t, committed.hasTransactionRow) + require.Equal(t, tc.wantXid, committed.xid.Uint64) + }) + } + + t.Run("a transaction is only reported once", func(t *testing.T) { + decoder := newLedgerDecoder() + for _, message := range []pglogrepl.Message{transactionRelation, begin, transactionInsert} { + committed, err := decoder.handleMessage(message) + require.NoError(t, err) + require.Nil(t, committed, "only a commit completes a transaction") + } + + committed, err := decoder.handleMessage(commit) + require.NoError(t, err) + require.NotNil(t, committed) + + _, err = decoder.handleMessage(commit) + require.ErrorContains(t, err, "COMMIT outside of a transaction") + }) +} + +// TestLedgerIdleConfirmSafety covers the condition under which the ledger may +// confirm a stream position it recorded nothing at. +// +// Confirming a position asserts that everything at or below it is recorded, and +// the cursor watch turns that assertion into a delivery bound. Confirming while +// anything is pending or in flight therefore does not merely retain less WAL: it +// advertises transactions as delivered that were never recorded, and the watch +// steps over them in silence. +func TestLedgerIdleConfirmSafety(t *testing.T) { + transactionRelation := testTransactionRelation() + + testCases := []struct { + name string + // prepare drives the batch and decoder into the state under test. + prepare func(batch *ledgerBatch, decoder *ledgerDecoder) + wantAllows bool + }{ + { + name: "an idle ledger may confirm what the stream reports", + prepare: func(*ledgerBatch, *ledgerDecoder) {}, + wantAllows: true, + }, + { + name: "a pending batch may not: its positions sit below the stream and are not durable", + prepare: func(batch *ledgerBatch, _ *ledgerDecoder) { + batch.add(ledgerPosition{xid: NewXid8(100), commitLSN: 0x100}, 0x108, time.Now()) + }, + }, + { + name: "an open transaction may not: its records are seen but not yet a position", + prepare: func(_ *ledgerBatch, decoder *ledgerDecoder) { + _, err := decoder.handleMessage(&pglogrepl.BeginMessage{Xid: 900, FinalLSN: 0x200}) + require.NoError(t, err) + }, + }, + { + name: "both at once may not", + prepare: func(batch *ledgerBatch, decoder *ledgerDecoder) { + batch.add(ledgerPosition{xid: NewXid8(100), commitLSN: 0x100}, 0x108, time.Now()) + _, err := decoder.handleMessage(&pglogrepl.BeginMessage{Xid: 900, FinalLSN: 0x200}) + require.NoError(t, err) + }, + }, + { + name: "a drained batch may again", + prepare: func(batch *ledgerBatch, _ *ledgerDecoder) { + batch.add(ledgerPosition{xid: NewXid8(100), commitLSN: 0x100}, 0x108, time.Now()) + batch.take() + }, + wantAllows: true, + }, + { + name: "a completed transaction may again", + prepare: func(_ *ledgerBatch, decoder *ledgerDecoder) { + for _, message := range []pglogrepl.Message{ + transactionRelation, + &pglogrepl.BeginMessage{Xid: 900, FinalLSN: 0x200}, + &pglogrepl.InsertMessage{RelationID: transactionRelation.RelationID, Tuple: transactionRowTuple("900")}, + &pglogrepl.CommitMessage{CommitLSN: 0x200, TransactionEndLSN: 0x240}, + } { + _, err := decoder.handleMessage(message) + require.NoError(t, err) + } + }, + wantAllows: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + batch := newLedgerBatch(8, time.Second) + decoder := newLedgerDecoder() + + tc.prepare(batch, decoder) + + require.Equal(t, tc.wantAllows, canConfirmLedgerIdle(batch, decoder)) + }) + } +} + +// TestLedgerSlotInUseDetection covers the error the ledger reads as "another +// instance already holds the slot", which is how it elects a single writer. +func TestLedgerSlotInUseDetection(t *testing.T) { + testCases := []struct { + name string + err error + wantR bool + }{ + { + name: "the slot is held by another session", + err: &pgconn.PgError{Code: pgObjectInUseErr, Message: `replication slot "spicedb_ledger" is active for PID 1`}, + wantR: true, + }, + { + name: "another PostgreSQL error is not slot contention", + err: &pgconn.PgError{Code: "42P01", Message: "relation does not exist"}, + }, + { + name: "a non-PostgreSQL error is not slot contention", + err: errLedgerGenesisMissing, + }, + { + name: "no error at all", + err: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.wantR, isReplicationSlotInUseError(tc.err)) + }) + } +} diff --git a/internal/datastore/postgres/migrations/zz_migration.0026_add_commit_lsn_ledger.go b/internal/datastore/postgres/migrations/zz_migration.0026_add_commit_lsn_ledger.go new file mode 100644 index 0000000000..a620de1b78 --- /dev/null +++ b/internal/datastore/postgres/migrations/zz_migration.0026_add_commit_lsn_ledger.go @@ -0,0 +1,87 @@ +package migrations + +import ( + "context" + + "github.com/jackc/pgx/v5" +) + +// commitLSNLedgerStatements create the three tables the commit LSN ledger owns. +// +// ledger_xid_lsn records, per transaction, the WAL position at which it +// committed. A transaction cannot write its own commit LSN, so the position is +// appended afterwards by the commit LSN ledger, which reads it out of the WAL. +// The watch reads this table to deliver transactions in true commit order. +// +// It is deliberately a side table rather than a column on +// relation_tuple_transaction. A column would have to be filled in by an UPDATE +// per write, and because the watch needs it indexed, that update could never be +// heap-only: every write would rewrite the ~100-byte transaction row, re-enter +// every index on it, and leave a dead tuple behind. Appending a narrow row +// here keeps relation_tuple_transaction insert/delete-only, which is what makes +// the feature affordable on a deployment retaining a day of high-rate writes. +// +// There is deliberately no foreign key to relation_tuple_transaction: the ledger +// appends positions from a replication stream, after those transactions +// committed and possibly after garbage collection has removed them. A position +// whose transaction is gone is inert, and readers join it away. +// +// ledger_state holds a single row. genesis_snapshot is the snapshot taken when +// the ledger's replication slot was first created, and distinguishes the two +// reasons a transaction can have no recorded position: one that committed before +// the ledger existed (visible in the snapshot, safely ordered ahead of +// everything recorded) from one lost to a slot recreation (not visible, and +// therefore unorderable). +// +// ledger_gap records WAL intervals the ledger never decoded, which exist when +// its replication slot was invalidated or dropped and had to be recreated. +// Transactions that committed inside such an interval have no recorded commit +// position, and after the recreation the ledger's frontier jumps past them, so +// nothing else can tell they were skipped. from_lsn is the last position known +// complete before the gap, and to_lsn the first position complete after it. A +// row whose to_lsn is the maximum pg_lsn marks a recreation still in progress +// (or one that crashed partway), and keeps failing watches until the recreation +// completes and bounds it. +var commitLSNLedgerStatements = []string{ + `CREATE TABLE ledger_xid_lsn ( + xid xid8 NOT NULL, + commit_lsn pg_lsn NOT NULL, + CONSTRAINT pk_ledger_xid_lsn PRIMARY KEY (xid));`, + + // The watch scans this index by position and reads the xid straight out of + // it, so its hot query never touches the heap. It is unique because a commit + // record occupies one position, which exactly one transaction wrote. + `CREATE UNIQUE INDEX ix_ledger_xid_lsn_by_lsn + ON ledger_xid_lsn (commit_lsn) INCLUDE (xid);`, + + `CREATE TABLE ledger_state ( + singleton BOOLEAN PRIMARY KEY DEFAULT true CHECK (singleton), + genesis_snapshot pg_snapshot NOT NULL, + genesis_lsn pg_lsn, + backfill_offset BIGINT NOT NULL DEFAULT 0, + backfill_complete BOOLEAN NOT NULL DEFAULT false, + slot_recreations BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT (NOW() AT TIME ZONE 'utc'));`, + + `CREATE TABLE ledger_gap ( + from_lsn pg_lsn NOT NULL, + to_lsn pg_lsn NOT NULL, + detected_at TIMESTAMPTZ NOT NULL DEFAULT (NOW() AT TIME ZONE 'utc'), + CONSTRAINT pk_ledger_gap PRIMARY KEY (from_lsn, to_lsn));`, +} + +func init() { + if err := DatabaseMigrations.Register("add-commit-lsn-ledger", "populate-schema-tables", + noNonatomicMigration, + func(ctx context.Context, tx pgx.Tx) error { + for _, stmt := range commitLSNLedgerStatements { + if _, err := tx.Exec(ctx, stmt); err != nil { + return err + } + } + + return nil + }); err != nil { + panic("failed to register migration: " + err.Error()) + } +} diff --git a/internal/datastore/postgres/options.go b/internal/datastore/postgres/options.go index e0ced6d95f..013acf5496 100644 --- a/internal/datastore/postgres/options.go +++ b/internal/datastore/postgres/options.go @@ -27,6 +27,18 @@ type postgresOptions struct { maxRetries uint8 filterMaximumIDCount uint16 + cursorWatchEnabled bool + watchBatchSize int + watchPollInterval time.Duration + logicalWatchStatusInterval time.Duration + + logicalWatchLedgerSlotName string + logicalWatchLedgerPublicationName string + logicalWatchLedgerBatchSize int + logicalWatchLedgerFlushMaxDelay time.Duration + logicalWatchLedgerRetryInterval time.Duration + logicalWatchLedgerWaitTimeout time.Duration + enablePrometheusStats bool analyzeBeforeStatistics bool gcEnabled bool @@ -82,6 +94,18 @@ const ( defaultFollowerReadDelay = 0 defaultRevisionHeartbeat = true defaultRelaxedIsolationLevel = false + + defaultCursorWatchEnabled = false + defaultWatchBatchSize = 1024 + defaultWatchPollInterval = 25 * time.Millisecond + defaultLogicalWatchStatusInterval = 10 * time.Second + + defaultLogicalWatchLedgerSlotName = "spicedb_ledger" + defaultLogicalWatchLedgerPublicationName = "spicedb_ledger" + defaultLogicalWatchLedgerBatchSize = 256 + defaultLogicalWatchLedgerFlushMaxDelay = time.Second + defaultLogicalWatchLedgerRetryInterval = 5 * time.Second + defaultLogicalWatchLedgerWaitTimeout = 30 * time.Second ) // Option provides the facility to configure how clients within the @@ -110,6 +134,17 @@ func generateConfig(options []Option) (postgresOptions, error) { followerReadDelay: defaultFollowerReadDelay, revisionHeartbeatEnabled: defaultRevisionHeartbeat, relaxedIsolationLevel: defaultRelaxedIsolationLevel, + cursorWatchEnabled: defaultCursorWatchEnabled, + watchBatchSize: defaultWatchBatchSize, + watchPollInterval: defaultWatchPollInterval, + logicalWatchStatusInterval: defaultLogicalWatchStatusInterval, + + logicalWatchLedgerSlotName: defaultLogicalWatchLedgerSlotName, + logicalWatchLedgerPublicationName: defaultLogicalWatchLedgerPublicationName, + logicalWatchLedgerBatchSize: defaultLogicalWatchLedgerBatchSize, + logicalWatchLedgerFlushMaxDelay: defaultLogicalWatchLedgerFlushMaxDelay, + logicalWatchLedgerRetryInterval: defaultLogicalWatchLedgerRetryInterval, + logicalWatchLedgerWaitTimeout: defaultLogicalWatchLedgerWaitTimeout, } for _, option := range options { @@ -459,6 +494,102 @@ func WithWatchDisabled(isDisabled bool) Option { return func(po *postgresOptions) { po.watchDisabled = isDisabled } } +// WithLogicalWatch enables the cursor-based Watch implementation, which +// delivers transactions in true commit order, keyed by the commit positions +// the commit LSN ledger records from the WAL through a logical replication +// slot (hence the name: logical replication is the operator-visible +// prerequisite). Revisions carry byte-sortable commit-LSN tokens. Requires the +// connected PostgreSQL server to run with wal_level=logical and the connected +// user to have the REPLICATION privilege. Only takes effect on primary +// (writable) datastores. +// +// Disabled by default; the polling watch remains the default implementation. +func WithLogicalWatch(isEnabled bool) Option { + return func(po *postgresOptions) { po.cursorWatchEnabled = isEnabled } +} + +// WatchBatchSize sets the number of transactions the cursor watch loads and +// emits per discovery poll, and per backfill batch. Consecutive full batches +// are drained without sleeping, so this bounds memory rather than throughput. +// +// This value defaults to 1024 transactions. +func WatchBatchSize(batchSize int) Option { + return func(po *postgresOptions) { po.watchBatchSize = batchSize } +} + +// WatchPollInterval sets how often the cursor watch polls its discovery query +// when the previous poll found nothing to deliver. It is the dominant term in +// commit-to-consumer latency, and each poll costs one indexed query plus one +// replication-slot catalog read per active watch. +// +// This value defaults to 25 milliseconds. +func WatchPollInterval(interval time.Duration) Option { + return func(po *postgresOptions) { po.watchPollInterval = interval } +} + +// LogicalWatchStatusInterval sets the interval at which the commit LSN ledger +// sends standby status updates (flush feedback) to the server. +// +// This value defaults to 10 seconds. +func LogicalWatchStatusInterval(interval time.Duration) Option { + return func(po *postgresOptions) { po.logicalWatchStatusInterval = interval } +} + +// LogicalWatchLedgerSlotName sets the name of the durable replication slot the +// commit LSN ledger consumes. The slot is created at startup if it does not +// exist, and is shared by every instance: exactly one holds it at a time. +// +// This value defaults to "spicedb_ledger". +func LogicalWatchLedgerSlotName(name string) Option { + return func(po *postgresOptions) { po.logicalWatchLedgerSlotName = name } +} + +// LogicalWatchLedgerPublicationName sets the name of the publication the commit +// LSN ledger consumes, which publishes inserts on the transactions table only. +// +// This value defaults to "spicedb_ledger". +func LogicalWatchLedgerPublicationName(name string) Option { + return func(po *postgresOptions) { po.logicalWatchLedgerPublicationName = name } +} + +// LogicalWatchLedgerBatchSize sets how many recorded commit positions the commit +// LSN ledger writes back per transaction. Larger batches amortize the write at +// the cost of a longer replay after a crash. +// +// This value defaults to 256 transactions. +func LogicalWatchLedgerBatchSize(batchSize int) Option { + return func(po *postgresOptions) { po.logicalWatchLedgerBatchSize = batchSize } +} + +// LogicalWatchLedgerFlushMaxDelay bounds how long the commit LSN ledger holds a +// partial batch when writes keep arriving. A quiet stream flushes promptly +// regardless, so this only takes effect under continuous load. +// +// This value defaults to 1 second. +func LogicalWatchLedgerFlushMaxDelay(delay time.Duration) Option { + return func(po *postgresOptions) { po.logicalWatchLedgerFlushMaxDelay = delay } +} + +// LogicalWatchLedgerRetryInterval sets how long an instance waits before trying +// to take over the commit LSN ledger's slot again. It bounds how long recording +// pauses after the holding instance dies. +// +// This value defaults to 5 seconds. +func LogicalWatchLedgerRetryInterval(interval time.Duration) Option { + return func(po *postgresOptions) { po.logicalWatchLedgerRetryInterval = interval } +} + +// LogicalWatchLedgerWaitTimeout bounds how long a starting Watch call waits for +// the commit LSN ledger to record the position of its own marker transaction, +// which is what proves every transaction it is about to replay has a recorded +// commit position. Exceeding it fails the Watch call with an explicit error +// rather than replaying transactions at unknown positions. +// +// This value defaults to 30 seconds. +func LogicalWatchLedgerWaitTimeout(timeout time.Duration) Option { + return func(po *postgresOptions) { po.logicalWatchLedgerWaitTimeout = timeout } +} + // WithRelaxedIsolationLevel relaxes the required Serializable isolation level to run SpiceDB on Postgres. // Please note this will defeat the new enemy problem and SpiceDB won't be able to provide strong consistency // guarantees. diff --git a/internal/datastore/postgres/pgoutput_row.go b/internal/datastore/postgres/pgoutput_row.go new file mode 100644 index 0000000000..bd3dc86e68 --- /dev/null +++ b/internal/datastore/postgres/pgoutput_row.go @@ -0,0 +1,123 @@ +package postgres + +import ( + "errors" + "fmt" + + "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5/pgtype" +) + +// pgoutput row decoding for the commit LSN ledger, the one remaining consumer +// of logical replication. The ledger only ever decodes the transaction table's +// xid column, but decoding is kept whole-row and codec-driven so that it reads +// values exactly the way pgx reads them over SQL. + +// logicalColumnValue is one column of a decoded row in `pgoutput` text format. +type logicalColumnValue struct { + // isNull indicates a SQL NULL. + isNull bool + // missing indicates an unchanged TOASTed value for which no old tuple + // value was available; the actual value is unknown. + missing bool + // dataType is the PostgreSQL type OID of the column, from the RELATION message. + dataType uint32 + text string +} + +// logicalRow is a decoded row keyed by column name, merged from the new tuple +// and (for unchanged TOASTed values) the old tuple. +type logicalRow struct { + tableName string + values map[string]logicalColumnValue + typeMap *pgtype.Map +} + +func decodeLogicalRow(typeMap *pgtype.Map, relation *pglogrepl.RelationMessage, tupleData *pglogrepl.TupleData, oldTupleData *pglogrepl.TupleData) (*logicalRow, error) { + if tupleData == nil { + return nil, fmt.Errorf("missing tuple data for table %s", relation.RelationName) + } + + if len(tupleData.Columns) != len(relation.Columns) { + return nil, fmt.Errorf("tuple data for table %s has %d columns, expected %d", relation.RelationName, len(tupleData.Columns), len(relation.Columns)) + } + + row := &logicalRow{ + tableName: relation.RelationName, + values: make(map[string]logicalColumnValue, len(relation.Columns)), + typeMap: typeMap, + } + + for index, column := range relation.Columns { + value, err := decodeLogicalColumn(column, tupleData.Columns[index]) + if err != nil { + return nil, fmt.Errorf("column %s of table %s: %w", column.Name, relation.RelationName, err) + } + + if value.missing && oldTupleData != nil && index < len(oldTupleData.Columns) { + // An unchanged TOASTed value is not sent in the new tuple; with a + // full replica identity the old tuple carries it instead. + oldValue, err := decodeLogicalColumn(column, oldTupleData.Columns[index]) + if err != nil { + return nil, fmt.Errorf("column %s of table %s: %w", column.Name, relation.RelationName, err) + } + if !oldValue.missing { + value = oldValue + } + } + + row.values[column.Name] = value + } + + return row, nil +} + +func decodeLogicalColumn(column *pglogrepl.RelationMessageColumn, tupleColumn *pglogrepl.TupleDataColumn) (logicalColumnValue, error) { + switch tupleColumn.DataType { + case pglogrepl.TupleDataTypeText: + return logicalColumnValue{dataType: column.DataType, text: string(tupleColumn.Data)}, nil + case pglogrepl.TupleDataTypeNull: + return logicalColumnValue{dataType: column.DataType, isNull: true}, nil + case pglogrepl.TupleDataTypeToast: + return logicalColumnValue{dataType: column.DataType, missing: true}, nil + case pglogrepl.TupleDataTypeBinary: + return logicalColumnValue{}, errors.New("unexpected binary-format tuple data") + default: + return logicalColumnValue{}, fmt.Errorf("unknown tuple data type %q", tupleColumn.DataType) + } +} + +func (r *logicalRow) column(name string) (logicalColumnValue, error) { + value, ok := r.values[name] + if !ok { + return logicalColumnValue{}, fmt.Errorf("column %s not found in replicated row for table %s", name, r.tableName) + } + if value.missing { + return logicalColumnValue{}, fmt.Errorf("column %s of table %s has an unchanged TOASTed value and no old tuple was available", name, r.tableName) + } + return value, nil +} + +// scanColumn decodes the named column's pgoutput text value into dst using the +// pgtype codec registered for the column's type OID. +func (r *logicalRow) scanColumn(name string, dst any) error { + value, err := r.column(name) + if err != nil { + return err + } + if value.isNull { + return fmt.Errorf("column %s of table %s is unexpectedly NULL", name, r.tableName) + } + if err := r.typeMap.Scan(value.dataType, pgtype.TextFormatCode, []byte(value.text), dst); err != nil { + return fmt.Errorf("unable to decode column %s of table %s: %w", name, r.tableName, err) + } + return nil +} + +func (r *logicalRow) xid8Column(name string) (xid8, error) { + var xid xid8 + if err := r.scanColumn(name, &xid); err != nil { + return xid8{}, err + } + return xid, nil +} diff --git a/internal/datastore/postgres/pgoutput_row_test.go b/internal/datastore/postgres/pgoutput_row_test.go new file mode 100644 index 0000000000..001197dad9 --- /dev/null +++ b/internal/datastore/postgres/pgoutput_row_test.go @@ -0,0 +1,231 @@ +//go:build datastore && postgres + +package postgres + +import ( + "testing" + "time" + + "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5/pgtype" + "github.com/stretchr/testify/require" + + "github.com/authzed/spicedb/pkg/spiceerrors" +) + +const ( + // testLiveSentinel is the text form of liveDeletedTxnID as it appears in + // replicated tuple data. + testLiveSentinel = "9223372036854775807" +) + +// column value markers for buildTuple +type ( + nullValue struct{} + toastValue struct{} +) + +// testColumn is a column definition for testRelation: a name plus the +// PostgreSQL type OID that a real RELATION message would carry. +type testColumn struct { + name string + oid uint32 +} + +// TestPGOutputColumnDecoding asserts that pgoutput text column values are +// decoded through the pgtype codecs registered for the column type OIDs carried +// in the RELATION message: the same decode path pgx uses when a row is scanned +// over SQL. The commit LSN ledger only reads the transaction table's xid, but it +// reads it out of the WAL rather than out of a result set, so the equivalence is +// what makes the two agree. +func TestPGOutputColumnDecoding(t *testing.T) { + transactionRelation := testTransactionRelation() + + testCases := []struct { + name string + relation *pglogrepl.RelationMessage + tuple *pglogrepl.TupleData + assert func(t *testing.T, row *logicalRow) + errContains string + }{ + { + name: "xid8 decodes to the transaction ID", + relation: transactionRelation, + tuple: transactionRowTuple("4294967297"), + assert: func(t *testing.T, row *logicalRow) { + xid, err := row.xid8Column("xid") + require.NoError(t, err) + require.Equal(t, uint64(4294967297), xid.Uint64, "an epoch-extended xid8 must survive decoding") + }, + }, + { + name: "a snapshot decodes through its registered codec", + relation: transactionRelation, + tuple: transactionRowTuple("900"), + assert: func(t *testing.T, row *logicalRow) { + var snapshot pgSnapshot + require.NoError(t, row.scanColumn("snapshot", &snapshot)) + require.Equal(t, uint64(900), snapshot.xmin) + }, + }, + { + name: "a timestamp decodes to the instant it names", + relation: transactionRelation, + tuple: transactionRowTuple("900"), + assert: func(t *testing.T, row *logicalRow) { + var timestamp time.Time + require.NoError(t, row.scanColumn("timestamp", ×tamp)) + require.Equal(t, "2026-07-22 10:11:12.123456 +0000 UTC", timestamp.UTC().String()) + }, + }, + { + name: "a NULL column is reported rather than decoded", + relation: transactionRelation, + tuple: buildTuple("1", "900", "2026-07-22 10:11:12.123456", nullValue{}, "900:900:"), + assert: func(t *testing.T, row *logicalRow) { + var metadata map[string]any + require.ErrorContains(t, row.scanColumn("metadata", &metadata), "unexpectedly NULL") + }, + }, + { + name: "an unresolved TOASTed value is an error, not a silent zero", + relation: transactionRelation, + tuple: buildTuple("1", toastValue{}, "2026-07-22 10:11:12.123456", `{}`, "900:900:"), + assert: func(t *testing.T, row *logicalRow) { + _, err := row.xid8Column("xid") + require.ErrorContains(t, err, "unchanged TOASTed value") + }, + }, + { + name: "a column the row does not have is an error", + relation: transactionRelation, + tuple: transactionRowTuple("900"), + assert: func(t *testing.T, row *logicalRow) { + _, err := row.xid8Column("nonexistent") + require.ErrorContains(t, err, "not found") + }, + }, + { + name: "a tuple whose width disagrees with the relation is rejected", + relation: transactionRelation, + tuple: buildTuple("1", "900"), + errContains: "expected 5", + }, + { + name: "missing tuple data is rejected", + relation: transactionRelation, + errContains: "missing tuple data", + }, + } + + typeMap := pgtype.NewMap() + RegisterTypes(typeMap) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + row, err := decodeLogicalRow(typeMap, tc.relation, tc.tuple, nil) + if tc.errContains != "" { + require.ErrorContains(t, err, tc.errContains) + return + } + require.NoError(t, err) + tc.assert(t, row) + }) + } + + t.Run("binary-format tuple data is rejected", func(t *testing.T) { + _, err := decodeLogicalColumn( + &pglogrepl.RelationMessageColumn{Name: "xid", DataType: xid8TypeOID}, + &pglogrepl.TupleDataColumn{DataType: pglogrepl.TupleDataTypeBinary}, + ) + require.ErrorContains(t, err, "binary-format") + }) + + t.Run("an unchanged TOASTed value falls back to the old tuple", func(t *testing.T) { + row, err := decodeLogicalRow( + typeMap, transactionRelation, + buildTuple("1", toastValue{}, "2026-07-22 10:11:12.123456", `{}`, "900:900:"), + transactionRowTuple("900"), + ) + require.NoError(t, err) + + xid, err := row.xid8Column("xid") + require.NoError(t, err) + require.Equal(t, uint64(900), xid.Uint64) + }) +} + +func testTransactionRelation() *pglogrepl.RelationMessage { + return testRelation( + 1, "relation_tuple_transaction", + testColumn{"id", pgtype.Int8OID}, + testColumn{"xid", xid8TypeOID}, + testColumn{"timestamp", pgtype.TimestampOID}, + testColumn{"metadata", pgtype.JSONBOID}, + testColumn{"snapshot", pgSnapshotTypeOID}, + ) +} + +func testTupleRelation() *pglogrepl.RelationMessage { + return testRelation( + 2, "relation_tuple", + testColumn{"namespace", pgtype.VarcharOID}, + testColumn{"object_id", pgtype.VarcharOID}, + testColumn{"relation", pgtype.VarcharOID}, + testColumn{"userset_namespace", pgtype.VarcharOID}, + testColumn{"userset_object_id", pgtype.VarcharOID}, + testColumn{"userset_relation", pgtype.VarcharOID}, + testColumn{"caveat_name", pgtype.VarcharOID}, + testColumn{"caveat_context", pgtype.JSONBOID}, + testColumn{"expiration", pgtype.TimestamptzOID}, + testColumn{"created_xid", xid8TypeOID}, + testColumn{"deleted_xid", xid8TypeOID}, + ) +} + +func testRelation(id uint32, name string, columns ...testColumn) *pglogrepl.RelationMessage { + messageColumns := make([]*pglogrepl.RelationMessageColumn, 0, len(columns)) + for _, column := range columns { + messageColumns = append(messageColumns, &pglogrepl.RelationMessageColumn{Name: column.name, DataType: column.oid}) + } + + return &pglogrepl.RelationMessage{ + RelationID: id, + Namespace: "public", + RelationName: name, + ColumnNum: spiceerrors.MustSafecast[uint16](len(messageColumns)), + Columns: messageColumns, + } +} + +func buildTuple(values ...any) *pglogrepl.TupleData { + columns := make([]*pglogrepl.TupleDataColumn, 0, len(values)) + for _, value := range values { + switch v := value.(type) { + case nullValue: + columns = append(columns, &pglogrepl.TupleDataColumn{DataType: pglogrepl.TupleDataTypeNull}) + case toastValue: + columns = append(columns, &pglogrepl.TupleDataColumn{DataType: pglogrepl.TupleDataTypeToast}) + case string: + columns = append(columns, &pglogrepl.TupleDataColumn{DataType: pglogrepl.TupleDataTypeText, Data: []byte(v)}) + default: + panic("unsupported test tuple value") + } + } + + return &pglogrepl.TupleData{ + ColumnNum: spiceerrors.MustSafecast[uint16](len(columns)), + Columns: columns, + } +} + +func transactionRowTuple(xid string) *pglogrepl.TupleData { + return buildTuple("1", xid, "2026-07-22 10:11:12.123456", `{"purpose":"testing"}`, xid+":"+xid+":") +} + +func relationshipTuple(subjectObjectID string, caveatContext any, deletedXid string) *pglogrepl.TupleData { + return buildTuple( + "document", "doc1", "viewer", "user", subjectObjectID, "...", + "somecaveat", caveatContext, nullValue{}, "900", deletedXid, + ) +} diff --git a/internal/datastore/postgres/postgres.go b/internal/datastore/postgres/postgres.go index 8ac1fbf88e..46f4b6a7fa 100644 --- a/internal/datastore/postgres/postgres.go +++ b/internal/datastore/postgres/postgres.go @@ -231,6 +231,12 @@ func newPostgresDatastore( return nil, err } + cursorWatchEnabled := config.cursorWatchEnabled && isPrimary && !config.watchDisabled + + // Both watches require track_commit_timestamp. The polling watch orders every + // revision by it; the cursor watch orders by recorded commit LSN, but replays + // a gap the ledger could not record in true commit order, which is the one + // thing only a commit timestamp can supply once the WAL holding it is gone. watchEnabled := trackTSOn == "on" && !config.watchDisabled if !watchEnabled { if config.watchDisabled { @@ -328,6 +334,17 @@ func newPostgresDatastore( schema: *schema.Schema(config.columnOptimizationOption, false), quantizationPeriodNanos: quantizationPeriodNanos, isolationLevel: isolationLevel, + cursorWatchEnabled: cursorWatchEnabled, + watchBatchSize: config.watchBatchSize, + watchPollInterval: config.watchPollInterval, + logicalWatchStatusInterval: config.logicalWatchStatusInterval, + + logicalWatchLedgerSlotName: config.logicalWatchLedgerSlotName, + logicalWatchLedgerPublicationName: config.logicalWatchLedgerPublicationName, + logicalWatchLedgerBatchSize: config.logicalWatchLedgerBatchSize, + logicalWatchLedgerFlushMaxDelay: config.logicalWatchLedgerFlushMaxDelay, + logicalWatchLedgerRetryInterval: config.logicalWatchLedgerRetryInterval, + logicalWatchLedgerWaitTimeout: config.logicalWatchLedgerWaitTimeout, } if isPrimary && config.readStrictMode { @@ -338,6 +355,17 @@ func newPostgresDatastore( datastore.writePool = pgxcommon.MustNewInterceptorPooler(writePool, config.queryInterceptor) } + if cursorWatchEnabled { + if err := datastore.prepareCursorWatch(initializationContext); err != nil { + return nil, fmt.Errorf("failed to prepare the cursor watch: %w", err) + } + } else if isPrimary { + // The feature being off does not remove the durable ledger slot an + // earlier configuration may have left behind, and nothing else would + // notice it retaining WAL. + datastore.warnIfAbandonedLedgerSlot(initializationContext) + } + datastore.SetOptimizedRevisionFunc(datastore.optimizedRevisionFunc) // Start a goroutine for garbage collection and the revision heartbeat. @@ -349,6 +377,16 @@ func newPostgresDatastore( }) } + // The commit LSN ledger records the commit position of every write, + // which is the order the cursor watch delivers in and the frontier it + // bounds delivery by. Every instance runs it; the replication slot + // admits one at a time. + if cursorWatchEnabled { + datastore.workerGroup.Go(func() error { + return datastore.runCommitLSNLedger(datastore.workerCtx) + }) + } + if datastore.gcInterval > 0*time.Minute && config.gcEnabled { datastore.workerGroup.Go(func() error { return startGarbageCollector( @@ -402,6 +440,24 @@ type pgDatastore struct { filterMaximumIDCount uint16 quantizationPeriodNanos int64 isolationLevel pgx.TxIsoLevel + + cursorWatchEnabled bool + watchBatchSize int + watchPollInterval time.Duration + logicalWatchStatusInterval time.Duration + + logicalWatchLedgerSlotName string + logicalWatchLedgerPublicationName string + logicalWatchLedgerBatchSize int + logicalWatchLedgerFlushMaxDelay time.Duration + logicalWatchLedgerRetryInterval time.Duration + logicalWatchLedgerWaitTimeout time.Duration + + // ledgerSlotName is logicalWatchLedgerSlotName qualified for this database, + // and ledgerDatabase is the database it decodes. Both are resolved once, + // while the logical watch is being prepared, before any worker starts. + ledgerSlotName string + ledgerDatabase string } func (pgd *pgDatastore) MetricsID() (string, error) { @@ -715,12 +771,17 @@ func (pgd *pgDatastore) OfflineFeatures() (*datastore.Features, error) { watchStatus = datastore.FeatureSupported } + watchEmitsImmediatelyStatus := datastore.FeatureUnsupported + if pgd.watchEnabled && pgd.cursorWatchEnabled { + watchEmitsImmediatelyStatus = datastore.FeatureSupported + } + return &datastore.Features{ Watch: datastore.Feature{ Status: watchStatus, }, WatchEmitsImmediately: datastore.Feature{ - Status: datastore.FeatureUnsupported, + Status: watchEmitsImmediatelyStatus, }, IntegrityData: datastore.Feature{ Status: datastore.FeatureUnsupported, @@ -843,6 +904,12 @@ func registerAndReturnPrometheusCollectors(replicaIndex int, isPrimary bool, rea return collectors, err } collectors = append(collectors, gcCollectors...) + + ledgerCollectors, err := registerLedgerMetrics() + if err != nil { + return collectors, err + } + collectors = append(collectors, ledgerCollectors...) } return collectors, nil diff --git a/internal/datastore/postgres/postgres_shared_test.go b/internal/datastore/postgres/postgres_shared_test.go index e9abdfe985..3cdd7e7583 100644 --- a/internal/datastore/postgres/postgres_shared_test.go +++ b/internal/datastore/postgres/postgres_shared_test.go @@ -1695,6 +1695,11 @@ func GCQueriesServedByExpectedIndexes(t *testing.T, _ testdatastore.RunningEngin case strings.HasPrefix(explanation, "Delete on namespace_config"): fallthrough + // Recorded commit positions are collected alongside the transactions + // they describe, served by the position table's primary key. + case strings.HasPrefix(explanation, "Delete on ledger_xid_lsn"): + fallthrough + case strings.HasPrefix(explanation, "Delete on relation_tuple"): require.Contains(explanation, "Index Scan") diff --git a/internal/datastore/postgres/revisions.go b/internal/datastore/postgres/revisions.go index b800346a4a..4c3e18aef7 100644 --- a/internal/datastore/postgres/revisions.go +++ b/internal/datastore/postgres/revisions.go @@ -25,6 +25,17 @@ const ( errCheckRevision = "unable to check revision: %w" errRevisionFormat = "invalid revision format: %w" + // lsnRevisionSeparator separates the fixed-width hex-encoded commit LSN prefix + // from the base64-encoded snapshot proto in the string form of an LSN-carrying + // revision. This character should not be part of the standard base64 alphabet, + // so the two forms cannot be confused. + lsnRevisionSeparator = '.' + + // lsnHexLength is the number of hex characters used to encode the commit LSN in + // the string form of an LSN-carrying revision. Fixed-width, big-endian hex means + // lexicographic ordering of the string prefix matches the numeric LSN ordering. + lsnHexLength = 16 + // %[1] Name of xid column // %[2] Relationship tuple transaction table // %[3] Name of timestamp column @@ -195,13 +206,23 @@ func (pgd *pgDatastore) CheckRevision(ctx context.Context, revisionRaw datastore return nil } -// RevisionFromString reverses the encoding process performed by MarshalBinary and String. +// RevisionFromString reverses the encoding process performed by String. Note that +// MarshalBinary covers only the snapshot proto. The position prefix of a +// position-carrying revision is added by String and is not part of the binary form. func (pgd *pgDatastore) RevisionFromString(revisionStr string) (datastore.Revision, error) { return ParseRevisionString(revisionStr) } // ParseRevisionString parses a revision string into a Postgres revision. func ParseRevisionString(revisionStr string) (rev datastore.Revision, err error) { + lsnRev, isPositioned, err := parseLSNRevisionString(revisionStr) + if err != nil { + return datastore.NoRevision, err + } + if isPositioned { + return lsnRev, nil + } + rev, err = parseRevisionProto(revisionStr) if err != nil { decimalRev, decimalErr := parseRevisionDecimal(revisionStr) @@ -214,6 +235,52 @@ func ParseRevisionString(revisionStr string) (rev datastore.Revision, err error) return rev, err } +// parseLSNRevisionString parses the string form of a position-carrying revision. +// It consists of a fixed-width hex-encoded commit LSN, a separator, and the +// base64 proto form of the snapshot revision. +func parseLSNRevisionString(revisionStr string) (postgresRevision, bool, error) { + if len(revisionStr) <= lsnHexLength || revisionStr[lsnHexLength] != lsnRevisionSeparator { + return postgresRevision{}, false, nil + } + + if len(revisionStr) < lsnHexLength+2 { + return postgresRevision{}, true, fmt.Errorf( + errRevisionFormat, + errors.New("missing revision payload after commit LSN prefix"), + ) + } + + lsn, err := strconv.ParseUint(revisionStr[:lsnHexLength], 16, 64) + if err != nil { + return postgresRevision{}, true, fmt.Errorf( + errRevisionFormat, + fmt.Errorf("invalid commit LSN prefix: %w", err), + ) + } + if lsn == 0 { + return postgresRevision{}, true, fmt.Errorf( + errRevisionFormat, + errors.New("commit LSN must be non-zero"), + ) + } + + parsed, err := parseRevisionProto(revisionStr[lsnHexLength+1:]) + if err != nil { + return postgresRevision{}, true, err + } + + rev, ok := parsed.(postgresRevision) + if !ok { + return postgresRevision{}, true, fmt.Errorf( + errRevisionFormat, + errors.New("decoded revision is not a Postgres revision"), + ) + } + + rev.optionalCommitLSN = lsn + return rev, true, nil +} + func parseRevisionProto(revisionStr string) (datastore.Revision, error) { protoBytes, err := base64.StdEncoding.DecodeString(revisionStr) if err != nil { @@ -351,15 +418,38 @@ type postgresRevision struct { // be overlapping) optionalInexactNanosTimestamp uint64 optionalMetadata dscommon.TransactionMetadata + + // optionalCommitLSN is the WAL commit LSN of the transaction that produced this + // revision, when known. It is only set on revisions emitted by the logical + // replication watch, where it provides a byte-sortable total commit order that + // the MVCC snapshot (a partial order) cannot. When both revisions in a + // comparison carry an LSN, ordering is by LSN, otherwise snapshot semantics + // are used. Note that zero is not a valid PostgreSQL WAL position. + // + // The value is a property of the transaction, not of the watch call that + // emitted it: the commit LSN ledger records it once, from the WAL, and both + // the live stream and the catch-up replay report that same value. + optionalCommitLSN uint64 } +// ByteSortable reports whether this revision carries a commit position, in which +// case the fixed-width hex prefix of its String form sorts lexicographically in +// commit order. func (pr postgresRevision) ByteSortable() bool { - return false + return pr.optionalCommitLSN != 0 } func (pr postgresRevision) Equal(rhsRaw datastore.Revision) bool { rhs, ok := rhsRaw.(postgresRevision) - return ok && pr.snapshot.Equal(rhs.snapshot) + if !ok { + return false + } + + if pr.ByteSortable() && rhs.ByteSortable() { + return pr.optionalCommitLSN == rhs.optionalCommitLSN + } + + return pr.snapshot.Equal(rhs.snapshot) } func (pr postgresRevision) GreaterThan(rhsRaw datastore.Revision) bool { @@ -368,20 +458,45 @@ func (pr postgresRevision) GreaterThan(rhsRaw datastore.Revision) bool { } rhs, ok := rhsRaw.(postgresRevision) - return ok && pr.snapshot.GreaterThan(rhs.snapshot) + if !ok { + return false + } + + if pr.ByteSortable() && rhs.ByteSortable() { + return pr.optionalCommitLSN > rhs.optionalCommitLSN + } + + return pr.snapshot.GreaterThan(rhs.snapshot) } func (pr postgresRevision) LessThan(rhsRaw datastore.Revision) bool { rhs, ok := rhsRaw.(postgresRevision) - return ok && pr.snapshot.LessThan(rhs.snapshot) + if !ok { + return false + } + + if pr.ByteSortable() && rhs.ByteSortable() { + return pr.optionalCommitLSN < rhs.optionalCommitLSN + } + + return pr.snapshot.LessThan(rhs.snapshot) } func (pr postgresRevision) DebugString() string { + if pr.optionalCommitLSN != 0 { + return fmt.Sprintf("%s@%X/%X", pr.snapshot.String(), pr.optionalCommitLSN>>32, pr.optionalCommitLSN&0xFFFFFFFF) + } + return pr.snapshot.String() } func (pr postgresRevision) String() string { - return base64.StdEncoding.EncodeToString(pr.mustMarshalBinary()) + encoded := base64.StdEncoding.EncodeToString(pr.mustMarshalBinary()) + if pr.optionalCommitLSN != 0 { + return fmt.Sprintf("%016x", pr.optionalCommitLSN) + string(lsnRevisionSeparator) + encoded + } + + return encoded } func (pr postgresRevision) mustMarshalBinary() []byte { diff --git a/internal/datastore/postgres/revisions_position_test.go b/internal/datastore/postgres/revisions_position_test.go new file mode 100644 index 0000000000..78dc747b43 --- /dev/null +++ b/internal/datastore/postgres/revisions_position_test.go @@ -0,0 +1,217 @@ +//go:build datastore && postgres + +package postgres + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" + + "github.com/authzed/spicedb/pkg/datastore" + core "github.com/authzed/spicedb/pkg/proto/core/v1" + "github.com/authzed/spicedb/pkg/tuple" +) + +// TestPositionedRevisionEncoding asserts the string round-trip and +// byte-sortability of revisions carrying a commit position. +func TestPositionedRevisionEncoding(t *testing.T) { + testCases := []struct { + name string + revision postgresRevision + wantByteSortable bool + }{ + { + name: "a revision with a commit position", + revision: postgresRevision{ + snapshot: pgSnapshot{xmin: 1000, xmax: 1005, xipList: []uint64{1002, 1003}}, + optionalTxID: NewXid8(1004), + optionalCommitLSN: 0x16CD3F0000028, + }, + wantByteSortable: true, + }, + { + name: "a revision delivered from a different cursor carries the same position", + revision: postgresRevision{ + snapshot: pgSnapshot{xmin: 1000, xmax: 1005, xipList: []uint64{1002, 1003}}, + optionalTxID: NewXid8(1001), + optionalCommitLSN: 0x16CD3F0000028, + }, + wantByteSortable: true, + }, + { + name: "a revision without a position keeps the legacy encoding", + revision: postgresRevision{snapshot: pgSnapshot{xmin: 5, xmax: 5}}, + wantByteSortable: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.wantByteSortable, tc.revision.ByteSortable()) + + parsed, err := ParseRevisionString(tc.revision.String()) + require.NoError(t, err) + + parsedRev, ok := parsed.(postgresRevision) + require.True(t, ok) + require.Equal(t, tc.revision.optionalCommitLSN, parsedRev.optionalCommitLSN) + require.Equal(t, tc.revision.snapshot, parsedRev.snapshot) + require.Equal(t, tc.revision.optionalTxID, parsedRev.optionalTxID) + require.True(t, tc.revision.Equal(parsedRev)) + }) + } + + t.Run("string ordering matches commit position ordering", func(t *testing.T) { + revisions := []postgresRevision{ + {snapshot: pgSnapshot{xmin: 90, xmax: 90}, optionalCommitLSN: 0xFF000000FF}, + {snapshot: pgSnapshot{xmin: 10, xmax: 10}, optionalCommitLSN: 1}, + {snapshot: pgSnapshot{xmin: 50, xmax: 50}, optionalCommitLSN: 0x1000000000000000}, + {snapshot: pgSnapshot{xmin: 20, xmax: 20}, optionalCommitLSN: 0x20}, + {snapshot: pgSnapshot{xmin: 30, xmax: 30}, optionalCommitLSN: 0x21}, + {snapshot: pgSnapshot{xmin: 40, xmax: 40}, optionalCommitLSN: 0x22}, + } + + asStrings := make([]string, 0, len(revisions)) + for _, rev := range revisions { + asStrings = append(asStrings, rev.String()) + } + sort.Strings(asStrings) + + sort.Slice(revisions, func(i, j int) bool { + return revisions[i].optionalCommitLSN < revisions[j].optionalCommitLSN + }) + + for index, rev := range revisions { + require.Equal(t, rev.String(), asStrings[index], "byte ordering of revision strings must match position ordering") + } + }) + + // The same transaction reported by two watch calls is the same token: the + // position belongs to the transaction, not to the call that delivered it. + t.Run("identical transactions produce identical tokens", func(t *testing.T) { + delivered := postgresRevision{ + snapshot: pgSnapshot{xmin: 700, xmax: 702, xipList: []uint64{701}}, + optionalTxID: NewXid8(700), + optionalInexactNanosTimestamp: 1_700_000_000_000_000_000, + optionalCommitLSN: 0x3F00A8, + } + redelivered := delivered + + require.Equal(t, delivered.String(), redelivered.String()) + require.True(t, delivered.Equal(redelivered)) + }) +} + +// TestPositionedRevisionOrdering asserts position-first comparison semantics, +// with snapshot fallback when either side lacks a position. +func TestPositionedRevisionOrdering(t *testing.T) { + testCases := []struct { + name string + lhs postgresRevision + rhs datastore.Revision + wantGreater bool + wantLess bool + wantEqual bool + }{ + { + name: "a later position is greater regardless of snapshots", + lhs: postgresRevision{snapshot: pgSnapshot{xmin: 90, xmax: 90}, optionalCommitLSN: 600}, + rhs: postgresRevision{snapshot: pgSnapshot{xmin: 100, xmax: 100}, optionalCommitLSN: 500}, + wantGreater: true, + }, + { + name: "an earlier position is less", + lhs: postgresRevision{snapshot: pgSnapshot{xmin: 100, xmax: 100}, optionalCommitLSN: 500}, + rhs: postgresRevision{snapshot: pgSnapshot{xmin: 90, xmax: 90}, optionalCommitLSN: 600}, + wantLess: true, + }, + { + name: "the same position is equal", + lhs: postgresRevision{snapshot: pgSnapshot{xmin: 100, xmax: 100}, optionalCommitLSN: 500}, + rhs: postgresRevision{snapshot: pgSnapshot{xmin: 100, xmax: 100}, optionalCommitLSN: 500}, + wantEqual: true, + }, + { + name: "snapshot semantics when either side lacks a position", + lhs: postgresRevision{snapshot: pgSnapshot{xmin: 100, xmax: 100}, optionalCommitLSN: 500}, + rhs: postgresRevision{snapshot: pgSnapshot{xmin: 100, xmax: 100}}, + wantEqual: true, + }, + { + name: "always greater than NoRevision", + lhs: postgresRevision{snapshot: pgSnapshot{xmin: 100, xmax: 100}, optionalCommitLSN: 500}, + rhs: datastore.NoRevision, + wantGreater: true, + }, + { + // An overlapping transaction pair: each snapshot knows its own + // transaction settled and has the other in flight, so the snapshots + // abstain and the commit positions supply the order. + name: "overlapping transactions are ordered by commit position", + lhs: postgresRevision{snapshot: pgSnapshot{xmin: 100, xmax: 102, xipList: []uint64{100}}, optionalCommitLSN: 0x740}, + rhs: postgresRevision{snapshot: pgSnapshot{xmin: 101, xmax: 101}, optionalCommitLSN: 0x500}, + wantGreater: true, + }, + { + // Without positions the same pair is incomparable in every + // direction, because the snapshot partial order abstains. + name: "overlapping transactions without positions are incomparable", + lhs: postgresRevision{snapshot: pgSnapshot{xmin: 100, xmax: 102, xipList: []uint64{100}}}, + rhs: postgresRevision{snapshot: pgSnapshot{xmin: 101, xmax: 101}}, + }, + { + name: "an earlier transaction sorts below a later one", + lhs: postgresRevision{snapshot: pgSnapshot{xmin: 90, xmax: 90}, optionalCommitLSN: 0x500}, + rhs: postgresRevision{snapshot: pgSnapshot{xmin: 100, xmax: 100}, optionalCommitLSN: 0x740}, + wantLess: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.wantGreater, tc.lhs.GreaterThan(tc.rhs)) + require.Equal(t, tc.wantLess, tc.lhs.LessThan(tc.rhs)) + require.Equal(t, tc.wantEqual, tc.lhs.Equal(tc.rhs)) + }) + } +} + +// TestDecomposeRevisionChanges asserts that an assembled change is split into +// independent single-item events that all share its revision, which is what the +// EmitImmediatelyStrategy delivers. +func TestDecomposeRevisionChanges(t *testing.T) { + metadata, err := structpb.NewStruct(map[string]any{"origin": "backfill"}) + require.NoError(t, err) + + revision := postgresRevision{snapshot: pgSnapshot{xmin: 10, xmax: 10}, optionalCommitLSN: 0x480} + atoms := decomposeRevisionChanges(datastore.RevisionChanges{ + Revision: revision, + RelationshipChanges: []tuple.RelationshipUpdate{ + tuple.Touch(tuple.MustParse("document:doc1#viewer@user:alice")), + }, + ChangedDefinitions: []datastore.SchemaDefinition{&core.NamespaceDefinition{Name: "somenamespace"}}, + DeletedNamespaces: []string{"deletedns"}, + DeletedCaveats: []string{"deletedcaveat"}, + Metadatas: []*structpb.Struct{metadata}, + }) + + require.Len(t, atoms, 5) + require.Len(t, atoms[0].RelationshipChanges, 1) + require.Equal(t, "somenamespace", atoms[1].ChangedDefinitions[0].GetName()) + require.Equal(t, []string{"deletedns"}, atoms[2].DeletedNamespaces) + require.Equal(t, []string{"deletedcaveat"}, atoms[3].DeletedCaveats) + require.Len(t, atoms[4].Metadatas, 1) + for _, atom := range atoms { + require.True(t, atom.Revision.Equal(revision)) + } +} + +// TestLedgerConnectRejectsInvalidURL asserts that the ledger's replication +// connection reports a malformed connection string instead of panicking. +func TestLedgerConnectRejectsInvalidURL(t *testing.T) { + pgd := &pgDatastore{dburl: "://not-a-valid-url"} + _, err := pgd.connectLogicalReplication(t.Context()) + require.Error(t, err) +} diff --git a/internal/datastore/postgres/revisions_test.go b/internal/datastore/postgres/revisions_test.go index 6047488331..1ad5e795dd 100644 --- a/internal/datastore/postgres/revisions_test.go +++ b/internal/datastore/postgres/revisions_test.go @@ -1,13 +1,17 @@ package postgres import ( + "encoding/base64" "fmt" "strconv" + "strings" "testing" "time" "github.com/ccoveille/go-safecast/v2" "github.com/stretchr/testify/require" + + "github.com/authzed/spicedb/pkg/datastore" ) const ( @@ -196,6 +200,182 @@ func TestBrokenInvalidRevision(t *testing.T) { require.Error(t, err) } +// TestParseLSNRevisionString tests parsing and serialization of position-carrying +// revision tokens used by watch consumers. It checks legacy-versus-position format +// handling, validation rules (such as disallowing a zero commit LSN, which is reserved +// for legacy revisions), and verifies that the string round-trip preserves byte-sortable, +// fixed-width leading-zero formatting. +func TestParseLSNRevisionString(t *testing.T) { + // MarshalBinary encodes only the snapshot proto, never the commit LSN, so a + // legacy token is exactly the payload that follows the position prefix. + base := postgresRevision{ + snapshot: pgSnapshot{xmin: 1000, xmax: 1005, xipList: []uint64{1002, 1003}}, + optionalTxID: NewXid8(1004), + optionalInexactNanosTimestamp: 1_700_000_000_000_000_000, + } + legacyToken := base.String() + require.NotContains(t, legacyToken, string(lsnRevisionSeparator), "a legacy token must not contain the position separator") + + positionToken := func(lsn uint64) string { + rev := base + rev.optionalCommitLSN = lsn + return rev.String() + } + + t.Run("round-trips", func(t *testing.T) { + roundTrips := []struct { + name string + lsn uint64 + }{ + {"a typical commit LSN", 0x16CD3F0000028}, + {"minimal non-zero LSN keeps its leading zeros", 1}, + {"maximum LSN", 0xFFFFFFFFFFFFFFFF}, + } + for _, tc := range roundTrips { + t.Run(tc.name, func(t *testing.T) { + token := positionToken(tc.lsn) + + prefix, _, found := strings.Cut(token, string(lsnRevisionSeparator)) + require.True(t, found, "a position token must contain the separator") + require.Len(t, prefix, lsnHexLength, "the position prefix must be fixed width so tokens sort byte-wise") + + rev, ok, err := parseLSNRevisionString(token) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, tc.lsn, rev.optionalCommitLSN) + require.Equal(t, base.snapshot, rev.snapshot) + require.Equal(t, base.optionalTxID, rev.optionalTxID) + require.True(t, rev.ByteSortable()) + }) + } + }) + + t.Run("guards", func(t *testing.T) { + // A well-formed prefix for LSN 0x500, used to isolate the payload guards + // from the prefix guards. + const validPrefix = "0000000000000500" + sep := string(lsnRevisionSeparator) + + guardCases := []struct { + name string + input string + wantOK bool + errContains string // empty means no error expected + }{ + { + name: "a legacy token is not a position token", + input: legacyToken, + wantOK: false, + }, + { + name: "missing payload after the position prefix", + input: validPrefix + sep, + wantOK: true, + errContains: "missing revision payload", + }, + { + name: "a non-hex commit LSN prefix", + input: "zzzzzzzzzzzzzzzz" + sep + legacyToken, + wantOK: true, + errContains: "invalid commit LSN prefix", + }, + { + name: "a zero commit LSN is reserved for legacy revisions", + input: "0000000000000000" + sep + legacyToken, + wantOK: true, + errContains: "commit LSN must be non-zero", + }, + { + name: "a payload that is not valid base64", + input: validPrefix + sep + "!!!not base64!!!", + wantOK: true, + errContains: "invalid revision format", + }, + { + name: "a payload that is valid base64 but not a revision proto", + input: validPrefix + sep + base64.StdEncoding.EncodeToString([]byte{0xff, 0xff, 0xff, 0xff}), + wantOK: true, + errContains: "invalid revision format", + }, + } + for _, tc := range guardCases { + t.Run(tc.name, func(t *testing.T) { + rev, ok, err := parseLSNRevisionString(tc.input) + require.Equal(t, tc.wantOK, ok, "ok reports whether the input is a position token") + if tc.errContains == "" { + require.NoError(t, err) + require.Zero(t, rev.optionalCommitLSN) + } else { + require.Error(t, err) + require.ErrorContains(t, err, tc.errContains) + } + }) + } + }) + + // ParseRevisionString must route a position token through the position + // parser, accept a legacy token, and surface a malformed position token as + // an error instead of silently misreading it as a legacy revision. + t.Run("ParseRevisionString routes position tokens", func(t *testing.T) { + parsed, err := ParseRevisionString(positionToken(0x900)) + require.NoError(t, err) + rev, ok := parsed.(postgresRevision) + require.True(t, ok) + require.Equal(t, uint64(0x900), rev.optionalCommitLSN) + }) + + t.Run("ParseRevisionString accepts a legacy token", func(t *testing.T) { + parsed, err := ParseRevisionString(legacyToken) + require.NoError(t, err) + rev, ok := parsed.(postgresRevision) + require.True(t, ok) + require.False(t, rev.ByteSortable()) + }) + + // A malformed position token must surface as an error paired with NoRevision, + // rather than falling through to the legacy parsers or handing back a + // zero-valued revision that compares like a real one. + t.Run("ParseRevisionString surfaces a malformed position token", func(t *testing.T) { + sep := string(lsnRevisionSeparator) + + malformed := []struct { + name string + input string + errContains string + }{ + { + name: "zero commit LSN", + input: "0000000000000000" + sep + legacyToken, + errContains: "commit LSN must be non-zero", + }, + { + name: "non-hex commit LSN", + input: "zzzzzzzzzzzzzzzz" + sep + legacyToken, + errContains: "invalid commit LSN prefix", + }, + { + name: "empty payload", + input: "0000000000000500" + sep, + errContains: "missing revision payload", + }, + { + name: "payload that is not a revision proto", + input: "0000000000000500" + sep + "!!!not base64!!!", + errContains: "invalid revision format", + }, + } + + for _, tc := range malformed { + t.Run(tc.name, func(t *testing.T) { + parsed, err := ParseRevisionString(tc.input) + require.Error(t, err) + require.ErrorContains(t, err, tc.errContains) + require.Equal(t, datastore.NoRevision, parsed, "a failed parse must yield NoRevision") + }) + } + }) +} + func FuzzRevision(f *testing.F) { // Attempt to find a decimal revision that is a valid base64 encoded proto revision f.Add(uint64(0), -1) diff --git a/internal/datastore/postgres/schema/schema.go b/internal/datastore/postgres/schema/schema.go index 674ab37d05..5e210c57f7 100644 --- a/internal/datastore/postgres/schema/schema.go +++ b/internal/datastore/postgres/schema/schema.go @@ -12,8 +12,12 @@ const ( TableTuple = "relation_tuple" TableCaveat = "caveat" TableRelationshipCounter = "relationship_counter" + TableLedgerXidLSN = "ledger_xid_lsn" + TableLedgerState = "ledger_state" + TableLedgerGap = "ledger_gap" ColXID = "xid" + ColCommitLSN = "commit_lsn" ColTimestamp = "timestamp" ColMetadata = "metadata" ColNamespace = "namespace" @@ -36,6 +40,18 @@ const ( ColCounterFilter = "serialized_filter" ColCounterCurrentCount = "current_count" ColCounterSnapshot = "updated_revision_snapshot" + + ColLedgerSingleton = "singleton" + ColLedgerGenesisSnapshot = "genesis_snapshot" + ColLedgerGenesisLSN = "genesis_lsn" + ColLedgerBackfillOffset = "backfill_offset" + ColLedgerBackfillComplete = "backfill_complete" + ColLedgerSlotRecreations = "slot_recreations" + ColLedgerUpdatedAt = "updated_at" + + ColGapFromLSN = "from_lsn" + ColGapToLSN = "to_lsn" + ColGapDetectedAt = "detected_at" ) func Schema(colOptimizationOpt common.ColumnOptimizationOption, expirationDisabled bool) *common.SchemaInformation { diff --git a/internal/datastore/postgres/snapshot.go b/internal/datastore/postgres/snapshot.go index 46f48fbe17..8eeb65c79c 100644 --- a/internal/datastore/postgres/snapshot.go +++ b/internal/datastore/postgres/snapshot.go @@ -12,16 +12,22 @@ import ( "github.com/authzed/spicedb/pkg/spiceerrors" ) +// PostgreSQL catalog type OIDs of the types SpiceDB registers custom codecs for. +const ( + pgSnapshotTypeOID = 5038 + xid8TypeOID = 5069 +) + // RegisterTypes registers pgSnapshot and xid8 with a pgtype.ConnInfo. func RegisterTypes(m *pgtype.Map) { m.RegisterType(&pgtype.Type{ Name: "snapshot", - OID: 5038, + OID: pgSnapshotTypeOID, Codec: SnapshotCodec{}, }) m.RegisterType(&pgtype.Type{ Name: "xid", - OID: 5069, + OID: xid8TypeOID, Codec: Uint64Codec{}, }) m.RegisterDefaultPgType(pgSnapshot{}, "snapshot") diff --git a/internal/datastore/postgres/watch.go b/internal/datastore/postgres/watch.go index 807b90eed7..9ff0190a11 100644 --- a/internal/datastore/postgres/watch.go +++ b/internal/datastore/postgres/watch.go @@ -82,6 +82,10 @@ func (pgd *pgDatastore) Watch( return updates, errs } + if pgd.cursorWatchEnabled { + return pgd.cursorWatch(ctx, afterRevisionRaw, options, updates, errs) + } + if options.EmissionStrategy == datastore.EmitImmediatelyStrategy { close(updates) errs <- errors.New("emit immediately strategy is unsupported in Postgres") diff --git a/internal/datastore/postgres/watch_cursor.go b/internal/datastore/postgres/watch_cursor.go new file mode 100644 index 0000000000..b14ad89334 --- /dev/null +++ b/internal/datastore/postgres/watch_cursor.go @@ -0,0 +1,1011 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/ccoveille/go-safecast/v2" + "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5" + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/protobuf/types/known/structpb" + + "github.com/authzed/spicedb/internal/datastore/common" + "github.com/authzed/spicedb/internal/datastore/postgres/schema" + log "github.com/authzed/spicedb/internal/logging" + "github.com/authzed/spicedb/pkg/datastore" + "github.com/authzed/spicedb/pkg/spiceerrors" + "github.com/authzed/spicedb/pkg/tuple" +) + +// The cursor watch is an alternative implementation of the datastore Watch API +// for PostgreSQL that delivers transactions in true commit order, keyed by the +// commit positions the commit LSN ledger (see lsn_ledger.go) records into +// relation_tuple_transaction.commit_lsn. +// +// It is the polling watch with a different discovery query. Where the polling +// watch tracks a snapshot and orders by pg_xact_commit_timestamp, the cursor +// watch tracks a single pg_lsn cursor and asks for +// +// commit_lsn > cursor AND commit_lsn <= frontier, ORDER BY commit_lsn +// +// where the frontier is the ledger slot's confirmed_flush_lsn. The frontier is +// a completeness bound: the ledger decodes WAL in order and confirms a position +// only after everything at or below it is durably recorded, so every +// transaction with commit_lsn <= frontier is present in the table. Advancing +// the cursor to the last delivered position therefore yields every +// watch-visible transaction exactly once, in commit order, with no gap and no +// duplicate. There is no second delivery path to agree with: the ledger is the +// only WAL decoder, and every token is a pure function of the transaction it +// names. +// +// What a consumer sees: +// +// - A revision's string form is the 16-hex big-endian commit LSN, a '.', and +// the base64 revision proto. The prefix is fixed width, so byte order is +// commit order, and comparing two tokens from different Watch calls, or +// from different consumers, is meaningful. Commit-LSN order is a linear +// extension of the MVCC snapshot partial order, so token comparison never +// contradicts snapshot-based ground truth. +// - Change events arrive strictly ascending by commit LSN, with no seam. +// - A checkpoint means delivery is complete through its position. Checkpoints +// are strictly monotone and exact: resuming from one neither loses nor +// repeats events. Change-event tokens are equally exact resume points. +// - Resuming from a snapshot-only revision (a polling-watch token, or a +// HeadRevision predating the feature) is served by a one-time backfill +// phase before the cursor loop takes over; see beginCursorWatch. +// - Transactions that committed before the ledger was provisioned have no +// recoverable position and are delivered first, in commit order, as +// unpositioned revisions: the same token shape the polling watch emits. +// Those tokens must not be byte-compared with positioned ones. +// - A transaction lost to a ledger slot recreation is recorded as a gap +// (see ledger_gap); a watch positioned below a gap fails with an explicit +// error instead of stepping over the missing transactions, and its +// consumer must restart from a current revision. +// - A cursor below the retained history fails with RevisionStale instead of +// silently truncating the stream. +const ( + // ledgerFrontierProbeInterval is how often a starting Watch call re-reads its + // marker transaction's recorded commit position while waiting for the commit + // LSN ledger to reach it. + ledgerFrontierProbeInterval = 20 * time.Millisecond + + // minimumCursorWatchPollInterval floors the discovery poll interval so a + // misconfigured value cannot spin the database. + minimumCursorWatchPollInterval = 5 * time.Millisecond +) + +var ( + // errCursorWatchDisconnected indicates that the `send` callback timed out + // writing to the `updates` channel. The callback itself has already + // delivered the disconnection error on the error channel. + errCursorWatchDisconnected = errors.New("cursor watch disconnected") + + // cursorWatchRevisionsQuery is the discovery query: every transaction whose + // recorded commit position lies in (cursor, frontier], in commit order. The + // interval's lower end is exclusive and its upper end inclusive, and the + // frontier bound is what makes an empty result mean "nothing committed" + // rather than "nothing recorded yet". + // + // It drives from the position table, whose index over (commit_lsn) INCLUDE + // (xid) answers the range scan without touching that table's heap, and then + // looks each transaction up by primary key. A position whose transaction has + // been garbage collected joins away, which is correct: it sits below every + // live consumer's cursor. + cursorWatchRevisionsQuery = fmt.Sprintf(` + SELECT t.%[1]s, t.%[2]s, t.%[3]s, t.%[4]s, p.%[6]s::text + FROM %[7]s p + JOIN %[5]s t ON t.%[1]s = p.%[1]s + WHERE p.%[6]s > $1::pg_lsn AND p.%[6]s <= $2::pg_lsn + ORDER BY p.%[6]s + LIMIT $3;`, + schema.ColXID, schema.ColSnapshot, schema.ColMetadata, schema.ColTimestamp, + schema.TableTransaction, schema.ColCommitLSN, schema.TableLedgerXidLSN) + + // catchupRevisionsQuery returns the transactions that are visible in the + // snapshot given as $2 but not in the snapshot given as $1, in commit order. + // It serves the backfill phase for a caller resuming from a snapshot-only + // revision. + // + // The join is an outer one because a transaction with no recorded position + // is meaningful here: it committed before the ledger existed, so it really + // did precede everything recorded, and NULLS FIRST puts it there. Ties among + // those are broken by the transaction row's timestamp rather than + // pg_xact_commit_timestamp, so the cursor watch does not require + // track_commit_timestamp=on. checkUnrecordedCatchupRevisions decides whether + // each such transaction is genuinely pre-ledger or a gap victim. + catchupRevisionsQuery = fmt.Sprintf(` + SELECT t.%[1]s, t.%[2]s, t.%[3]s, t.%[4]s, p.%[6]s::text + FROM %[5]s t + LEFT JOIN %[7]s p ON p.%[1]s = t.%[1]s + WHERE pg_visible_in_snapshot(t.%[1]s, $2) AND ( + t.%[1]s >= pg_snapshot_xmax($1) OR ( + t.%[1]s >= pg_snapshot_xmin($1) AND NOT pg_visible_in_snapshot(t.%[1]s, $1) + ) + ) ORDER BY p.%[6]s ASC NULLS FIRST, t.%[4]s, t.%[1]s;`, + schema.ColXID, schema.ColSnapshot, schema.ColMetadata, schema.ColTimestamp, + schema.TableTransaction, schema.ColCommitLSN, schema.TableLedgerXidLSN) + + // insertWatchMarkerQuery writes the marker transaction whose recorded commit + // position proves the commit LSN ledger has caught up past everything the + // backfill phase will replay. Like the revision heartbeat, the marker is a + // bare transaction row: it carries no metadata and produces no watch-visible + // changes for any other watcher. + insertWatchMarkerQuery = fmt.Sprintf( + `INSERT INTO %[1]s (%[2]s, %[3]s) VALUES (pg_current_xact_id(), pg_current_snapshot()) RETURNING %[2]s;`, + schema.TableTransaction, schema.ColXID, schema.ColSnapshot, + ) + + // maxVisibleRecordedCommitLSNQuery reports the highest recorded commit + // position among transactions visible in the given snapshot, which is where + // the backfill phase hands off to the cursor loop: visibility in a snapshot + // partitions transactions by commit time, and commit time order is commit + // LSN order, so everything invisible in the snapshot lies strictly above it. + // + // The position table carries the xid, so the visibility test needs no join + // to the transaction row. + maxVisibleRecordedCommitLSNQuery = fmt.Sprintf(` + SELECT %[1]s::text FROM %[2]s + WHERE pg_visible_in_snapshot(%[3]s, $1) + ORDER BY %[1]s DESC LIMIT 1;`, + schema.ColCommitLSN, schema.TableLedgerXidLSN, schema.ColXID) +) + +var ( + watchFrontierLagGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "watch_frontier_lag_bytes", + Help: "WAL bytes between the ledger's frontier and the most recently observed cursor watch position.", + }) + + watchPollDurationHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "watch_poll_duration_seconds", + Help: "The latency of the cursor watch's discovery query.", + Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1}, + }) + + watchBatchTransactionsHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "watch_batch_transactions", + Help: "Transactions returned per cursor watch poll; saturation at the batch size means a backlog is draining.", + Buckets: []float64{1, 4, 16, 64, 256, 1024, 4096}, + }) + + watchStaleRevisionCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "watch_stale_revision_total", + Help: "Watches rejected because the revision they resumed from is older than the garbage collection window.", + }) + + watchGapRejectionsCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "spicedb", + Subsystem: "datastore_postgres", + Name: "watch_gap_rejections_total", + Help: "Watches rejected because a recorded ledger gap lies above their cursor.", + }) +) + +// prepareCursorWatch validates the preconditions for the cursor watch and +// provisions the commit LSN ledger it reads from. It is invoked once at +// datastore construction, only when the watch is enabled, and fails fast when +// the server cannot support logical replication (which the ledger needs). +func (pgd *pgDatastore) prepareCursorWatch(ctx context.Context) error { + var walLevel string + if err := pgd.writePool.QueryRow(ctx, "SHOW wal_level;").Scan(&walLevel); err != nil { + return fmt.Errorf("unable to determine wal_level: %w", err) + } + + if walLevel != "logical" { + return fmt.Errorf("the commit LSN ledger requires wal_level=logical, but the connected PostgreSQL server reports wal_level=%s", walLevel) + } + + // Checked here rather than left to disable the watch, because the ledger's + // slot would otherwise be provisioned and retain WAL, for a feature that + // cannot serve. Both settings need a restart, so an operator enabling this + // watch is already restarting and can set them together. + var trackCommitTimestamp string + if err := pgd.writePool.QueryRow(ctx, "SHOW track_commit_timestamp;").Scan(&trackCommitTimestamp); err != nil { + return fmt.Errorf("unable to determine track_commit_timestamp: %w", err) + } + + if trackCommitTimestamp != "on" { + return fmt.Errorf("the cursor watch requires track_commit_timestamp=on, but the connected PostgreSQL server reports track_commit_timestamp=%s; a gap the ledger could not record is replayed in commit timestamp order, which is the only commit order left once the WAL holding it is gone", trackCommitTimestamp) + } + + // Best-effort check of the REPLICATION privilege. Some managed providers + // (e.g., RDS) grant replication through role membership rather than the + // rolreplication attribute, so a negative result is only a warning; the + // ledger's replication connection will fail with a clear error if the + // privilege is truly missing. + var canReplicate bool + if err := pgd.writePool.QueryRow( + ctx, + "SELECT COALESCE((SELECT rolreplication OR rolsuper FROM pg_roles WHERE rolname = current_user), false);", + ).Scan(&canReplicate); err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("unable to verify REPLICATION privilege for the commit LSN ledger") + } else if !canReplicate { + log.Ctx(ctx).Warn().Msg("the connected PostgreSQL user does not appear to have the REPLICATION privilege; the commit LSN ledger may fail to connect") + } + + pgd.noteLegacyStreamingWatchLeftovers(ctx) + + return pgd.prepareCommitLSNLedger(ctx) +} + +// noteLegacyStreamingWatchLeftovers reports database objects an earlier, +// streaming revision of this feature created and the cursor watch no longer +// uses. Neither is reverted automatically: a replica identity or publication +// SpiceDB set is indistinguishable from one an operator set for their own +// replication consumers, and both are harmless when unused. +func (pgd *pgDatastore) noteLegacyStreamingWatchLeftovers(ctx context.Context) { + for _, table := range []string{schema.TableTuple, schema.TableNamespace, schema.TableCaveat} { + var replicaIdentity string + if err := pgd.writePool.QueryRow( + ctx, + "SELECT relreplident::text FROM pg_class WHERE oid = $1::regclass;", table, + ).Scan(&replicaIdentity); err != nil { + log.Ctx(ctx).Warn().Err(err).Str("table", table).Msg("unable to inspect replica identity") + continue + } + if replicaIdentity == "f" { + log.Ctx(ctx).Info().Str("table", table). + Msgf("table has REPLICA IDENTITY FULL, which the watch no longer needs and which logs whole old tuples on every soft delete; if it was set for SpiceDB, revert it: ALTER TABLE %s REPLICA IDENTITY DEFAULT;", table) + } + } + + // The streaming watch's default publication name; a customized name cannot + // be recovered here, so only the default is checked. + const legacyWatchPublication = "spicedb_watch" + var exists bool + if err := pgd.writePool.QueryRow( + ctx, + "SELECT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = $1);", legacyWatchPublication, + ).Scan(&exists); err == nil && exists { + log.Ctx(ctx).Info().Str("publication", legacyWatchPublication). + Msgf("publication exists but the watch no longer uses one; if nothing else consumes it, drop it: DROP PUBLICATION %s;", legacyWatchPublication) + } +} + +// cursorWatch implements the Watch contract on top of the commit LSN ledger. +// The updates and errs channels are created by the caller (Watch) and are owned +// by this function from this point on. +func (pgd *pgDatastore) cursorWatch( + ctx context.Context, + afterRevisionRaw datastore.Revision, + options datastore.WatchOptions, + updates chan datastore.RevisionChanges, + errs chan error, +) (<-chan datastore.RevisionChanges, <-chan error) { + if options.EmissionStrategy == datastore.EmitImmediatelyStrategy && + (options.Content&datastore.WatchCheckpoints != datastore.WatchCheckpoints) { + close(updates) + errs <- errors.New("EmitImmediatelyStrategy requires WatchCheckpoints to be set") + return updates, errs + } + + afterRevision, ok := afterRevisionRaw.(postgresRevision) + if !ok { + close(updates) + errs <- datastore.NewInvalidRevisionErr(afterRevisionRaw, datastore.CouldNotDetermineRevision) + return updates, errs + } + + watchBufferWriteTimeout := options.WatchBufferWriteTimeout + if watchBufferWriteTimeout <= 0 { + watchBufferWriteTimeout = pgd.watchBufferWriteTimeout + } + + sendChange := func(change datastore.RevisionChanges) bool { + select { + case updates <- change: + return true + + default: + // If we cannot immediately write, set up the timer and try again. + } + + timer := time.NewTimer(watchBufferWriteTimeout) + defer timer.Stop() + + select { + case updates <- change: + return true + + case <-timer.C: + errs <- datastore.NewWatchDisconnectedErr() + return false + } + } + + go func() { + defer close(updates) + defer close(errs) + + err := pgd.runCursorWatch(ctx, afterRevision, options, sendChange) + if err == nil { + return + } + + if errors.Is(err, errCursorWatchDisconnected) { + // The disconnection error was already delivered by sendChange. + return + } + + switch { + case errors.Is(ctx.Err(), context.Canceled): + errs <- datastore.NewWatchCanceledErr() + case common.IsCancellationError(err): + errs <- datastore.NewWatchCanceledErr() + case common.IsResettableError(err): + errs <- datastore.NewWatchTemporaryErr(err) + default: + errs <- err + } + }() + + return updates, errs +} + +// runCursorWatch resolves the caller's starting position, then polls the +// frontier-bounded discovery query, emitting each batch and its checkpoint, +// until the context ends or a guard fails. +func (pgd *pgDatastore) runCursorWatch( + ctx context.Context, + afterRevision postgresRevision, + options datastore.WatchOptions, + sendChange func(datastore.RevisionChanges) bool, +) error { + batchSize := max(pgd.watchBatchSize, 1) + pollInterval := max(pgd.watchPollInterval, minimumCursorWatchPollInterval) + checkpointInterval := max(options.CheckpointInterval, minimumWatchSleep) + requestedCheckpoints := options.Content&datastore.WatchCheckpoints == datastore.WatchCheckpoints + livenessTimeout := max(pgd.logicalWatchLedgerWaitTimeout, ledgerFrontierProbeInterval) + + // Before anything is delivered: a revision outside the garbage collection + // window names transactions that may already be gone. + if err := pgd.checkWatchRevisionRetained(ctx, afterRevision); err != nil { + return err + } + + start, err := pgd.beginCursorWatch(ctx, afterRevision, options, sendChange) + if err != nil { + return err + } + + cursor := start.cursor + checkpointSnapshot := start.snapshot + checkpointNanos := afterRevision.optionalInexactNanosTimestamp + lastCheckpointLSN := cursor + lastCheckpointAt := time.Now() + + // inactiveSince tracks how long the ledger slot has been without a writer. + // Brief detachment is routine (the writer died and another instance is + // taking over); only a sustained one fails the watch. + var inactiveSince time.Time + + for { + if ctx.Err() != nil { + return datastore.NewWatchCanceledErr() + } + + state, err := pgd.readLedgerSlotState(ctx) + if err != nil { + return err + } + + // Liveness guard: with no frontier there is no basis for delivery, and + // with no writer the frontier will not move. Failing is deliberate; the + // alternative is a silent stall indistinguishable from "no writes". + if !state.exists { + return datastore.NewWatchTemporaryErr(fmt.Errorf( + "the commit LSN ledger's replication slot %q does not exist, so the watch has no delivery frontier", + pgd.ledgerSlotName)) + } + if state.active { + inactiveSince = time.Time{} + } else { + if inactiveSince.IsZero() { + inactiveSince = time.Now() + } + if time.Since(inactiveSince) > livenessTimeout { + return datastore.NewWatchTemporaryErr(fmt.Errorf( + "the commit LSN ledger has had no writer for %s: slot %q is confirmed through %s, attached=%t, wal_status=%q", + livenessTimeout, pgd.ledgerSlotName, state.confirmed, state.active, state.walStatus)) + } + } + + frontier := state.confirmed + if frontier > cursor { + watchFrontierLagGauge.Set(float64(frontier - cursor)) + } else { + watchFrontierLagGauge.Set(0) + } + + if frontier > cursor { + // Gap guard: a recorded gap above the cursor means transactions this + // watch is responsible for were never recorded, so what the frontier + // covers is not what was delivered. It is checked here, before every + // delivery and before every idle checkpoint, because those are the + // only two ways the omission could be passed off as completeness: a + // slot recreation moves the frontier past the gap in one step, and an + // unguarded poll would step over it in silence. + if err := pgd.checkLedgerGapAbove(ctx, cursor); err != nil { + return err + } + + startedAt := time.Now() + revisions, err := pgd.getRecordedRevisions(ctx, cursor, frontier, batchSize) + if err != nil { + return err + } + watchPollDurationHistogram.Observe(time.Since(startedAt).Seconds()) + watchBatchTransactionsHistogram.Observe(float64(len(revisions))) + + if len(revisions) > 0 { + checkpoint, err := pgd.emitRevisionBatch(ctx, revisions, options, sendChange) + if err != nil { + return err + } + + cursor = pglogrepl.LSN(checkpoint.optionalCommitLSN) + checkpointSnapshot = checkpoint.snapshot + checkpointNanos = checkpoint.optionalInexactNanosTimestamp + lastCheckpointLSN = cursor + lastCheckpointAt = time.Now() + + if len(revisions) == batchSize { + // A full batch means a backlog: drain it at query speed + // rather than one batch per poll interval. + continue + } + } else if requestedCheckpoints && frontier > lastCheckpointLSN && time.Since(lastCheckpointAt) >= checkpointInterval { + // Nothing new committed, but delivery is provably complete + // through the frontier, so consumers watching a subset of + // content still observe progress. The checkpoint carries the + // last delivered transaction's snapshot: pg_current_snapshot() + // would cover transactions in (frontier, now] that have NOT + // been delivered, and feeding such a token to a + // snapshot-filtered path (e.g. the polling watch) would lose + // them silently. + if !sendChange(datastore.RevisionChanges{ + Revision: postgresRevision{ + snapshot: checkpointSnapshot, + optionalInexactNanosTimestamp: checkpointNanos, + optionalCommitLSN: uint64(frontier), + }, + IsCheckpoint: true, + }) { + return errCursorWatchDisconnected + } + lastCheckpointLSN = frontier + lastCheckpointAt = time.Now() + } + } + + select { + case <-time.After(pollInterval): + case <-ctx.Done(): + return datastore.NewWatchCanceledErr() + } + } +} + +// cursorWatchStart is the resolved starting state of a cursor watch. +type cursorWatchStart struct { + // cursor is the position delivery resumes strictly above. + cursor pglogrepl.LSN + + // snapshot seeds the idle checkpoints' snapshot component until the first + // delivery replaces it. + snapshot pgSnapshot +} + +// beginCursorWatch resolves the caller's afterRevision into a cursor. +// +// A position-carrying revision is the normal case: the cursor is its commit +// LSN, complete on its own, because every stream delivers in commit order and +// the consumer received everything at or below it before the token was minted. +// The snapshot the token carries is not consulted for delivery at all. +// +// A snapshot-only revision (a polling-watch token, or a HeadRevision taken +// before the feature was enabled) requires a one-time backfill phase: +// +// 1. Take a snapshot S. Write a marker transaction and wait until the ledger +// records its commit position. The marker commits after S, so the probe +// proves every transaction visible in S has its position recorded — except +// those predating the ledger, which the genesis snapshot classifies. +// 2. Deliver the transactions visible in S but not in the caller's snapshot, +// ordered by recorded position with the pre-ledger prefix first, in +// batches, each with its checkpoint. +// 3. Hand off to the cursor loop at the highest recorded position visible in +// S. Visibility in S partitions transactions by commit time and commit +// time order is commit LSN order, so everything not visible in S — the +// loop's responsibility — lies strictly above the handoff cursor: no gap +// and no duplicate at the boundary. +// +// The probe in step 1 is what closes the accounting. Without it, a transaction +// visible in S but not yet recorded at query time would be indistinguishable +// from a slot-recreation gap (a spurious hard error), and, worse, the handoff +// cursor could be read below such a transaction's eventual position, letting +// the loop deliver it again after the backfill phase's snapshot filter had +// already skipped it as seen. +// +// The handoff position in step 3 is the highest recorded position *visible in +// S*, and deliberately not the ledger's confirmed frontier. The frontier can +// have advanced past S while the backfill ran, and a transaction that committed +// in between is neither visible in S (so the backfill does not deliver it) nor +// above the frontier (so a loop starting there would skip it): handing off at +// the frontier loses exactly those transactions. +func (pgd *pgDatastore) beginCursorWatch( + ctx context.Context, + afterRevision postgresRevision, + options datastore.WatchOptions, + sendChange func(datastore.RevisionChanges) bool, +) (cursorWatchStart, error) { + if afterRevision.ByteSortable() { + return cursorWatchStart{ + cursor: pglogrepl.LSN(afterRevision.optionalCommitLSN), + snapshot: afterRevision.snapshot, + }, nil + } + + var backfillSnapshot pgSnapshot + if err := pgd.readPool.QueryRow(ctx, "SELECT pg_current_snapshot();").Scan(&backfillSnapshot); err != nil { + return cursorWatchStart{}, fmt.Errorf("unable to load the backfill snapshot: %w", err) + } + + markerXid, err := pgd.writeWatchMarker(ctx) + if err != nil { + return cursorWatchStart{}, datastore.NewWatchTemporaryErr(err) + } + + if err := pgd.awaitLedgerFrontier(ctx, markerXid); err != nil { + return cursorWatchStart{}, err + } + + // The ledger recorded the marker, so the slot existed a moment ago; confirm + // it still does before delivering anything, so that a watch whose frontier + // has just disappeared fails here rather than mid-backfill. + state, err := pgd.readLedgerSlotState(ctx) + if err != nil { + return cursorWatchStart{}, err + } + if !state.exists { + return cursorWatchStart{}, datastore.NewWatchTemporaryErr(fmt.Errorf( + "the commit LSN ledger's replication slot %q does not exist, so the watch has no delivery frontier", + pgd.ledgerSlotName)) + } + + revisions, err := pgd.getLegacyCatchupRevisions(ctx, afterRevision, backfillSnapshot) + if err != nil { + return cursorWatchStart{}, err + } + + start := cursorWatchStart{snapshot: afterRevision.snapshot} + + batchSize := max(pgd.watchBatchSize, 1) + for offset := 0; offset < len(revisions); offset += batchSize { + chunk := revisions[offset:min(offset+batchSize, len(revisions))] + checkpoint, err := pgd.emitRevisionBatch(ctx, chunk, options, sendChange) + if err != nil { + return cursorWatchStart{}, err + } + start.snapshot = checkpoint.snapshot + } + + var handoffText *string + if err := pgd.readPool.QueryRow(ctx, maxVisibleRecordedCommitLSNQuery, backfillSnapshot).Scan(&handoffText); err != nil { + if !errors.Is(err, pgx.ErrNoRows) { + return cursorWatchStart{}, fmt.Errorf("unable to determine the backfill handoff position: %w", err) + } + } + if handoffText != nil { + handoff, err := pglogrepl.ParseLSN(*handoffText) + if err != nil { + return cursorWatchStart{}, fmt.Errorf("unable to parse the backfill handoff position: %w", err) + } + start.cursor = handoff + } + + return start, nil +} + +// writeWatchMarker commits the marker transaction and returns its transaction ID. +func (pgd *pgDatastore) writeWatchMarker(ctx context.Context) (xid8, error) { + var markerXid xid8 + if err := pgd.writePool.QueryRow(ctx, insertWatchMarkerQuery).Scan(&markerXid); err != nil { + return markerXid, fmt.Errorf("unable to write the watch marker transaction: %w", err) + } + return markerXid, nil +} + +// awaitLedgerFrontier waits until the commit LSN ledger has recorded the +// marker's commit position. The ledger records positions in commit order, so +// once the marker has one, so does every transaction that committed before it. +// +// This is the only point at which a starting Watch call blocks on the ledger. +// Failing here is deliberate: without it, the backfill could neither classify +// unrecorded transactions nor read a handoff position it can trust. +func (pgd *pgDatastore) awaitLedgerFrontier(ctx context.Context, markerXid xid8) error { + waitTimeout := max(pgd.logicalWatchLedgerWaitTimeout, ledgerFrontierProbeInterval) + startedAt := time.Now() + + deadlineCtx, cancelWait := context.WithTimeout(ctx, waitTimeout) + defer cancelWait() + + for { + // The marker having no row yet is the ordinary case being waited on; any + // other failure is real. + var commitLSNText string + err := pgd.readPool.QueryRow(deadlineCtx, commitLSNForXidQuery, markerXid).Scan(&commitLSNText) + switch { + case err == nil: + watchLedgerWaitHistogram.Observe(time.Since(startedAt).Seconds()) + return nil + + case errors.Is(err, pgx.ErrNoRows): + // Not recorded yet; fall through to wait. + + default: + if ctx.Err() == nil && deadlineCtx.Err() != nil { + return pgd.ledgerFrontierTimeoutError(ctx, markerXid, waitTimeout) + } + return fmt.Errorf("unable to read the watch marker's recorded commit position: %w", err) + } + + select { + case <-deadlineCtx.Done(): + if ctx.Err() != nil { + return datastore.NewWatchCanceledErr() + } + return pgd.ledgerFrontierTimeoutError(ctx, markerXid, waitTimeout) + case <-time.After(ledgerFrontierProbeInterval): + } + } +} + +// ledgerFrontierTimeoutError reports a stalled ledger with the state an operator +// needs to act on: which slot is behind, how far it has confirmed, and whether +// any instance is attached to it at all. +func (pgd *pgDatastore) ledgerFrontierTimeoutError(ctx context.Context, markerXid xid8, waited time.Duration) error { + state, err := pgd.readLedgerSlotState(ctx) + if err != nil { + return fmt.Errorf( + "the commit LSN ledger did not record the watch marker transaction %d within %s, and its slot %q could not be inspected: %w", + markerXid.Uint64, waited, pgd.ledgerSlotName, err) + } + + if !state.exists { + return fmt.Errorf( + "the commit LSN ledger did not record the watch marker transaction %d within %s: its replication slot %q does not exist", + markerXid.Uint64, waited, pgd.ledgerSlotName) + } + + return fmt.Errorf( + "the commit LSN ledger did not record the watch marker transaction %d within %s: slot %q is confirmed through %s, attached=%t, wal_status=%q", + markerXid.Uint64, waited, pgd.ledgerSlotName, state.confirmed, state.active, state.walStatus) +} + +// checkLedgerGapAbove fails the watch when a recorded ledger gap lies above the +// given position: the gap's transactions were never recorded, so delivery from +// this position cannot be complete. The failure is permanent for this cursor — +// the consumer must restart from a current revision — so it is not wrapped as +// temporary. +func (pgd *pgDatastore) checkLedgerGapAbove(ctx context.Context, position pglogrepl.LSN) error { + gap, found, err := pgd.firstLedgerGapAbove(ctx, position) + if err != nil { + return err + } + if !found { + return nil + } + + watchGapRejectionsCounter.Inc() + return fmt.Errorf( + "the commit LSN ledger's replication slot was recreated at %s and transactions that committed in (%s, %s] have no recorded commit position; a watch positioned at %s cannot be resumed across the gap and must restart from a current revision", + gap.detectedAt, gap.from, gap.to, position) +} + +// checkWatchRevisionRetained fails a starting watch whose revision names a +// transaction older than the garbage collection window. Everything at that +// position is collectable, so the transactions the consumer has not seen may +// already be gone, and no stream can deliver them. Reporting that is an +// improvement on both existing watches, which serve such a consumer a silently +// truncated stream. +// +// The test is on the revision's own recorded timestamp against the database's +// clock, which is the same basis garbage collection deletes on. Two other +// formulations were tried and are unsound: +// +// - CheckRevision's snapshot comparison against the oldest in-window +// transaction calls a revision stale whenever the oldest surviving +// transaction is newer than it, which on any database without history older +// than the window (a fresh one, for instance) is every revision. +// - "the cursor is below the oldest recorded commit position" cannot tell +// transactions collected from above the cursor apart from there having been +// none, so it fails a consumer that is perfectly caught up when collection +// prunes below it. +// +// A revision with no timestamp (a HeadRevision, or an idle checkpoint minted +// before any delivery) is exempt: there is nothing to compare, and no evidence +// of loss. +func (pgd *pgDatastore) checkWatchRevisionRetained(ctx context.Context, afterRevision postgresRevision) error { + revisionNanos, ok := afterRevision.OptionalNanosTimestamp() + if !ok { + return nil + } + + var now time.Time + if err := pgd.readPool.QueryRow(ctx, "SELECT NOW() AT TIME ZONE 'utc';").Scan(&now); err != nil { + return fmt.Errorf("unable to determine the database time: %w", err) + } + + nanos, err := safecast.Convert[uint64](now.Add(-pgd.gcWindow).UnixNano()) + if err != nil { + return fmt.Errorf("unable to determine the retained transaction window: %w", err) + } + + if revisionNanos < nanos { + watchStaleRevisionCounter.Inc() + return datastore.NewInvalidRevisionErr(afterRevision, datastore.RevisionStale) + } + + return nil +} + +// getRecordedRevisions returns the transactions whose recorded commit positions +// lie in (after, upTo], in commit order, at most batchSize of them. +func (pgd *pgDatastore) getRecordedRevisions(ctx context.Context, after, upTo pglogrepl.LSN, batchSize int) ([]postgresRevision, error) { + rows, err := pgd.readPool.Query(ctx, cursorWatchRevisionsQuery, after.String(), upTo.String(), batchSize) + if err != nil { + return nil, fmt.Errorf("unable to load recorded revisions: %w", err) + } + defer rows.Close() + + var revisions []postgresRevision + for rows.Next() { + revision, commitLSNText, err := scanWatchRevisionRow(rows) + if err != nil { + return nil, err + } + if commitLSNText == nil { + return nil, spiceerrors.MustBugf("the cursor discovery query returned a transaction with no recorded position") + } + + commitLSN, err := pglogrepl.ParseLSN(*commitLSNText) + if err != nil { + return nil, fmt.Errorf("unable to decode the recorded commit position of transaction %d: %w", revision.optionalTxID.Uint64, err) + } + revision.optionalCommitLSN = uint64(commitLSN) + + revisions = append(revisions, revision) + } + if rows.Err() != nil { + return nil, fmt.Errorf("unable to load recorded revisions: %w", rows.Err()) + } + + return revisions, nil +} + +// getLegacyCatchupRevisions returns the transactions visible in `upTo` but not +// in afterRevision, in commit order, each carrying the commit position the +// ledger recorded for it, with the pre-ledger prefix unpositioned. +func (pgd *pgDatastore) getLegacyCatchupRevisions(ctx context.Context, afterRevision postgresRevision, upTo pgSnapshot) ([]postgresRevision, error) { + var revisions []postgresRevision + unrecorded := false + if err := pgx.BeginTxFunc(ctx, pgd.readPool, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}, func(tx pgx.Tx) error { + rows, err := tx.Query(ctx, catchupRevisionsQuery, afterRevision.snapshot, upTo) + if err != nil { + return fmt.Errorf("unable to load catch-up revisions: %w", err) + } + defer rows.Close() + + for rows.Next() { + revision, commitLSNText, err := scanWatchRevisionRow(rows) + if err != nil { + return err + } + + if commitLSNText == nil { + unrecorded = true + } else { + commitLSN, err := pglogrepl.ParseLSN(*commitLSNText) + if err != nil { + return fmt.Errorf("unable to decode the recorded commit position of transaction %d: %w", revision.optionalTxID.Uint64, err) + } + revision.optionalCommitLSN = uint64(commitLSN) + } + + revisions = append(revisions, revision) + } + if rows.Err() != nil { + return fmt.Errorf("unable to load catch-up revisions: %w", rows.Err()) + } + return nil + }); err != nil { + return nil, fmt.Errorf("transaction error: %w", err) + } + + if unrecorded { + if err := pgd.checkUnrecordedCatchupRevisions(ctx, revisions, afterRevision); err != nil { + return nil, err + } + } + + return revisions, nil +} + +// scanWatchRevisionRow decodes one transaction-table row of the shape shared by +// the discovery and backfill queries into a revision, returning the recorded +// commit position's text form (or nil) for the caller to interpret. +func scanWatchRevisionRow(rows pgx.Rows) (postgresRevision, *string, error) { + var nextXID xid8 + var nextSnapshot pgSnapshot + var metadata map[string]any + var timestamp time.Time + var commitLSNText *string + if err := rows.Scan(&nextXID, &nextSnapshot, &metadata, ×tamp, &commitLSNText); err != nil { + return postgresRevision{}, nil, fmt.Errorf("unable to decode watch revision: %w", err) + } + + nanosTimestamp, err := safecast.Convert[uint64](timestamp.UnixNano()) + if err != nil { + return postgresRevision{}, nil, fmt.Errorf("could not cast timestamp to uint64: %w", err) + } + + return postgresRevision{ + snapshot: nextSnapshot.markComplete(nextXID.Uint64), + optionalTxID: nextXID, + optionalInexactNanosTimestamp: nanosTimestamp, + optionalMetadata: metadata, + }, commitLSNText, nil +} + +// checkUnrecordedCatchupRevisions decides what to do about replayable +// transactions that have no recorded commit position. There are exactly two +// reasons for that, and the ledger's genesis snapshot tells them apart: +// +// - The transaction committed before the ledger existed, so it is visible in +// the genesis snapshot. It really did commit before everything the ledger +// recorded, and the query sorts it accordingly, so it is emitted as an +// unpositioned revision: the same token the polling watch would emit. +// +// - The transaction committed while the ledger's slot was invalid, so it is not +// visible in the genesis snapshot. Its commit position is unrecoverable, so +// the watch refuses rather than guessing one. +// +// A caller resuming from a position-carrying revision cannot be served an +// unpositioned one either way, because it would not be comparable with the token +// the caller already holds. +func (pgd *pgDatastore) checkUnrecordedCatchupRevisions(ctx context.Context, revisions []postgresRevision, afterRevision postgresRevision) error { + genesis, err := pgd.ledgerGenesisSnapshot(ctx) + if err != nil { + return err + } + + for _, revision := range revisions { + if revision.ByteSortable() { + continue + } + + xid := revision.optionalTxID.Uint64 + if !genesis.txVisible(xid) { + return fmt.Errorf( + "transaction %d has no recorded commit LSN and postdates the commit LSN ledger's genesis, which means the ledger's replication slot %q was recreated and this transaction's commit position is unrecoverable; watches cannot be ordered across the gap and must restart from a current revision", + xid, pgd.ledgerSlotName) + } + + if afterRevision.ByteSortable() { + return fmt.Errorf( + "transaction %d predates the commit LSN ledger and so has no position to emit, but this watch resumed from a position-carrying revision; restart the watch from a current revision", + xid) + } + } + + return nil +} + +// emitRevisionBatch loads and sends the changes for one batch of transactions, +// already in commit order, followed by the batch's checkpoint. The returned +// checkpoint revision sits at the last transaction, with every transaction of +// the batch marked complete in its snapshot, mirroring the polling watch's +// per-batch checkpoints. +func (pgd *pgDatastore) emitRevisionBatch( + ctx context.Context, + revisions []postgresRevision, + options datastore.WatchOptions, + sendChange func(datastore.RevisionChanges) bool, +) (postgresRevision, error) { + changesToWrite, err := pgd.loadChanges(ctx, revisions, options) + if err != nil { + return postgresRevision{}, err + } + + for change, err := range changesToWrite { + if err != nil { + return postgresRevision{}, err + } + + if options.EmissionStrategy == datastore.EmitImmediatelyStrategy { + for _, atom := range decomposeRevisionChanges(change) { + if !sendChange(atom) { + return postgresRevision{}, errCursorWatchDisconnected + } + } + continue + } + + if !sendChange(change) { + return postgresRevision{}, errCursorWatchDisconnected + } + } + + lastRevision := revisions[len(revisions)-1] + checkpoint := postgresRevision{ + snapshot: lastRevision.snapshot, + optionalTxID: lastRevision.optionalTxID, + optionalInexactNanosTimestamp: lastRevision.optionalInexactNanosTimestamp, + optionalCommitLSN: lastRevision.optionalCommitLSN, + } + for _, revision := range revisions { + checkpoint.snapshot = checkpoint.snapshot.markComplete(revision.optionalTxID.Uint64) + } + + if options.Content&datastore.WatchCheckpoints == datastore.WatchCheckpoints { + if !sendChange(datastore.RevisionChanges{ + Revision: checkpoint, + IsCheckpoint: true, + }) { + return postgresRevision{}, errCursorWatchDisconnected + } + } + + return checkpoint, nil +} + +// decomposeRevisionChanges splits an assembled RevisionChanges into independent +// single-item events for the EmitImmediatelyStrategy. +func decomposeRevisionChanges(change datastore.RevisionChanges) []datastore.RevisionChanges { + atoms := make([]datastore.RevisionChanges, 0, + len(change.RelationshipChanges)+len(change.ChangedDefinitions)+len(change.DeletedNamespaces)+len(change.DeletedCaveats)+len(change.Metadatas)) + + for _, relChange := range change.RelationshipChanges { + atoms = append(atoms, datastore.RevisionChanges{ + Revision: change.Revision, + RelationshipChanges: []tuple.RelationshipUpdate{relChange}, + }) + } + for _, def := range change.ChangedDefinitions { + atoms = append(atoms, datastore.RevisionChanges{ + Revision: change.Revision, + ChangedDefinitions: []datastore.SchemaDefinition{def}, + }) + } + for _, namespaceName := range change.DeletedNamespaces { + atoms = append(atoms, datastore.RevisionChanges{ + Revision: change.Revision, + DeletedNamespaces: []string{namespaceName}, + }) + } + for _, caveatName := range change.DeletedCaveats { + atoms = append(atoms, datastore.RevisionChanges{ + Revision: change.Revision, + DeletedCaveats: []string{caveatName}, + }) + } + for _, metadata := range change.Metadatas { + atoms = append(atoms, datastore.RevisionChanges{ + Revision: change.Revision, + Metadatas: []*structpb.Struct{metadata}, + }) + } + + return atoms +} diff --git a/internal/datastore/postgres/watch_cursor_test.go b/internal/datastore/postgres/watch_cursor_test.go new file mode 100644 index 0000000000..0984aec53b --- /dev/null +++ b/internal/datastore/postgres/watch_cursor_test.go @@ -0,0 +1,3606 @@ +//go:build datastore && postgres + +package postgres + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/types/known/structpb" + + testdatastore "github.com/authzed/spicedb/internal/testserver/datastore" + "github.com/authzed/spicedb/pkg/datastore" + "github.com/authzed/spicedb/pkg/datastore/options" + "github.com/authzed/spicedb/pkg/datastore/test" + core "github.com/authzed/spicedb/pkg/proto/core/v1" + "github.com/authzed/spicedb/pkg/tuple" +) + +const cursorWatchTestTimeout = 30 * time.Second + +// newDatastoreFunc constructs a datastore against the shared test container, +// with the cursor watch enabled or disabled. +type newDatastoreFunc func(t testing.TB, revisionQuantization, gcWindow time.Duration, watchBufferLength uint16, cursorWatch bool) datastore.Datastore + +// TestPostgresCursorWatch runs the datastore Watch conformance suite plus +// targeted cursor-watch tests against a Postgres with wal_level=logical, with +// the cursor watch (and therefore the commit LSN ledger) enabled. +func TestPostgresCursorWatch(t *testing.T) { + b := testdatastore.RunPostgresForTestingWithLogicalReplication(t, postgresTestVersion()) + + newDatastore := newDatastoreFunc(func(t testing.TB, revisionQuantization, gcWindow time.Duration, watchBufferLength uint16, cursorWatch bool) datastore.Datastore { + ctx := t.Context() + return b.NewDatastore(t, func(engine, uri string) datastore.Datastore { + ds, err := newPostgresDatastore( + ctx, uri, primaryInstanceID, + RevisionQuantization(revisionQuantization), + GCWindow(gcWindow), + GCInterval(veryLargeGCInterval), + WatchBufferLength(watchBufferLength), + DebugAnalyzeBeforeStatistics(), + WithRevisionHeartbeat(false), + WithLogicalWatch(cursorWatch), + ) + require.NoError(t, err) + return ds + }) + }) + + cursorTester := test.DatastoreTesterFunc(func(t testing.TB, revisionQuantization, _, gcWindow time.Duration, watchBufferLength uint16) (datastore.Datastore, error) { + return newDatastore(t, revisionQuantization, gcWindow, watchBufferLength, true), nil + }) + + conformanceTests := map[string]func(*testing.T, test.DatastoreTester){ + "TestWatchBasic": test.WatchTest, + "TestWatchCancel": test.WatchCancelTest, + "TestCaveatedRelationshipWatch": test.CaveatedRelationshipWatchTest, + "TestWatchWithTouch": test.WatchWithTouchTest, + "TestWatchWithDelete": test.WatchWithDeleteTest, + "TestWatchWithMetadata": test.WatchWithMetadataTest, + "TestWatchWithExpiration": test.WatchWithExpirationTest, + "TestWatchEmissionStrategy": test.WatchEmissionStrategyTest, + "TestWatchSchema": test.WatchSchemaTest, + "TestWatchRelationshipsAndSchemaChanges": test.WatchRelationshipsAndSchemaChangesTest, + "TestWatchObservesEveryReturnedRevision": test.WatchObservesEveryReturnedRevisionTest, + "TestWatchEmitsCheckpointAfterWriteWithChanges": test.WatchEmitsCheckpointAfterWriteWithChangesTest, + "TestWatchRelationshipsAndSchemaAndCheckpoints": test.WatchRelationshipsAndSchemaAndCheckpointsTest, + } + + for name, testFunc := range conformanceTests { + t.Run(name, func(t *testing.T) { + testFunc(t, cursorTester) + }) + } + + // The invariant the whole design rests on. + t.Run("FrontierBoundedDeliveryIsExactlyOnce", func(t *testing.T) { + testFrontierBoundedDeliveryIsExactlyOnce(t, b) + }) + + t.Run("GCDoesNotEmitChanges", func(t *testing.T) { + testCursorWatchIgnoresGC(t, newDatastore) + }) + + t.Run("ParityWithPollingWatch", func(t *testing.T) { + testCursorWatchParity(t, b) + }) + + t.Run("OverlappingTransactions", func(t *testing.T) { + testCursorWatchOverlappingTransactions(t, newDatastore) + }) + + t.Run("TokenReadRoundTrip", func(t *testing.T) { + testCursorWatchTokenReadRoundTrip(t, newDatastore) + }) + + t.Run("CommitOrderLinearExtension", func(t *testing.T) { + testCursorWatchCommitOrderLinearExtension(t, newDatastore) + }) + + t.Run("WatchArgumentValidation", func(t *testing.T) { + testCursorWatchArgumentValidation(t, newDatastore) + }) + + t.Run("CrossEpochTokenStability", func(t *testing.T) { + testCursorWatchCrossEpochTokenStability(t, newDatastore) + }) + + t.Run("SameTokenFromAnyCursor", func(t *testing.T) { + testCursorWatchSameTokenFromAnyCursor(t, newDatastore) + }) + + t.Run("ReconnectWithPositionedToken", func(t *testing.T) { + testCursorWatchReconnectWithPositionedToken(t, newDatastore) + }) + + t.Run("ConcurrentWatchers", func(t *testing.T) { + testCursorWatchConcurrentWatchers(t, newDatastore) + }) + + t.Run("DisconnectDuringBackfillLosesNothing", func(t *testing.T) { + testCursorWatchDisconnectDuringBackfillLosesNothing(t, newDatastore) + }) + + t.Run("LegacyTokenHandoff", func(t *testing.T) { + testCursorWatchLegacyTokenHandoff(t, b) + }) + + t.Run("CheckpointsAreExactOnResume", func(t *testing.T) { + testCursorWatchCheckpointsAreExactOnResume(t, b) + }) + + t.Run("BacklogDrainsWithoutSleeping", func(t *testing.T) { + testCursorWatchBacklogDrainsWithoutSleeping(t, b) + }) + + t.Run("IdleCheckpointsFollowTheFrontier", func(t *testing.T) { + testCursorWatchIdleCheckpointsFollowTheFrontier(t, b) + }) + + t.Run("StaleRevisionRejected", func(t *testing.T) { + testCursorWatchStaleRevisionRejected(t, newDatastore) + }) + + t.Run("GapBelowCursorFailsLoudly", func(t *testing.T) { + testCursorWatchGapBelowCursorFailsLoudly(t, b) + }) + + t.Run("GapRecordedOnOperatorDrop", func(t *testing.T) { + testLedgerGapRecordedOnOperatorDrop(t, b) + }) + + t.Run("GapIsReplayedFromTheTables", func(t *testing.T) { + testLedgerGapIsReplayedFromTheTables(t, b) + }) + + t.Run("PreLedgerPositionsAreBackfilled", func(t *testing.T) { + testPreLedgerPositionsAreBackfilled(t, b) + }) + + t.Run("PreLedgerBackfillStopsAtGCWindow", func(t *testing.T) { + testPreLedgerBackfillStopsAtGCWindow(t, b) + }) + + t.Run("LedgerWithoutWriterFailsWatch", func(t *testing.T) { + testCursorWatchLedgerWithoutWriterFails(t, b) + }) + + t.Run("LedgerFrontierWaitTimeout", func(t *testing.T) { + testLedgerFrontierWaitTimeout(t, b) + }) + + t.Run("LedgerRecordsWhatTheStreamReports", func(t *testing.T) { + testLedgerRecordsWhatTheStreamReports(t, b) + }) + + t.Run("LedgerTakeover", func(t *testing.T) { + testLedgerTakeover(t, b) + }) + + t.Run("PreLedgerHistoryIsDeliveredPositioned", func(t *testing.T) { + testLedgerPreLedgerHistoryIsDeliveredPositioned(t, b) + }) + + t.Run("UnrecordedTransactionFailsLoudly", func(t *testing.T) { + testLedgerUnrecordedTransactionFailsLoudly(t, b) + }) + + t.Run("LedgerWritesAreInvisibleToWatchers", func(t *testing.T) { + testLedgerWritesAreInvisibleToWatchers(t, newDatastore) + }) + + t.Run("DisabledFeatureDetectsAbandonedSlot", func(t *testing.T) { + testLedgerDisabledFeatureDetectsAbandonedSlot(t, b) + }) + + t.Run("LedgerStorageShape", func(t *testing.T) { + testLedgerStorageShape(t, b) + }) + + t.Run("LedgerPositionsAreGarbageCollected", func(t *testing.T) { + testLedgerPositionsAreGarbageCollected(t, b) + }) + + t.Run("LedgerOrphanPositionsAreIgnored", func(t *testing.T) { + testLedgerOrphanPositionsAreIgnored(t, b) + }) +} + +// newCursorWatchTestDatastore builds a datastore with the cursor watch enabled +// against a fresh database, returning it alongside that database's URI so a test +// can inspect or damage the ledger's state directly. +func newCursorWatchTestDatastore(t *testing.T, b testdatastore.RunningEngineForTest, options ...Option) (datastore.Datastore, string) { + t.Helper() + + var dbURI string + ds := b.NewDatastore(t, func(engine, uri string) datastore.Datastore { + dbURI = uri + allOptions := append([]Option{ + RevisionQuantization(0), + GCWindow(1000 * time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(512), + WithRevisionHeartbeat(false), + WithLogicalWatch(true), + }, options...) + + ds, err := newPostgresDatastore(t.Context(), uri, primaryInstanceID, allOptions...) + require.NoError(t, err) + return ds + }) + + return ds, dbURI +} + +// testFrontierBoundedDeliveryIsExactlyOnce is the test the design rests on. +// +// Concurrent writers commit a known set of transactions while a cursor loop +// reads (cursor, frontier] in commit-position order, exactly as the watch does, +// using the production queries against a raw connection. Every committed +// transaction must be delivered exactly once, in strictly ascending position +// order, with no position ever above the frontier that bounded its own batch. +func testFrontierBoundedDeliveryIsExactlyOnce(t *testing.T, b testdatastore.RunningEngineForTest) { + testCases := []struct { + name string + writers int + perWriter int + batchSize int + }{ + {name: "a single writer, one transaction per batch", writers: 1, perWriter: 12, batchSize: 1}, + {name: "concurrent writers, small batches", writers: 4, perWriter: 8, batchSize: 3}, + {name: "concurrent writers, one batch could hold everything", writers: 6, perWriter: 5, batchSize: 1024}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds, dbURI := newCursorWatchTestDatastore(t, b) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + RegisterTypes(conn.TypeMap()) + + pgds, ok := ds.(*pgDatastore) + require.True(ok) + slotName := pgds.ledgerSlotName + + // Delivery starts at the frontier as it stands now, so the ground + // truth is exactly the transactions written below. + cursor := readLedgerFrontier(t, ctx, conn, slotName) + + var mu sync.Mutex + expected := make(map[uint64]struct{}, tc.writers*tc.perWriter) + + group, groupCtx := errgroup.WithContext(ctx) + for writer := 0; writer < tc.writers; writer++ { + group.Go(func() error { + for iteration := 0; iteration < tc.perWriter; iteration++ { + rel := tuple.MustParse(fmt.Sprintf("document:frontier#viewer@user:w%d_i%d", writer, iteration)) + revision, err := ds.ReadWriteTx(groupCtx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + if err != nil { + return err + } + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + if !ok { + return errors.New("write revision missing its transaction ID") + } + mu.Lock() + expected[txid.Uint64] = struct{}{} + mu.Unlock() + } + return nil + }) + } + require.NoError(group.Wait()) + + mu.Lock() + groundTruth := make(map[uint64]struct{}, len(expected)) + for xid := range expected { + groundTruth[xid] = struct{}{} + } + mu.Unlock() + + delivered := make(map[uint64]int, len(groundTruth)) + var lastPosition pglogrepl.LSN + remaining := len(groundTruth) + deadline := time.After(cursorWatchTestTimeout) + + for remaining > 0 { + select { + case <-deadline: + require.Failf("timed out", "%d transactions still undelivered", remaining) + default: + } + + frontier := readLedgerFrontier(t, ctx, conn, slotName) + if frontier <= cursor { + time.Sleep(10 * time.Millisecond) + continue + } + + rows, err := conn.Query(ctx, cursorWatchRevisionsQuery, cursor.String(), frontier.String(), tc.batchSize) + require.NoError(err) + + type deliveredRow struct { + xid uint64 + position pglogrepl.LSN + } + batch := make([]deliveredRow, 0, tc.batchSize) + for rows.Next() { + revision, positionText, err := scanWatchRevisionRow(rows) + require.NoError(err) + require.NotNil(positionText, "the discovery query must never return an unrecorded transaction") + + position, err := pglogrepl.ParseLSN(*positionText) + require.NoError(err) + batch = append(batch, deliveredRow{xid: revision.optionalTxID.Uint64, position: position}) + } + require.NoError(rows.Err()) + rows.Close() + + require.LessOrEqual(len(batch), tc.batchSize, "a batch must never exceed the requested size") + + for _, row := range batch { + require.Greater(row.position, lastPosition, "positions must strictly ascend across batches") + require.LessOrEqual(row.position, frontier, "no delivered position may exceed the frontier that bounded its batch") + lastPosition = row.position + + delivered[row.xid]++ + if _, isGroundTruth := groundTruth[row.xid]; isGroundTruth && delivered[row.xid] == 1 { + remaining-- + } + } + + if len(batch) > 0 { + cursor = batch[len(batch)-1].position + continue + } + + // Nothing in the window: the frontier covers only transactions + // this cursor has already passed. + cursor = frontier + } + + for xid := range groundTruth { + require.Equal(1, delivered[xid], "transaction %d was not delivered exactly once", xid) + } + for xid, count := range delivered { + require.Equal(1, count, "transaction %d was delivered %d times", xid, count) + } + }) + } +} + +// readLedgerFrontier reads the ledger slot's confirmed position, which is the +// bound the cursor watch delivers up to. +func readLedgerFrontier(t *testing.T, ctx context.Context, conn *pgx.Conn, slotName string) pglogrepl.LSN { + t.Helper() + + var confirmedText *string + var active bool + var walStatus, database string + require.NoError(t, conn.QueryRow(ctx, selectSlotStateQuery, slotName).Scan(&confirmedText, &active, &walStatus, &database)) + if confirmedText == nil { + return 0 + } + + confirmed, err := pglogrepl.ParseLSN(*confirmedText) + require.NoError(t, err) + return confirmed +} + +// recordedCommitLSNs reads the commit positions the ledger has recorded, keyed by +// transaction ID. A transaction with no recorded position is absent from the result. +func recordedCommitLSNs(t *testing.T, ctx context.Context, conn *pgx.Conn) map[uint64]string { + t.Helper() + + rows, err := conn.Query(ctx, "SELECT xid::text::bigint, commit_lsn::text FROM ledger_xid_lsn;") + require.NoError(t, err) + defer rows.Close() + + recorded := make(map[uint64]string) + for rows.Next() { + var xid uint64 + var commitLSN string + require.NoError(t, rows.Scan(&xid, &commitLSN)) + recorded[xid] = commitLSN + } + require.NoError(t, rows.Err()) + + return recorded +} + +// testCursorWatchIgnoresGC asserts that physical DELETEs performed by garbage +// collection never surface as relationship removals, and that a watch keeps +// running across a collection pass. +func testCursorWatchIgnoresGC(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // A tiny GC window makes everything soft-deleted immediately collectable. + ds := newDatastore(t, 0, 1*time.Millisecond, 128, true) + + relCreated := tuple.MustParse("document:gcdoc#viewer@user:kept") + relDeleted := tuple.MustParse("document:gcdoc#viewer@user:removed") + + _, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(relCreated), tuple.Touch(relDeleted)}) + }) + require.NoError(err) + + _, err = ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Delete(relDeleted)}) + }) + require.NoError(err) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + // Prove the watch is delivering before running GC. + marker := tuple.MustParse("document:gcdoc#viewer@user:marker") + markerRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(marker)}) + }) + require.NoError(err) + + seen := collectChangesUntilRevision(t, changes, errchan, markerRevision) + + // Run garbage collection, physically deleting the soft-deleted row and old + // transaction rows. + pgds, ok := ds.(*pgDatastore) + require.True(ok) + + gc, err := pgds.BuildGarbageCollector(ctx) + require.NoError(err) + defer gc.Close() + + now, err := gc.Now(ctx) + require.NoError(err) + + collectBefore, err := gc.TxIDBefore(ctx, now) + require.NoError(err) + + deleted, err := gc.DeleteBeforeTx(ctx, collectBefore) + require.NoError(err) + require.GreaterOrEqual(deleted.Relationships, int64(1), "expected garbage collection to delete at least the soft-deleted relationship row") + + // A final write proves the watch survived GC and stayed ordered. A running + // watch is not failed by collection below its own position: it has already + // delivered everything down there. + relAfter := tuple.MustParse("document:gcdoc#viewer@user:after") + afterRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(relAfter)}) + }) + require.NoError(err) + + seen = append(seen, collectChangesUntilRevision(t, changes, errchan, afterRevision)...) + + // The only relationship changes observed must be the two live writes: the + // collected rows must not have produced phantom removals. + var observed []string + for _, change := range seen { + for _, relChange := range change.RelationshipChanges { + observed = append(observed, relChange.DebugString()) + } + } + + require.ElementsMatch([]string{ + tuple.Touch(marker).DebugString(), + tuple.Touch(relAfter).DebugString(), + }, observed, "garbage collection must not emit any relationship changes") +} + +// testCursorWatchParity runs the polling watch and the cursor watch side by side +// over the same database, drives an identical workload including concurrent +// writers committing out of transaction-ID order, and asserts that both produce +// observationally equivalent changes, transaction by transaction. +func testCursorWatchParity(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + cursorDS, dbURI := newCursorWatchTestDatastore(t, b) + + pollingDS, err := newPostgresDatastore( + ctx, dbURI, primaryInstanceID, + RevisionQuantization(0), + GCWindow(1000*time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(512), + WithRevisionHeartbeat(false), + WithLogicalWatch(false), + ) + require.NoError(err) + t.Cleanup(func() { _ = pollingDS.Close() }) + + headRevision, err := cursorDS.HeadRevision(ctx) + require.NoError(err) + + watchOptions := datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchSchema | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + } + + cursorChanges, cursorErrs := cursorDS.Watch(ctx, headRevision.Revision, watchOptions) + require.Empty(cursorErrs) + pollingChanges, pollingErrs := pollingDS.Watch(ctx, headRevision.Revision, watchOptions) + require.Empty(pollingErrs) + + metadata, err := structpb.NewStruct(map[string]any{"reason": "parity"}) + require.NoError(err) + + plainRel := tuple.MustParse("paritydoc:doc1#viewer@parityuser:alice") + caveatedRel := tuple.MustParse(`paritydoc:doc1#viewer@parityuser:bob[paritycaveat:{"tenant":"one"}]`) + retouchedRel := tuple.MustParse(`paritydoc:doc1#viewer@parityuser:bob[paritycaveat:{"tenant":"two"}]`) + + workloadSteps := []struct { + name string + run func(ctx context.Context) error + }{ + { + name: "write schema definitions", + run: func(ctx context.Context) error { + _, err := cursorDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + if err := rwt.LegacyWriteNamespaces(ctx, &core.NamespaceDefinition{Name: "paritydoc"}, &core.NamespaceDefinition{Name: "parityuser"}); err != nil { + return err + } + return rwt.LegacyWriteCaveats(ctx, []*core.CaveatDefinition{{Name: "paritycaveat", SerializedExpression: []byte("parity-expression")}}) + }) + return err + }, + }, + { + name: "write caveated relationships with transaction metadata", + run: func(ctx context.Context) error { + _, err := cursorDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(plainRel), tuple.Touch(caveatedRel)}) + }, options.WithMetadata(metadata)) + return err + }, + }, + { + name: "touch changing the caveat context (delete-old + insert-new)", + run: func(ctx context.Context) error { + _, err := cursorDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(retouchedRel)}) + }) + return err + }, + }, + { + name: "delete a relationship", + run: func(ctx context.Context) error { + _, err := cursorDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Delete(plainRel)}) + }) + return err + }, + }, + { + name: "concurrent writers commit out of transaction-ID order", + run: func(ctx context.Context) error { + group, groupCtx := errgroup.WithContext(ctx) + for writer := 0; writer < 5; writer++ { + group.Go(func() error { + for iteration := 0; iteration < 4; iteration++ { + rel := tuple.MustParse(fmt.Sprintf("paritydoc:concurrent#viewer@parityuser:w%d_i%d", writer, iteration)) + if _, err := cursorDS.ReadWriteTx(groupCtx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }); err != nil { + return err + } + } + return nil + }) + } + return group.Wait() + }, + }, + { + name: "rewrite a namespace definition", + run: func(ctx context.Context) error { + _, err := cursorDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.LegacyWriteNamespaces(ctx, &core.NamespaceDefinition{Name: "paritydoc", Metadata: &core.Metadata{}}) + }) + return err + }, + }, + { + name: "delete a namespace", + run: func(ctx context.Context) error { + _, err := cursorDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.LegacyDeleteNamespaces(ctx, []string{"paritydoc"}, datastore.DeleteNamespacesOnly) + }) + return err + }, + }, + { + name: "delete a caveat", + run: func(ctx context.Context) error { + _, err := cursorDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.LegacyDeleteCaveats(ctx, []string{"paritycaveat"}) + }) + return err + }, + }, + } + + for _, step := range workloadSteps { + require.NoError(step.run(ctx), "workload step %q failed", step.name) + } + + finalRevision, err := cursorDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("paritysentinel:final#viewer@parityuser:done"))}) + }) + require.NoError(err) + + cursorCollected := collectChangesUntilRevision(t, cursorChanges, cursorErrs, finalRevision) + pollingCollected := collectChangesUntilRevision(t, pollingChanges, pollingErrs, finalRevision) + + cursorNormalized := normalizeChangesByTransaction(t, cursorCollected) + pollingNormalized := normalizeChangesByTransaction(t, pollingCollected) + + require.Equal(pollingNormalized, cursorNormalized, "the cursor watch and the polling watch must produce observationally equivalent changes") +} + +// runOverlappingTransactionPair deterministically produces a pair of +// overlapping (snapshot-concurrent) transactions. The holder transaction opens +// and writes holderRel, then blocks until the inner transaction has written +// innerRel and committed, and only then commits. The inner therefore always +// commits strictly inside the holder's open window: the two are genuinely +// concurrent (each snapshot has the other in flight) and the inner commits +// first. Retries are disabled so an unexpected conflict fails fast instead of +// silently rerunning the choreography. Both relationships must be distinct so +// the two transactions never conflict. +// +// It returns (innerRevision, holderRevision): the inner committed first (and +// therefore carries the smaller commit position), the holder last. +func runOverlappingTransactionPair(t *testing.T, ctx context.Context, ds datastore.Datastore, holderRel, innerRel tuple.Relationship) (postgresRevision, postgresRevision) { + t.Helper() + + holderWritten := make(chan struct{}) + innerCommitted := make(chan struct{}) + holderDone := make(chan struct{}) + + var rawHolderRev datastore.Revision + var holderErr error + go func() { + defer close(holderDone) + rawHolderRev, holderErr = ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + if err := rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(holderRel)}); err != nil { + return err + } + close(holderWritten) + select { + case <-innerCommitted: + return nil + case <-time.After(cursorWatchTestTimeout): + return errors.New("timed out waiting for the inner transaction to commit") + } + }, options.WithDisableRetries(true)) + }() + + select { + case <-holderWritten: + case <-time.After(cursorWatchTestTimeout): + require.FailNow(t, "timed out waiting for the holder transaction to open") + } + + rawInnerRev, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(innerRel)}) + }, options.WithDisableRetries(true)) + require.NoError(t, err) + close(innerCommitted) + + <-holderDone + require.NoError(t, holderErr) + + innerRev, ok := rawInnerRev.(postgresRevision) + require.True(t, ok) + holderRev, ok := rawHolderRev.(postgresRevision) + require.True(t, ok) + return innerRev, holderRev +} + +// testCursorWatchOverlappingTransactions choreographs two deterministically +// overlapping transactions (B starts first, A starts and commits entirely within +// B's lifetime, B commits last) and asserts the full ordering for the pair: the +// snapshots abstain from ordering them, the commit positions order them by commit +// order, wherever the snapshots do speak the positions agree, and reads at the +// emitted tokens follow snapshot visibility. +func testCursorWatchOverlappingTransactions(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, 1000*time.Second, 512, true) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + relA := tuple.MustParse("document:overlap#viewer@user:committed_first") + relB := tuple.MustParse("document:overlap#viewer@user:committed_last") + relC := tuple.MustParse("document:overlap#viewer@user:sequential_control") + + // Deterministically overlap A and B, where B (the holder) holds its + // transaction open until A (the inner) has committed. + writeRevA, writeRevB := runOverlappingTransactionPair(t, ctx, ds, relB, relA) + + // A sequential control transaction, committed after both, that the snapshots + // CAN order against A and B. + revC, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(relC)}) + }) + require.NoError(err) + writeRevC, ok := revC.(postgresRevision) + require.True(ok) + + // The overlapping pair is concurrent, so the write revisions' snapshots + // abstain from ordering them in either direction. + require.False(writeRevA.GreaterThan(writeRevB)) + require.False(writeRevA.LessThan(writeRevB)) + require.False(writeRevA.Equal(writeRevB)) + require.False(writeRevB.GreaterThan(writeRevA)) + require.False(writeRevB.LessThan(writeRevA)) + + // Where the snapshots do define an order, it is the commit order. + require.True(writeRevC.GreaterThan(writeRevA)) + require.True(writeRevC.GreaterThan(writeRevB)) + + events := collectChangesUntilXids(t, changes, errchan, revisionXids(t, writeRevA, writeRevB, writeRevC)) + + indexA, eventA := requireChangeForSubject(t, events, "committed_first") + indexB, eventB := requireChangeForSubject(t, events, "committed_last") + indexC, eventC := requireChangeForSubject(t, events, "sequential_control") + + // Delivery is in commit order: A first, even though B started first. + require.Less(indexA, indexB, "the first-committed transaction must be delivered first") + require.Less(indexB, indexC) + + streamRevA, ok := eventA.Revision.(postgresRevision) + require.True(ok) + streamRevB, ok := eventB.Revision.(postgresRevision) + require.True(ok) + streamRevC, ok := eventC.Revision.(postgresRevision) + require.True(ok) + + // All three carry commit positions, ascending in commit order. + require.NotZero(streamRevA.optionalCommitLSN) + require.NotZero(streamRevB.optionalCommitLSN) + require.NotZero(streamRevC.optionalCommitLSN) + require.Less(streamRevA.optionalCommitLSN, streamRevB.optionalCommitLSN) + require.Less(streamRevB.optionalCommitLSN, streamRevC.optionalCommitLSN) + + // Token comparison agrees with the delivery order for the overlapping pair, + // which the snapshots alone could not order... + require.True(streamRevB.GreaterThan(streamRevA)) + require.True(streamRevA.LessThan(streamRevB)) + require.Less(streamRevA.String(), streamRevB.String(), "revision strings must be byte-sortable in commit order") + + // ...and never contradicts the snapshots where they speak: the position + // order is a linear extension of the snapshot partial order. + require.True(streamRevC.GreaterThan(streamRevA)) + require.True(streamRevC.GreaterThan(streamRevB)) + + // The string form round-trips both components. + for _, streamRev := range []postgresRevision{streamRevA, streamRevB, streamRevC} { + parsed, err := ParseRevisionString(streamRev.String()) + require.NoError(err) + require.True(parsed.Equal(streamRev)) + require.Equal(streamRev.optionalCommitLSN, parsed.(postgresRevision).optionalCommitLSN) + } + + // Reads at the tokens follow snapshot visibility. B's token does not see the + // concurrent A write (B's snapshot predates A's commit), while the sequential + // C token sees everything. A change revision means "the writer's view plus + // its own write", not "everything with a smaller position". + subjectsAtB := readSubjectIDs(t, ctx, ds, eventB.Revision, "document") + require.Contains(subjectsAtB, "committed_last") + require.NotContains(subjectsAtB, "committed_first") + + subjectsAtC := readSubjectIDs(t, ctx, ds, eventC.Revision, "document") + require.Contains(subjectsAtC, "committed_first") + require.Contains(subjectsAtC, "committed_last") + require.Contains(subjectsAtC, "sequential_control") +} + +// testCursorWatchTokenReadRoundTrip asserts that a revision received over the +// cursor watch is usable, after a full round-trip through its string (token) +// form, against the rest of the datastore API of the same deployment: +// CheckRevision accepts it and SnapshotReader serves the state as of that write. +func testCursorWatchTokenReadRoundTrip(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, 1000*time.Second, 512, true) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + relFirst := tuple.MustParse("document:tokendoc#viewer@user:first") + relSecond := tuple.MustParse("document:tokendoc#viewer@user:second") + + revFirst, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(relFirst)}) + }) + require.NoError(err) + + revSecond, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(relSecond)}) + }) + require.NoError(err) + + events := collectChangesUntilXids(t, changes, errchan, revisionXids(t, revFirst, revSecond)) + _, eventFirst := requireChangeForSubject(t, events, "first") + _, eventSecond := requireChangeForSubject(t, events, "second") + + streamRevFirst, ok := eventFirst.Revision.(postgresRevision) + require.True(ok) + streamRevSecond, ok := eventSecond.Revision.(postgresRevision) + require.True(ok) + + // Sequential writes: the snapshot order and the position order agree. + require.True(streamRevSecond.GreaterThan(streamRevFirst)) + require.True(streamRevSecond.snapshot.GreaterThan(streamRevFirst.snapshot)) + require.Less(streamRevFirst.optionalCommitLSN, streamRevSecond.optionalCommitLSN) + + // The token round-trip: what a Watch API consumer receives, parsed back the + // way the API layer parses incoming ZedTokens. + parsedRevisions := make([]datastore.Revision, 0, 2) + for _, event := range []datastore.RevisionChanges{eventFirst, eventSecond} { + streamRev, ok := event.Revision.(postgresRevision) + require.True(ok) + + parsed, err := ParseRevisionString(streamRev.String()) + require.NoError(err) + require.True(parsed.Equal(streamRev)) + require.Equal(streamRev.optionalCommitLSN, parsed.(postgresRevision).optionalCommitLSN) + + // The parsed token is a valid read revision on the same deployment. + require.NoError(ds.CheckRevision(ctx, parsed)) + parsedRevisions = append(parsedRevisions, parsed) + } + + // Reads at the parsed tokens serve the state as of each write. + subjectsAtFirst := readSubjectIDs(t, ctx, ds, parsedRevisions[0], "document") + require.Contains(subjectsAtFirst, "first") + require.NotContains(subjectsAtFirst, "second") + + subjectsAtSecond := readSubjectIDs(t, ctx, ds, parsedRevisions[1], "document") + require.Contains(subjectsAtSecond, "first") + require.Contains(subjectsAtSecond, "second") +} + +// testCursorWatchCommitOrderLinearExtension drives overlapping concurrent +// writers and asserts the stream-level invariants over everything emitted: +// exactly-once delivery, total ordering by commit position, and the +// linear-extension property (a later-delivered change is never snapshot-ordered +// before an earlier one, so the position order refines but never contradicts the +// snapshot partial order). +func testCursorWatchCommitOrderLinearExtension(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, 1000*time.Second, 1024, true) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + // Deterministically produce overlapping (snapshot-concurrent) pairs: in each + // pair the holder holds its write open until the inner has committed. Pairs + // run sequentially, so at most two write connections are held at once and + // the choreography cannot deadlock the pool. + const pairCount = 8 + + type overlappingPair struct { + innerXid uint64 + holderXid uint64 + } + pairs := make([]overlappingPair, 0, pairCount) + awaitedXids := make(map[uint64]struct{}, pairCount*2) + + for p := 0; p < pairCount; p++ { + holderRel := tuple.MustParse(fmt.Sprintf("document:linext#viewer@user:holder_%d", p)) + innerRel := tuple.MustParse(fmt.Sprintf("document:linext#viewer@user:inner_%d", p)) + + innerRev, holderRev := runOverlappingTransactionPair(t, ctx, ds, holderRel, innerRel) + + // The choreographed pair is concurrent: neither write revision's + // snapshot orders the other. + require.False(innerRev.GreaterThan(holderRev)) + require.False(innerRev.LessThan(holderRev)) + require.False(holderRev.GreaterThan(innerRev)) + require.False(holderRev.LessThan(innerRev)) + + innerXid, ok := innerRev.OptionalTransactionID() + require.True(ok) + holderXid, ok := holderRev.OptionalTransactionID() + require.True(ok) + + pairs = append(pairs, overlappingPair{innerXid: innerXid.Uint64, holderXid: holderXid.Uint64}) + awaitedXids[innerXid.Uint64] = struct{}{} + awaitedXids[holderXid.Uint64] = struct{}{} + } + + events := collectChangesUntilXids(t, changes, errchan, awaitedXids) + + observedCounts := make(map[uint64]int, len(awaitedXids)) + emissionIndexByXid := make(map[uint64]int, len(awaitedXids)) + changeRevisions := make([]postgresRevision, 0, len(awaitedXids)) + var lastChangeLSN, lastCheckpointLSN, lastAnyLSN uint64 + + for _, event := range events { + revision, ok := event.Revision.(postgresRevision) + require.True(ok) + require.True(revision.ByteSortable(), "every delivered event must carry a commit position") + + // Delivery never goes backwards, and changes and checkpoints each + // strictly increase (a change and its own checkpoint share a position). + require.GreaterOrEqual(revision.optionalCommitLSN, lastAnyLSN) + lastAnyLSN = revision.optionalCommitLSN + + if event.IsCheckpoint { + require.Greater(revision.optionalCommitLSN, lastCheckpointLSN, "checkpoints must strictly advance") + lastCheckpointLSN = revision.optionalCommitLSN + continue + } + + require.Greater(revision.optionalCommitLSN, lastChangeLSN, "changes must strictly advance") + lastChangeLSN = revision.optionalCommitLSN + + txid, ok := revision.OptionalTransactionID() + require.True(ok) + observedCounts[txid.Uint64]++ + emissionIndexByXid[txid.Uint64] = len(changeRevisions) + changeRevisions = append(changeRevisions, revision) + } + + // Exactly-once delivery: every committed transaction observed once, and + // nothing else observed at all. + for xid := range awaitedXids { + require.Equal(1, observedCounts[xid], "transaction %d must be observed exactly once", xid) + } + for xid := range observedCounts { + _, expected := awaitedXids[xid] + require.True(expected, "unexpected transaction %d was delivered", xid) + } + + // The linear-extension property against real snapshots: an + // earlier-delivered change is never snapshot-greater than a later one. + for i := 0; i < len(changeRevisions); i++ { + for j := i + 1; j < len(changeRevisions); j++ { + require.False(changeRevisions[i].snapshot.GreaterThan(changeRevisions[j].snapshot), + "ordering reversal: change %d is snapshot-greater than later-delivered change %d", i, j) + } + } + + // Every choreographed pair overlapped by construction, so delivery must + // place the inner (committed-first) transaction before the holder, with a + // strictly smaller position. This is the concurrent case where only the + // commit position, not the snapshots, can supply the order. + for _, pair := range pairs { + innerIndex := emissionIndexByXid[pair.innerXid] + holderIndex := emissionIndexByXid[pair.holderXid] + require.Less(innerIndex, holderIndex, "the inner (committed-first) transaction must be delivered before the holder") + require.Less(changeRevisions[innerIndex].optionalCommitLSN, changeRevisions[holderIndex].optionalCommitLSN, + "the inner transaction must carry a smaller commit position than the holder") + } +} + +// testCursorWatchArgumentValidation asserts the argument checks of the cursor +// watch entry point. +func testCursorWatchArgumentValidation(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, 1000*time.Second, 128, true) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + // EmitImmediatelyStrategy requires checkpoints. + _, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships, + EmissionStrategy: datastore.EmitImmediatelyStrategy, + }) + select { + case err := <-errchan: + require.ErrorContains(err, "EmitImmediatelyStrategy requires WatchCheckpoints") + case <-time.After(cursorWatchTestTimeout): + require.Fail("expected an immediate error for EmitImmediatelyStrategy without checkpoints") + } + + // A revision of a foreign type is rejected instead of panicking. + _, errchan = ds.Watch(ctx, datastore.NoRevision, datastore.WatchOptions{ + Content: datastore.WatchRelationships, + }) + select { + case err := <-errchan: + require.Error(err) + case <-time.After(cursorWatchTestTimeout): + require.Fail("expected an immediate error for a non-postgres revision") + } +} + +// testCursorWatchCrossEpochTokenStability asserts that a transaction's position +// is a property of that transaction, not of the Watch call that delivered it. +// Two Watch calls from the same revision must mint byte-identical tokens for the +// same transaction, and tokens minted by different calls must still sort in +// commit order. +// +// This is the defect the ledger exists to fix: with positions fabricated per +// Watch call, a second call replaying old history minted tokens that outranked +// tokens an earlier call had already emitted for newer transactions. +func testCursorWatchCrossEpochTokenStability(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, 1000*time.Second, 512, true) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + writeRelationship := func(subjectID string) uint64 { + rel := tuple.MustParse("document:epoch#viewer@user:" + subjectID) + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + return txid.Uint64 + } + + // Sequential writes, so write order is commit order is snapshot order: the + // ground truth every call's tokens must agree with. + writtenXidsInOrder := make([]uint64, 0, 4) + for i := 0; i < 3; i++ { + writtenXidsInOrder = append(writtenXidsInOrder, writeRelationship(fmt.Sprintf("historical_%d", i))) + } + + watchOptions := datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + } + + // The first call backfills the historical writes, then observes one further + // write as it commits. + firstCtx, cancelFirst := context.WithCancel(ctx) + firstChanges, firstErrs := ds.Watch(firstCtx, headRevision.Revision, watchOptions) + require.Empty(firstErrs) + + firstTokens := collectTokensByXid(t, firstChanges, firstErrs, setOfXids(writtenXidsInOrder...)) + + liveXid := writeRelationship("live") + writtenXidsInOrder = append(writtenXidsInOrder, liveXid) + for xid, token := range collectTokensByXid(t, firstChanges, firstErrs, setOfXids(liveXid)) { + firstTokens[xid] = token + } + cancelFirst() + + // The second call starts from the same revision, so every transaction above, + // including the one the first call saw as it happened, now arrives via the + // backfill. + secondCtx, cancelSecond := context.WithCancel(ctx) + defer cancelSecond() + secondChanges, secondErrs := ds.Watch(secondCtx, headRevision.Revision, watchOptions) + require.Empty(secondErrs) + + secondTokens := collectTokensByXid(t, secondChanges, secondErrs, setOfXids(writtenXidsInOrder...)) + + for _, xid := range writtenXidsInOrder { + require.Equal(firstTokens[xid], secondTokens[xid], + "transaction %d was delivered with different tokens by two watch calls", xid) + } + + // Pairing every second-call token against every first-call token covers the + // inversion an invocation-scoped position produces. + for lhsIndex, lhsXid := range writtenXidsInOrder { + for rhsIndex, rhsXid := range writtenXidsInOrder { + if lhsIndex == rhsIndex { + continue + } + + lhs := positionPrefixOf(t, secondTokens[lhsXid]) + rhs := positionPrefixOf(t, firstTokens[rhsXid]) + if lhsIndex < rhsIndex { + require.Less(lhs, rhs, + "transaction %d committed before %d, but its token does not sort below", lhsXid, rhsXid) + } else { + require.Greater(lhs, rhs, + "transaction %d committed after %d, but its token does not sort above", lhsXid, rhsXid) + } + } + } +} + +// testCursorWatchSameTokenFromAnyCursor asserts that a transaction is positioned +// identically whether it is delivered by a watch that was already running when +// it committed or by one that started afterwards from an older revision. +func testCursorWatchSameTokenFromAnyCursor(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, 1000*time.Second, 512, true) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + watchOptions := datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + } + + runningCtx, cancelRunning := context.WithCancel(ctx) + defer cancelRunning() + runningChanges, runningErrs := ds.Watch(runningCtx, headRevision.Revision, watchOptions) + require.Empty(runningErrs) + + rel := tuple.MustParse("document:bothpaths#viewer@user:subject") + writeRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + writtenXid, ok := writeRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + runningTokens := collectTokensByXid(t, runningChanges, runningErrs, setOfXids(writtenXid.Uint64)) + + // A watcher starting from the older revision delivers the same transaction + // out of the backfill. + replayCtx, cancelReplay := context.WithCancel(ctx) + defer cancelReplay() + replayChanges, replayErrs := ds.Watch(replayCtx, headRevision.Revision, watchOptions) + require.Empty(replayErrs) + + replayedTokens := collectTokensByXid(t, replayChanges, replayErrs, setOfXids(writtenXid.Uint64)) + + require.Equal(runningTokens[writtenXid.Uint64], replayedTokens[writtenXid.Uint64], + "the two cursors delivered transaction %d at different tokens", writtenXid.Uint64) +} + +// testCursorWatchReconnectWithPositionedToken verifies the resume guarantee: a +// consumer that stops and reconnects with its last token receives exactly the +// remaining unseen changes, in order, at the very same tokens. +func testCursorWatchReconnectWithPositionedToken(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, 1000*time.Second, 512, true) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + // All writes commit before either watch exists, so both are served from the + // backfill: fully deterministic. + const writeCount = 8 + writtenXidsInOrder := make([]uint64, 0, writeCount) + for i := 0; i < writeCount; i++ { + rel := tuple.MustParse(fmt.Sprintf("document:reconnect#viewer@user:step_%d", i)) + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + writtenXidsInOrder = append(writtenXidsInOrder, txid.Uint64) + } + + watchOptions := datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + } + + // The first consumer observes the writes, then "crashes" partway, with its + // durable state being the token of the third change. + firstCtx, cancelFirst := context.WithCancel(ctx) + firstChanges, firstErrs := ds.Watch(firstCtx, headRevision.Revision, watchOptions) + require.Empty(firstErrs) + + firstEvents := collectChangesUntilXids(t, firstChanges, firstErrs, setOfXids(writtenXidsInOrder...)) + + firstChangeRevisions := make([]postgresRevision, 0, writeCount) + for _, event := range firstEvents { + if event.IsCheckpoint { + continue + } + firstChangeRevisions = append(firstChangeRevisions, event.Revision.(postgresRevision)) + } + require.Len(firstChangeRevisions, writeCount) + + // Delivery is in commit order, each transaction at its own recorded + // position, which strictly ascends because the writes were sequential. + var lastPosition uint64 + for index, revision := range firstChangeRevisions { + txid, ok := revision.OptionalTransactionID() + require.True(ok) + require.Equal(writtenXidsInOrder[index], txid.Uint64, "delivery order must match write order") + require.Greater(revision.optionalCommitLSN, lastPosition, "positions must strictly ascend") + lastPosition = revision.optionalCommitLSN + } + + const resumeAfter = 3 + resumeToken := firstChangeRevisions[resumeAfter-1].String() + cancelFirst() + + // The second consumer resumes from that token, round-tripped through its + // string form the way a real consumer would store it. + parsedToken, err := ParseRevisionString(resumeToken) + require.NoError(err) + + secondChanges, secondErrs := ds.Watch(ctx, parsedToken, watchOptions) + require.Empty(secondErrs) + + secondEvents := collectChangesUntilXids(t, secondChanges, secondErrs, setOfXids(writtenXidsInOrder[resumeAfter:]...)) + + secondChangeRevisions := make([]postgresRevision, 0, writeCount-resumeAfter) + for _, event := range secondEvents { + // Continuity: every event after the resume sorts strictly above the token. + require.Greater(event.Revision.String(), resumeToken, "event does not sort above the resume token") + if event.IsCheckpoint { + continue + } + secondChangeRevisions = append(secondChangeRevisions, event.Revision.(postgresRevision)) + } + + // Exactly the unseen remainder arrives, in the same order, at the very same + // tokens the first watch delivered them at: a resumed watch is not a new + // epoch, because positions belong to transactions. + require.Len(secondChangeRevisions, writeCount-resumeAfter) + for index, revision := range secondChangeRevisions { + txid, ok := revision.OptionalTransactionID() + require.True(ok) + require.Equal(writtenXidsInOrder[resumeAfter+index], txid.Uint64, "resumed delivery order must match write order") + require.Equal(firstChangeRevisions[resumeAfter+index].String(), revision.String(), + "transaction %d was re-delivered at a different token", txid.Uint64) + } + + // And what commits afterwards continues sorting above everything before it. + liveRel := tuple.MustParse("document:reconnect#viewer@user:live") + revLive, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(liveRel)}) + }) + require.NoError(err) + + liveEvents := collectChangesUntilXids(t, secondChanges, secondErrs, revisionXids(t, revLive)) + _, liveEvent := requireChangeForSubject(t, liveEvents, "live") + liveRevision := liveEvent.Revision.(postgresRevision) + require.Greater(liveRevision.optionalCommitLSN, lastPosition) + require.Greater(liveRevision.String(), secondChangeRevisions[len(secondChangeRevisions)-1].String()) +} + +// testCursorWatchConcurrentWatchers runs two watchers from the same revision. +// Each writes a marker transaction that the other can see, but no marker may be +// delivered as a change. Both must receive the same transactions exactly once, in +// strictly increasing position order. +func testCursorWatchConcurrentWatchers(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, 1000*time.Second, 512, true) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + // With sequential writes, write order is commit order, so both watchers must + // deliver write order regardless of where each backfill boundary falls. + writtenXidsInOrder := make([]uint64, 0, 12) + writeOne := func(index int) { + rel := tuple.MustParse(fmt.Sprintf("document:duo#viewer@user:write_%d", index)) + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + writtenXidsInOrder = append(writtenXidsInOrder, txid.Uint64) + } + + for i := 0; i < 6; i++ { + writeOne(i) + } + + watchOptions := datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + } + + changesA, errsA := ds.Watch(ctx, headRevision.Revision, watchOptions) + require.Empty(errsA) + changesB, errsB := ds.Watch(ctx, headRevision.Revision, watchOptions) + require.Empty(errsB) + + // More writes racing both watchers' handoffs. + for i := 6; i < 12; i++ { + writeOne(i) + } + + awaited := setOfXids(writtenXidsInOrder...) + + for name, stream := range map[string]struct { + changes <-chan datastore.RevisionChanges + errs <-chan error + }{ + "A": {changesA, errsA}, + "B": {changesB, errsB}, + } { + events := collectChangesUntilXids(t, stream.changes, stream.errs, awaited) + + observedXidsInOrder := make([]uint64, 0, len(writtenXidsInOrder)) + var lastChangeToken string + for index, event := range events { + revision, ok := event.Revision.(postgresRevision) + require.True(ok) + require.True(revision.ByteSortable(), "watcher %s: event %d is missing a position", name, index) + + if event.IsCheckpoint { + continue + } + + token := revision.String() + require.Greater(token, lastChangeToken, "watcher %s: change %d does not sort strictly above the previous change", name, index) + lastChangeToken = token + + txid, ok := revision.OptionalTransactionID() + require.True(ok) + observedXidsInOrder = append(observedXidsInOrder, txid.Uint64) + } + + // The identical sequence, exactly once, and nothing else: in particular, + // neither watcher's marker may surface as a change on the other's stream. + require.Equal(writtenXidsInOrder, observedXidsInOrder, "watcher %s must observe the writes exactly once, in order, with no extras", name) + } +} + +// testCursorWatchDisconnectDuringBackfillLosesNothing verifies that a watch +// disconnected mid-backfill loses nothing: retrying from the same revision +// delivers every transaction exactly once, at the same tokens. The disconnect is +// forced deterministically with a one-slot buffer, a nanosecond write timeout, +// and a consumer that never reads. +func testCursorWatchDisconnectDuringBackfillLosesNothing(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, 1000*time.Second, 512, true) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + const writeCount = 5 + writtenXidsInOrder := make([]uint64, 0, writeCount) + for i := 0; i < writeCount; i++ { + rel := tuple.MustParse(fmt.Sprintf("document:backfillfail#viewer@user:write_%d", i)) + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + writtenXidsInOrder = append(writtenXidsInOrder, txid.Uint64) + } + + // This watch is expected to fail: a one-slot buffer, a nanosecond write + // timeout, and no reader. It writes one event to the buffer, then + // disconnects on the second. + doomedChanges, doomedErrs := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + WatchBufferLength: 1, + WatchBufferWriteTimeout: time.Nanosecond, + }) + + select { + case err := <-doomedErrs: + require.ErrorAs(err, &datastore.WatchDisconnectedError{}) + case <-time.After(cursorWatchTestTimeout): + require.Fail("timed out waiting for the unread watch to be disconnected") + } + + // At most the buffered prefix was delivered before the disconnect: exactly + // the first change, and nothing after it. + delivered := make([]datastore.RevisionChanges, 0, 1) + for change := range doomedChanges { + delivered = append(delivered, change) + } + require.Len(delivered, 1, "exactly the buffered prefix must have been delivered") + prefixRevision := delivered[0].Revision.(postgresRevision) + require.True(prefixRevision.ByteSortable()) + prefixXid, ok := prefixRevision.OptionalTransactionID() + require.True(ok) + require.Equal(writtenXidsInOrder[0], prefixXid.Uint64) + + // The retry from the same revision delivers everything exactly once: the + // failed delivery lost nothing, and the one event that did get through + // arrives at the same token it had before. + retryChanges, retryErrs := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(retryErrs) + + events := collectChangesUntilXids(t, retryChanges, retryErrs, setOfXids(writtenXidsInOrder...)) + + observedXidsInOrder := make([]uint64, 0, writeCount) + var lastPosition uint64 + for _, event := range events { + revision := event.Revision.(postgresRevision) + require.True(revision.ByteSortable()) + if event.IsCheckpoint { + continue + } + + require.GreaterOrEqual(revision.optionalCommitLSN, lastPosition, "positions must ascend") + lastPosition = revision.optionalCommitLSN + + txid, ok := revision.OptionalTransactionID() + require.True(ok) + observedXidsInOrder = append(observedXidsInOrder, txid.Uint64) + + if txid.Uint64 == prefixXid.Uint64 { + require.Equal(prefixRevision.String(), revision.String(), + "the re-delivered event must carry the token it was first delivered at") + } + } + + require.Equal(writtenXidsInOrder, observedXidsInOrder, "the retry must deliver every write exactly once, in order, with no extras") +} + +// testCursorWatchLegacyTokenHandoff covers the transition from the backfill +// phase to the cursor loop for a consumer resuming from a snapshot-only token. +// +// It runs with the ledger deliberately stalled while the writes commit, so that +// at the moment the watch starts none of them have a recorded position. That is +// the case a fast local ledger never exercises, and the one the handoff's +// accounting turns on: without the frontier probe, an unrecorded transaction +// visible in the backfill snapshot would be skipped as already-seen and then +// delivered again by the loop once the ledger caught up. +func testCursorWatchLegacyTokenHandoff(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // A long retry interval keeps the ledger down until the test is ready. + ds, dbURI := newCursorWatchTestDatastore(t, b, + LogicalWatchLedgerRetryInterval(3*time.Second), + ) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + legacyToken := headRevision.Revision.(postgresRevision) + require.False(legacyToken.ByteSortable(), "a HeadRevision carries no position") + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + pgds, ok := ds.(*pgDatastore) + require.True(ok) + slotName := pgds.ledgerSlotName + + // Wait for the ledger to attach, then evict it so the writes below commit + // while nothing is recording. + requireLedgerAttached(t, ctx, conn, slotName, true) + _, err = conn.Exec(ctx, "SELECT pg_terminate_backend(active_pid) FROM pg_replication_slots WHERE slot_name = $1 AND active_pid IS NOT NULL;", slotName) + require.NoError(err) + requireLedgerAttached(t, ctx, conn, slotName, false) + + const writeCount = 3 + writtenXidsInOrder := make([]uint64, 0, writeCount) + for i := 0; i < writeCount; i++ { + rel := tuple.MustParse(fmt.Sprintf("document:handoff#viewer@user:unrecorded_%d", i)) + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + writtenXidsInOrder = append(writtenXidsInOrder, txid.Uint64) + } + + // None of them has a recorded position yet, which is the state the handoff + // has to survive. + recorded := recordedCommitLSNs(t, ctx, conn) + for _, xid := range writtenXidsInOrder { + require.NotContains(recorded, xid, "the ledger was expected to be stalled") + } + + // The watch blocks on its frontier probe until the ledger returns, then + // delivers. Everything must arrive exactly once, in order, positioned. + changes, errchan := ds.Watch(ctx, legacyToken, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + events := collectChangesUntilXids(t, changes, errchan, setOfXids(writtenXidsInOrder...)) + + backfilled := make([]uint64, 0, writeCount) + for _, event := range events { + revision := event.Revision.(postgresRevision) + require.True(revision.ByteSortable(), "the backfill must position transactions the ledger recorded") + if event.IsCheckpoint { + continue + } + txid, ok := revision.OptionalTransactionID() + require.True(ok) + backfilled = append(backfilled, txid.Uint64) + } + require.Equal(writtenXidsInOrder, backfilled, "the backfill must deliver every write exactly once, in order") + + // And the loop takes over cleanly: a write after the handoff is delivered + // once, above everything the backfill emitted, with nothing re-delivered. + rel := tuple.MustParse("document:handoff#viewer@user:after_handoff") + afterRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + afterXid, ok := afterRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + afterEvents := collectChangesUntilXids(t, changes, errchan, setOfXids(afterXid.Uint64)) + for _, event := range afterEvents { + if event.IsCheckpoint { + continue + } + txid, ok := event.Revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + require.Equal(afterXid.Uint64, txid.Uint64, + "transaction %d was delivered twice across the backfill/loop handoff", txid.Uint64) + } +} + +// testCursorWatchCheckpointsAreExactOnResume asserts the checkpoint contract: +// resuming from a checkpoint token delivers the transactions above it and +// nothing else, neither losing nor repeating. A small batch size produces +// several checkpoints so the resume happens mid-stream. +func testCursorWatchCheckpointsAreExactOnResume(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + const batchSize = 2 + const writeCount = 7 + + ds, _ := newCursorWatchTestDatastore(t, b, WatchBatchSize(batchSize)) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + writtenXidsInOrder := make([]uint64, 0, writeCount) + for i := 0; i < writeCount; i++ { + rel := tuple.MustParse(fmt.Sprintf("document:exact#viewer@user:write_%d", i)) + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + writtenXidsInOrder = append(writtenXidsInOrder, txid.Uint64) + } + + watchOptions := datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + } + + firstCtx, cancelFirst := context.WithCancel(ctx) + firstChanges, firstErrs := ds.Watch(firstCtx, headRevision.Revision, watchOptions) + require.Empty(firstErrs) + + events := collectChangesUntilXids(t, firstChanges, firstErrs, setOfXids(writtenXidsInOrder...)) + + // Batched delivery: write order, each event positioned, with a checkpoint + // after each batch rather than only at the end. + observedXidsInOrder := make([]uint64, 0, writeCount) + var checkpointTokens []string + var lastPosition uint64 + firstCheckpointIndex := -1 + for index, event := range events { + revision, ok := event.Revision.(postgresRevision) + require.True(ok) + require.True(revision.ByteSortable(), "event %d carries no commit position", index) + require.GreaterOrEqual(revision.optionalCommitLSN, lastPosition, "positions must ascend across batches") + lastPosition = revision.optionalCommitLSN + + if event.IsCheckpoint { + if firstCheckpointIndex < 0 { + firstCheckpointIndex = index + } + checkpointTokens = append(checkpointTokens, revision.String()) + continue + } + + txid, ok := revision.OptionalTransactionID() + require.True(ok) + observedXidsInOrder = append(observedXidsInOrder, txid.Uint64) + } + require.Equal(writtenXidsInOrder, observedXidsInOrder, "delivery must cover every write exactly once, in order") + require.Equal(batchSize, firstCheckpointIndex, "the first batch's checkpoint must directly follow its changes") + require.GreaterOrEqual(len(checkpointTokens), writeCount/batchSize, "expected a checkpoint per completed batch") + cancelFirst() + + // A consumer that persisted the first checkpoint resumes from it and + // receives exactly the writes above it: no loss, no repetition. + resumeToken, err := ParseRevisionString(checkpointTokens[0]) + require.NoError(err) + + secondChanges, secondErrs := ds.Watch(ctx, resumeToken, watchOptions) + require.Empty(secondErrs) + + resumedEvents := collectChangesUntilXids(t, secondChanges, secondErrs, setOfXids(writtenXidsInOrder[batchSize:]...)) + + resumedXidsInOrder := make([]uint64, 0, writeCount-batchSize) + for _, event := range resumedEvents { + require.Greater(event.Revision.String(), checkpointTokens[0], "resumed events must sort above the resume token") + if event.IsCheckpoint { + continue + } + txid, ok := event.Revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + resumedXidsInOrder = append(resumedXidsInOrder, txid.Uint64) + } + require.Equal(writtenXidsInOrder[batchSize:], resumedXidsInOrder, + "resuming from a checkpoint must deliver exactly the transactions above it") +} + +// testCursorWatchBacklogDrainsWithoutSleeping asserts that a backlog larger than +// one batch is delivered in consecutive full batches rather than one batch per +// poll interval. The poll interval is set far longer than the test's patience, so +// only a draining implementation can finish in time. +func testCursorWatchBacklogDrainsWithoutSleeping(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + const batchSize = 2 + const backlogCount = 7 + const pollInterval = 5 * time.Second + + ds, _ := newCursorWatchTestDatastore(t, b, + WatchBatchSize(batchSize), + WatchPollInterval(pollInterval), + ) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + watchOptions := datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + } + + // A first watch yields a positioned token, so the backlog below is drained + // by the cursor loop rather than by the backfill phase. + seedCtx, cancelSeed := context.WithCancel(ctx) + seedChanges, seedErrs := ds.Watch(seedCtx, headRevision.Revision, watchOptions) + require.Empty(seedErrs) + + seedRel := tuple.MustParse("document:backlog#viewer@user:seed") + seedRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(seedRel)}) + }) + require.NoError(err) + + seedEvents := collectChangesUntilXids(t, seedChanges, seedErrs, revisionXids(t, seedRevision)) + _, seedEvent := requireChangeForSubject(t, seedEvents, "seed") + positionedToken := seedEvent.Revision.(postgresRevision) + require.True(positionedToken.ByteSortable()) + cancelSeed() + + backlogXidsInOrder := make([]uint64, 0, backlogCount) + for i := 0; i < backlogCount; i++ { + rel := tuple.MustParse(fmt.Sprintf("document:backlog#viewer@user:item_%d", i)) + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + backlogXidsInOrder = append(backlogXidsInOrder, txid.Uint64) + } + + // Wait until the ledger has recorded the whole backlog and confirmed past + // it, so the frontier already covers all of it when the watch starts and the + // first poll has the entire backlog available to drain. + pgds, ok := ds.(*pgDatastore) + require.True(ok) + lastBacklogXid := NewXid8(backlogXidsInOrder[len(backlogXidsInOrder)-1]) + require.EventuallyWithT(func(collect *assert.CollectT) { + var lastPositionText *string + if !assert.NoError(collect, pgds.readPool.QueryRow(ctx, commitLSNForXidQuery, lastBacklogXid).Scan(&lastPositionText)) { + return + } + if !assert.NotNil(collect, lastPositionText, "the last backlog transaction has no recorded position yet") { + return + } + + lastPosition, err := pglogrepl.ParseLSN(*lastPositionText) + if !assert.NoError(collect, err) { + return + } + state, err := pgds.readLedgerSlotState(ctx) + if !assert.NoError(collect, err) { + return + } + assert.GreaterOrEqual(collect, state.confirmed, lastPosition, "the frontier has not reached the end of the backlog") + }, cursorWatchTestTimeout, 20*time.Millisecond) + + changes, errchan := ds.Watch(ctx, positionedToken, watchOptions) + require.Empty(errchan) + + // The backlog spans four batches. Delivering it within a fraction of one + // poll interval is only possible without sleeping between full batches. + startedAt := time.Now() + events := collectChangesUntilXids(t, changes, errchan, setOfXids(backlogXidsInOrder...)) + elapsed := time.Since(startedAt) + require.Less(elapsed, pollInterval, "the backlog was drained one batch per poll interval instead of consecutively") + + observed := make([]uint64, 0, backlogCount) + for _, event := range events { + if event.IsCheckpoint { + continue + } + txid, ok := event.Revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + observed = append(observed, txid.Uint64) + } + require.Equal(backlogXidsInOrder, observed, "the drained backlog must arrive exactly once, in order") +} + +// testCursorWatchIdleCheckpointsFollowTheFrontier covers the idle path: with no +// SpiceDB writes but other WAL activity, the ledger confirms its progress +// anyway, and the watch turns that into checkpoints that advance. +// +// It also pins what such a checkpoint's snapshot must be. Delivery is complete +// only through the frontier, so an idle checkpoint carries the last delivered +// transaction's snapshot. Using a fresh pg_current_snapshot() would cover +// transactions above the frontier that have NOT been delivered, and feeding that +// token to any snapshot-filtered consumer would drop them silently. +func testCursorWatchIdleCheckpointsFollowTheFrontier(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // A one-second status interval bounds how long the ledger holds its + // confirmed position back from the slot. + ds, dbURI := newCursorWatchTestDatastore(t, b, LogicalWatchStatusInterval(time.Second)) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + // One write, so there is a "last delivered transaction" to compare against. + rel := tuple.MustParse("document:idle#viewer@user:only_write") + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + writtenXid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + events := collectChangesUntilXids(t, changes, errchan, setOfXids(writtenXid.Uint64)) + + var deliveredCheckpoint postgresRevision + for _, event := range events { + if event.IsCheckpoint { + deliveredCheckpoint = event.Revision.(postgresRevision) + } + } + if !deliveredCheckpoint.ByteSortable() { + // The batch's checkpoint trails its changes; wait for it. + deliveredCheckpoint = awaitCheckpoint(t, changes, errchan, 0) + } + + // WAL activity that has nothing to do with SpiceDB: it produces no watch + // event, and nothing for the ledger to record, but it does move the server's + // WAL position, which the ledger may confirm and the watch may checkpoint. + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + _, err = conn.Exec(ctx, "CREATE TABLE unrelated_wal (id BIGSERIAL PRIMARY KEY, payload TEXT);") + require.NoError(err) + + // The generator gets its own connection: a pgx connection is not safe for + // concurrent use, and sharing one here would race this test's teardown + // against an insert still in flight. + generateCtx, stopGenerating := context.WithCancel(ctx) + var generating sync.WaitGroup + generating.Add(1) + go func() { + defer generating.Done() + + generateConn, err := pgx.Connect(generateCtx, dbURI) + if err != nil { + return + } + defer func() { _ = generateConn.Close(context.Background()) }() + + for generateCtx.Err() == nil { + _, _ = generateConn.Exec(generateCtx, "INSERT INTO unrelated_wal (payload) SELECT repeat('x', 1000) FROM generate_series(1, 50);") + time.Sleep(50 * time.Millisecond) + } + }() + defer func() { + stopGenerating() + generating.Wait() + }() + + // An idle checkpoint above the delivered one, carrying its snapshot + // unchanged, and no change event in between. + idleCheckpoint := awaitCheckpoint(t, changes, errchan, deliveredCheckpoint.optionalCommitLSN) + stopGenerating() + + require.Greater(idleCheckpoint.optionalCommitLSN, deliveredCheckpoint.optionalCommitLSN, + "an idle checkpoint must advance with the frontier") + require.Equal(deliveredCheckpoint.snapshot, idleCheckpoint.snapshot, + "an idle checkpoint must carry the last delivered transaction's snapshot, not a fresh one") + + // The honesty check: a consumer that resumes from the idle checkpoint still + // receives everything committed after it. + nextRel := tuple.MustParse("document:idle#viewer@user:after_idle") + nextRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(nextRel)}) + }) + require.NoError(err) + + resumeChanges, resumeErrs := ds.Watch(ctx, idleCheckpoint, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(resumeErrs) + + resumed := collectChangesUntilXids(t, resumeChanges, resumeErrs, revisionXids(t, nextRevision)) + _, _ = requireChangeForSubject(t, resumed, "after_idle") +} + +// awaitCheckpoint reads until a checkpoint above the given position arrives, +// requiring that no change event arrives before it. +func awaitCheckpoint(t *testing.T, changes <-chan datastore.RevisionChanges, errchan <-chan error, above uint64) postgresRevision { + t.Helper() + + timeout := time.After(cursorWatchTestTimeout) + for { + select { + case change, ok := <-changes: + require.True(t, ok, "the watch closed while waiting for a checkpoint") + revision, ok := change.Revision.(postgresRevision) + require.True(t, ok) + require.True(t, change.IsCheckpoint, "an unexpected change event arrived while idle: %v", change) + if revision.optionalCommitLSN > above { + return revision + } + case err := <-errchan: + require.NoError(t, err, "unexpected watch error") + case <-timeout: + require.Fail(t, "timed out waiting for a checkpoint to advance") + } + } +} + +// testCursorWatchStaleRevisionRejected asserts that a consumer resuming from a +// revision older than the garbage collection window is told so, rather than +// served a silently truncated stream. Both existing watches do the latter. +// +// The window is set to a millisecond, so a token delivered a moment ago already +// names a collectable transaction. +func testCursorWatchStaleRevisionRejected(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, time.Millisecond, 128, true) + + watchOptions := datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + } + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + // A HeadRevision names no transaction, so it is served: there is nothing to + // have fallen behind of. + seedCtx, cancelSeed := context.WithCancel(ctx) + seedChanges, seedErrs := ds.Watch(seedCtx, headRevision.Revision, watchOptions) + require.Empty(seedErrs) + + seedRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:stale#viewer@user:old"))}) + }) + require.NoError(err) + + seedEvents := collectChangesUntilXids(t, seedChanges, seedErrs, revisionXids(t, seedRevision)) + _, seedEvent := requireChangeForSubject(t, seedEvents, "old") + staleToken := seedEvent.Revision.(postgresRevision) + cancelSeed() + + // Past the window. + time.Sleep(20 * time.Millisecond) + + changes, errchan := ds.Watch(ctx, staleToken, watchOptions) + + select { + case err := <-errchan: + var invalid datastore.InvalidRevisionError + require.ErrorAs(err, &invalid) + require.Equal(datastore.RevisionStale, invalid.Reason()) + case change := <-changes: + require.Fail("the watch delivered an event from a revision outside the retained window", "%v", change) + case <-time.After(cursorWatchTestTimeout): + require.Fail("the watch neither failed nor delivered for a stale revision") + } +} + +// testCursorWatchGapBelowCursorFailsLoudly asserts that a recorded gap above a +// watch's cursor fails the watch instead of being stepped over. +// +// This is the hazard the cursor design introduces and the gap table exists to +// close: transactions that committed while the ledger's slot was invalid have no +// recorded position, so they match neither the cursor's lower bound nor the +// frontier's upper bound, and nothing else would notice their absence. +func testCursorWatchGapBelowCursorFailsLoudly(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds, dbURI := newCursorWatchTestDatastore(t, b) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + watchOptions := datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + } + + // A positioned token to resume from. + seedCtx, cancelSeed := context.WithCancel(ctx) + seedChanges, seedErrs := ds.Watch(seedCtx, headRevision.Revision, watchOptions) + require.Empty(seedErrs) + + seedRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:gapguard#viewer@user:seed"))}) + }) + require.NoError(err) + + seedEvents := collectChangesUntilXids(t, seedChanges, seedErrs, revisionXids(t, seedRevision)) + _, seedEvent := requireChangeForSubject(t, seedEvents, "seed") + positionedToken := seedEvent.Revision.(postgresRevision) + cancelSeed() + + // Record a gap starting at that position: from here on, this consumer + // cannot be shown to have received everything. + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + gapFrom := pglogrepl.LSN(positionedToken.optionalCommitLSN) + _, err = conn.Exec(ctx, + "INSERT INTO ledger_gap (from_lsn, to_lsn) VALUES ($1::pg_lsn, $2::pg_lsn);", + gapFrom.String(), (gapFrom + 0x1000).String()) + require.NoError(err) + + // Writes that a gap-blind implementation would happily deliver. + _, err = ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:gapguard#viewer@user:across_the_gap"))}) + }) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, positionedToken, watchOptions) + + select { + case err := <-errchan: + require.ErrorContains(err, "no recorded commit position") + require.ErrorContains(err, "restart from a current revision") + case change := <-changes: + require.Fail("the watch delivered an event across a recorded gap", "%v", change) + case <-time.After(cursorWatchTestTimeout): + require.Fail("the watch neither failed nor delivered across a recorded gap") + } +} + +// newPreLedgerHistory creates a database whose history predates the commit LSN +// ledger: the writes happen with the cursor watch disabled, so they commit with +// no recorded position, exactly as a database adopting the feature would have. +// It returns the database URI and the transaction ids of that history, oldest +// first. +func newPreLedgerHistory(t *testing.T, b testdatastore.RunningEngineForTest, subjects ...string) (string, []uint64) { + t.Helper() + require := require.New(t) + + var dbURI string + preLedgerDS := b.NewDatastore(t, func(engine, uri string) datastore.Datastore { + dbURI = uri + ds, err := newPostgresDatastore(t.Context(), uri, primaryInstanceID, + RevisionQuantization(0), + GCWindow(1000*time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(512), + WithRevisionHeartbeat(false), + WithLogicalWatch(false), + ) + require.NoError(err) + return ds + }) + + xids := make([]uint64, 0, len(subjects)) + for _, subject := range subjects { + revision, err := preLedgerDS.ReadWriteTx(t.Context(), func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{ + tuple.Touch(tuple.MustParse("document:pre#viewer@user:" + subject)), + }) + }) + require.NoError(err) + xid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + xids = append(xids, xid.Uint64) + } + require.NoError(preLedgerDS.Close()) + + return dbURI, xids +} + +// enableCursorWatchOn brings the cursor watch up against an existing database, +// which is what provisions the ledger for the first time. +func enableCursorWatchOn(t *testing.T, dbURI string, options ...Option) datastore.Datastore { + t.Helper() + + allOptions := append([]Option{ + RevisionQuantization(0), + GCWindow(1000 * time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(512), + WithRevisionHeartbeat(false), + WithLogicalWatch(true), + }, options...) + + ds, err := newPostgresDatastore(t.Context(), dbURI, primaryInstanceID, allOptions...) + require.NoError(t, err) + t.Cleanup(func() { _ = ds.Close() }) + return ds +} + +// testPreLedgerPositionsAreBackfilled asserts that history predating the ledger +// is given positions once the ledger is provisioned, so that a consumer resuming +// across it compares tokens rather than meeting an unpositioned prefix. +// +// The positions must carry commit order and must sort below everything the +// ledger records itself, which is what makes them safe to mix with real ones. +func testPreLedgerPositionsAreBackfilled(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + dbURI, preXids := newPreLedgerHistory(t, b, "first", "second", "third") + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + require.NotContains(recordedCommitLSNs(t, ctx, conn), preXids[0], + "history written before the ledger must start out unpositioned") + + ds := enableCursorWatchOn(t, dbURI, LogicalWatchLedgerRetryInterval(100*time.Millisecond)) + + // The backfill rides on the ledger's flushes, which writes drive. + var recorded map[uint64]string + require.EventuallyWithT(func(collect *assert.CollectT) { + _, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{ + tuple.Touch(tuple.MustParse("document:pre#viewer@user:driver")), + }) + }) + assert.NoError(collect, err) + + recorded = recordedCommitLSNs(t, ctx, conn) + for _, xid := range preXids { + assert.Contains(collect, recorded, xid, "pre-ledger history was not backfilled") + } + }, cursorWatchTestTimeout, 100*time.Millisecond) + + // Commit order, oldest lowest. + positions := make([]pglogrepl.LSN, 0, len(preXids)) + for _, xid := range preXids { + lsn, err := pglogrepl.ParseLSN(recorded[xid]) + require.NoError(err) + positions = append(positions, lsn) + } + for i := 1; i < len(positions); i++ { + require.Less(positions[i-1], positions[i], + "backfilled positions must follow commit order, oldest lowest") + } + + // And below everything the ledger records for itself. + liveRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{ + tuple.Touch(tuple.MustParse("document:pre#viewer@user:live")), + }) + }) + require.NoError(err) + liveXid, ok := liveRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + var liveLSN pglogrepl.LSN + require.EventuallyWithT(func(collect *assert.CollectT) { + text, found := recordedCommitLSNs(t, ctx, conn)[liveXid.Uint64] + if !assert.True(collect, found) { + return + } + parsed, err := pglogrepl.ParseLSN(text) + assert.NoError(collect, err) + liveLSN = parsed + }, cursorWatchTestTimeout, 50*time.Millisecond) + + require.Less(positions[len(positions)-1], liveLSN, + "a backfilled position must sort below what the ledger records") + + // Re-running must not move a position already handed out. + before := recordedCommitLSNs(t, ctx, conn) + for range 3 { + _, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{ + tuple.Touch(tuple.MustParse("document:pre#viewer@user:again")), + }) + }) + require.NoError(err) + } + after := recordedCommitLSNs(t, ctx, conn) + for _, xid := range preXids { + require.Equal(before[xid], after[xid], "a backfilled position must be stable") + } +} + +// testPreLedgerBackfillStopsAtGCWindow asserts that history older than the +// collection horizon is left unpositioned. Such a revision is already refused as +// stale, so it can never be resumed from, and positioning it would only invite +// comparisons against transactions that may already be collected. +func testPreLedgerBackfillStopsAtGCWindow(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + dbURI, preXids := newPreLedgerHistory(t, b, "stale") + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + // A collection horizon this short puts the existing history behind it. + ds := enableCursorWatchOn(t, dbURI, + GCWindow(time.Nanosecond), + LogicalWatchLedgerRetryInterval(100*time.Millisecond), + ) + + liveRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{ + tuple.Touch(tuple.MustParse("document:pre#viewer@user:live")), + }) + }) + require.NoError(err) + liveXid, ok := liveRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + // Once the ledger has recorded a live write it has had its chance to backfill. + require.EventuallyWithT(func(collect *assert.CollectT) { + assert.Contains(collect, recordedCommitLSNs(t, ctx, conn), liveXid.Uint64) + }, cursorWatchTestTimeout, 50*time.Millisecond) + + require.NotContains(recordedCommitLSNs(t, ctx, conn), preXids[0], + "history behind the collection horizon must be left unpositioned") +} + +// testLedgerGapIsReplayedFromTheTables asserts that transactions a slot +// recreation swallowed are replayed out of the tables rather than failing every +// watch below them, and that a replay carries removals as well as additions. +// +// A gap loses commit positions, not changes: the rows are still there, and a +// removal is a soft delete that survives until collection. So the window can be +// given positions after the fact, and the watch delivers it as ordinary +// changes. +func testLedgerGapIsReplayedFromTheTables(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds, dbURI := newCursorWatchTestDatastore(t, b, + LogicalWatchLedgerRetryInterval(100*time.Millisecond), + ) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + pgds, ok := ds.(*pgDatastore) + require.True(ok) + slotName := pgds.ledgerSlotName + + // Established and recorded before the outage, so the gap has a sound lower + // bound and there is something for the outage to remove. + seeded := tuple.MustParse("document:heal#viewer@user:removed") + beforeRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(seeded)}) + }) + require.NoError(err) + + beforeXid, ok := beforeRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + require.EventuallyWithT(func(collect *assert.CollectT) { + assert.Contains(collect, recordedCommitLSNs(t, ctx, conn), beforeXid.Uint64) + }, cursorWatchTestTimeout, 20*time.Millisecond) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + requireLedgerAttached(t, ctx, conn, slotName, true) + + // Drop the slot so the ledger stops recording, then commit through the + // outage: one addition and one removal, the removal being the case a + // re-assertion could never recover. + _, err = conn.Exec(ctx, "SELECT pg_terminate_backend(active_pid) FROM pg_replication_slots WHERE slot_name = $1 AND active_pid IS NOT NULL;", slotName) + require.NoError(err) + _, err = conn.Exec(ctx, "SELECT pg_drop_replication_slot($1);", slotName) + require.NoError(err) + + added := tuple.MustParse("document:heal#viewer@user:added") + addedRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(added)}) + }) + require.NoError(err) + + removedRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Delete(seeded)}) + }) + require.NoError(err) + + addedXid, ok := addedRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + removedXid, ok := removedRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + // The gap has to be recorded before it can be replayed; asserting on its + // absence first would pass before it was ever opened. + require.EventuallyWithT(func(collect *assert.CollectT) { + var gapCount int + assert.NoError(collect, conn.QueryRow(ctx, "SELECT count(*) FROM ledger_gap;").Scan(&gapCount)) + assert.NotZero(collect, gapCount, "the dropped slot must be recorded as a gap") + }, cursorWatchTestTimeout, 50*time.Millisecond) + + // The replay runs once the re-provisioned ledger has confirmed past the + // outage, which the writes in this loop drive. + require.EventuallyWithT(func(collect *assert.CollectT) { + _, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:heal#viewer@user:after"))}) + }) + assert.NoError(collect, err) + + recorded := recordedCommitLSNs(t, ctx, conn) + assert.Contains(collect, recorded, addedXid.Uint64, "the addition the outage swallowed was not replayed") + assert.Contains(collect, recorded, removedXid.Uint64, "the removal the outage swallowed was not replayed") + }, cursorWatchTestTimeout, 100*time.Millisecond) + + // With every swallowed transaction positioned, the gap is retired and a + // watch from before the outage resumes instead of failing across it. + var gapCount int + require.NoError(conn.QueryRow(ctx, "SELECT count(*) FROM ledger_gap;").Scan(&gapCount)) + require.Zero(gapCount, "the gap must be retired once its transactions are replayed") + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + events := collectChangesUntilXids(t, changes, errchan, revisionXids(t, removedRevision)) + _, addedEvent := requireChangeForSubject(t, events, "added") + require.True(addedEvent.Revision.(postgresRevision).ByteSortable(), + "a replayed transaction must be delivered with a position") + + // The removal is the point of replaying rather than re-asserting. + _, removedEvent := requireChangeForSubject(t, events, "removed") + require.True(removedEvent.Revision.(postgresRevision).ByteSortable()) + + // Replayed positions land inside the gap's interval, so they order below + // everything the recreated slot went on to record. + lastRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:heal#viewer@user:last"))}) + }) + require.NoError(err) + lastXid, ok := lastRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + var lastLSN pglogrepl.LSN + require.EventuallyWithT(func(collect *assert.CollectT) { + text, found := recordedCommitLSNs(t, ctx, conn)[lastXid.Uint64] + if !assert.True(collect, found, "the ledger did not record the write following the replay") { + return + } + parsed, err := pglogrepl.ParseLSN(text) + assert.NoError(collect, err) + lastLSN = parsed + }, cursorWatchTestTimeout, 50*time.Millisecond) + + require.Less( + removedEvent.Revision.(postgresRevision).optionalCommitLSN, uint64(lastLSN), + "a replayed position must sort below what the recreated slot records", + ) +} + +// testLedgerGapRecordedOnOperatorDrop asserts that a replication slot dropped +// out from under the ledger is recorded as a gap and that recording resumes +// afterwards. +// +// The gap has to be recorded here because after the slot is recreated its +// confirmed position starts past the WAL that was skipped, so the frontier jumps +// over the affected transactions and no later observation can tell they are +// missing. An invalidated slot (wal_status = 'lost') takes the same path. +func testLedgerGapRecordedOnOperatorDrop(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds, dbURI := newCursorWatchTestDatastore(t, b, + LogicalWatchLedgerRetryInterval(100*time.Millisecond), + ) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + pgds, ok := ds.(*pgDatastore) + require.True(ok) + slotName := pgds.ledgerSlotName + + // A write that is recorded before the slot is dropped, so the gap's start + // has a sound lower bound to be derived from. + beforeRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:drop#viewer@user:before"))}) + }) + require.NoError(err) + + beforeXid, ok := beforeRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + require.EventuallyWithT(func(collect *assert.CollectT) { + assert.Contains(collect, recordedCommitLSNs(t, ctx, conn), beforeXid.Uint64) + }, cursorWatchTestTimeout, 20*time.Millisecond) + + requireLedgerAttached(t, ctx, conn, slotName, true) + + // The operator drops the slot. Its confirmed position dies with it, so the + // ledger has to bound the gap by what it is known to have recorded. + _, err = conn.Exec(ctx, "SELECT pg_terminate_backend(active_pid) FROM pg_replication_slots WHERE slot_name = $1 AND active_pid IS NOT NULL;", slotName) + require.NoError(err) + require.EventuallyWithT(func(collect *assert.CollectT) { + _, err := conn.Exec(ctx, "SELECT pg_drop_replication_slot($1);", slotName) + assert.NoError(collect, err) + }, cursorWatchTestTimeout, 50*time.Millisecond) + + // The ledger re-provisions the slot, records the gap, and resumes. + var gapFrom, gapTo string + require.EventuallyWithT(func(collect *assert.CollectT) { + err := conn.QueryRow(ctx, "SELECT from_lsn::text, to_lsn::text FROM ledger_gap ORDER BY from_lsn LIMIT 1;").Scan(&gapFrom, &gapTo) + assert.NoError(collect, err, "no gap was recorded for the dropped slot") + }, cursorWatchTestTimeout, 50*time.Millisecond) + + fromLSN, err := pglogrepl.ParseLSN(gapFrom) + require.NoError(err) + toLSN, err := pglogrepl.ParseLSN(gapTo) + require.NoError(err) + require.Less(fromLSN, toLSN, "a recorded gap must be bounded once the slot is back") + require.NotEqual(ledgerGapPendingToLSN, gapTo, "the pending sentinel must be replaced once the recreation completes") + + // Recording resumes for what commits afterwards. + afterRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:drop#viewer@user:after"))}) + }) + require.NoError(err) + + afterXid, ok := afterRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + require.EventuallyWithT(func(collect *assert.CollectT) { + assert.Contains(collect, recordedCommitLSNs(t, ctx, conn), afterXid.Uint64, + "recording did not resume after the slot was re-provisioned") + }, cursorWatchTestTimeout, 20*time.Millisecond) + + // Nothing committed while the slot was gone, so the recorded interval + // swallowed no transactions and is replayed away as an empty window. A + // consumer positioned below it then resumes: there is nothing it missed. + // The loud failure is asserted where a gap cannot be replayed, in + // testGapBelowCursorFailsLoudly. + require.EventuallyWithT(func(collect *assert.CollectT) { + var gapCount int + assert.NoError(collect, conn.QueryRow(ctx, "SELECT count(*) FROM ledger_gap;").Scan(&gapCount)) + assert.Zero(collect, gapCount, "an empty gap must be replayed away once the ledger passes it") + }, cursorWatchTestTimeout, 100*time.Millisecond) + + staleCursor := postgresRevision{ + snapshot: beforeRevision.(postgresRevision).snapshot, + optionalCommitLSN: uint64(fromLSN), + } + changes, errchan := ds.Watch(ctx, staleCursor, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + events := collectChangesUntilXids(t, changes, errchan, revisionXids(t, afterRevision)) + _, _ = requireChangeForSubject(t, events, "after") +} + +// testCursorWatchLedgerWithoutWriterFails asserts that a watch whose frontier +// has stopped moving because nothing is recording fails with the state an +// operator acts on, rather than stalling indistinguishably from "no writes". +func testCursorWatchLedgerWithoutWriterFails(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // A long retry interval keeps the ledger from coming back, and a short wait + // timeout bounds how long the watch tolerates that. + ds, dbURI := newCursorWatchTestDatastore(t, b, + LogicalWatchLedgerRetryInterval(time.Minute), + LogicalWatchLedgerWaitTimeout(time.Second), + ) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + watchOptions := datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + } + + // A positioned token, so the failing watch below is exercising the loop's + // liveness guard rather than the starting probe. + seedCtx, cancelSeed := context.WithCancel(ctx) + seedChanges, seedErrs := ds.Watch(seedCtx, headRevision.Revision, watchOptions) + require.Empty(seedErrs) + + seedRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:nowriter#viewer@user:seed"))}) + }) + require.NoError(err) + + seedEvents := collectChangesUntilXids(t, seedChanges, seedErrs, revisionXids(t, seedRevision)) + _, seedEvent := requireChangeForSubject(t, seedEvents, "seed") + positionedToken := seedEvent.Revision.(postgresRevision) + cancelSeed() + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + pgds, ok := ds.(*pgDatastore) + require.True(ok) + + _, err = conn.Exec(ctx, "SELECT pg_terminate_backend(active_pid) FROM pg_replication_slots WHERE slot_name = $1 AND active_pid IS NOT NULL;", pgds.ledgerSlotName) + require.NoError(err) + requireLedgerAttached(t, ctx, conn, pgds.ledgerSlotName, false) + + changes, errchan := ds.Watch(ctx, positionedToken, watchOptions) + + // Checkpoints up to the frontier the ledger confirmed before it died are + // legitimate: delivery really is complete through them. What must not happen + // is the watch sitting there indefinitely, or inventing a change. + timeout := time.After(cursorWatchTestTimeout) + for { + select { + case err := <-errchan: + require.ErrorContains(err, "no writer") + require.ErrorContains(err, pgds.ledgerSlotName) + return + case change := <-changes: + require.True(change.IsCheckpoint, "the watch delivered a change with nothing recording: %v", change) + case <-timeout: + require.Fail("the watch neither failed nor delivered while the ledger had no writer") + } + } +} + +// requireLedgerAttached waits until the ledger slot is (or is no longer) +// attached to a consumer. +func requireLedgerAttached(t *testing.T, ctx context.Context, conn *pgx.Conn, slotName string, attached bool) { + t.Helper() + + require.EventuallyWithT(t, func(collect *assert.CollectT) { + var active bool + if !assert.NoError(collect, conn.QueryRow(ctx, "SELECT active FROM pg_replication_slots WHERE slot_name = $1;", slotName).Scan(&active)) { + return + } + assert.Equal(collect, attached, active) + }, cursorWatchTestTimeout, 20*time.Millisecond) +} + +// testLedgerFrontierWaitTimeout asserts that a watch whose positions cannot be +// trusted fails, loudly and quickly, rather than delivering transactions at +// unknown positions or hanging forever. Dropping the ledger's publication is one +// way to stall recording: on PostgreSQL 18 the walsender warns and keeps +// streaming, so the ledger stays attached but never decodes anything again. +func testLedgerFrontierWaitTimeout(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds, dbURI := newCursorWatchTestDatastore(t, b, LogicalWatchLedgerWaitTimeout(time.Second)) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + _, err = conn.Exec(ctx, "DROP PUBLICATION spicedb_ledger;") + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + + select { + case err := <-errchan: + require.ErrorContains(err, "did not record the watch marker transaction") + // The message has to name the slot and its state, since that is what an + // operator acts on. + require.ErrorContains(err, "spicedb_ledger") + case change := <-changes: + require.Fail("the watch delivered an event despite an unusable ledger", "%v", change) + case <-time.After(cursorWatchTestTimeout): + require.Fail("the watch neither failed nor delivered while the ledger was stalled") + } +} + +// testLedgerRecordsWhatTheStreamReports asserts that the position the ledger +// records for a transaction is the position its commit record actually occupies +// in the WAL, as reported by the transaction's own commit LSN on the stream. +// Every token the watch mints is that value, so a discrepancy here would be a +// discrepancy in every token. +func testLedgerRecordsWhatTheStreamReports(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds, dbURI := newCursorWatchTestDatastore(t, b) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + const writeCount = 5 + writtenXids := make([]uint64, 0, writeCount) + for i := 0; i < writeCount; i++ { + rel := tuple.MustParse(fmt.Sprintf("document:recorded#viewer@user:write_%d", i)) + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + writtenXids = append(writtenXids, txid.Uint64) + } + + deliveredTokens := collectTokensByXid(t, changes, errchan, setOfXids(writtenXids...)) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + recorded := recordedCommitLSNs(t, ctx, conn) + for _, xid := range writtenXids { + require.Contains(recorded, xid) + + deliveredRevision, err := ParseRevisionString(deliveredTokens[xid]) + require.NoError(err) + + deliveredLSN := pglogrepl.LSN(deliveredRevision.(postgresRevision).optionalCommitLSN) + require.Equal(recorded[xid], deliveredLSN.String(), + "transaction %d was delivered at a position other than the one recorded for it", xid) + } +} + +// testLedgerTakeover asserts that recording survives the loss of the instance +// doing it. The replication slot admits one consumer, so a survivor takes over as +// soon as the holder's session is gone, and writes that landed in between are +// still recorded because the slot retained their WAL. +func testLedgerTakeover(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // A short retry interval bounds how long the standby waits to notice. + ds, dbURI := newCursorWatchTestDatastore(t, b, LogicalWatchLedgerRetryInterval(100*time.Millisecond)) + + // A second instance against the same database stands by: it cannot attach to + // the slot while the first holds it. + standby, err := newPostgresDatastore( + t.Context(), dbURI, primaryInstanceID, + RevisionQuantization(0), + GCWindow(1000*time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(512), + WithRevisionHeartbeat(false), + WithLogicalWatch(true), + LogicalWatchLedgerRetryInterval(100*time.Millisecond), + ) + require.NoError(err) + t.Cleanup(func() { _ = standby.Close() }) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + pgds, ok := ds.(*pgDatastore) + require.True(ok) + slotName := pgds.ledgerSlotName + + // Exactly one instance is recording, even though both are running it. + requireLedgerAttached(t, ctx, conn, slotName, true) + + // Evict the holder, then write. The write's WAL is retained by the slot, so + // whichever instance attaches next must still record its position. + _, err = conn.Exec(ctx, "SELECT pg_terminate_backend(active_pid) FROM pg_replication_slots WHERE slot_name = $1 AND active_pid IS NOT NULL;", slotName) + require.NoError(err) + + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:takeover#viewer@user:after_eviction"))}) + }) + require.NoError(err) + + writtenXid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + require.EventuallyWithT(func(collect *assert.CollectT) { + assert.Contains(collect, recordedCommitLSNs(t, ctx, conn), writtenXid.Uint64, + "recording did not resume after the holding instance was evicted") + }, cursorWatchTestTimeout, 20*time.Millisecond) + + // And a watch on the survivor is served normally, which is what a live + // frontier means in practice. + standbyHead, err := standby.HeadRevision(ctx) + require.NoError(err) + + watchCtx, cancelWatch := context.WithCancel(ctx) + defer cancelWatch() + changes, errchan := standby.Watch(watchCtx, standbyHead.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + probeRevision, err := standby.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:takeover#viewer@user:probe"))}) + }) + require.NoError(err) + + probeEvents := collectChangesUntilXids(t, changes, errchan, revisionXids(t, probeRevision)) + _, _ = requireChangeForSubject(t, probeEvents, "probe") +} + +// testLedgerPreLedgerHistoryIsDeliveredPositioned covers enabling the watch on a +// database that already holds history. Those transactions committed before the +// ledger existed, so they have no recoverable commit LSN, but inside the +// collection window their commit timestamps still order them: the backfill gives +// them reconstructed positions below the ledger's genesis. +// +// A consumer resuming from a polling-watch token therefore receives one +// comparable stream across the upgrade, in commit order, rather than an +// unpositioned prefix it must not compare. +func testLedgerPreLedgerHistoryIsDeliveredPositioned(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // The database starts out serving the polling watch, so nothing records + // commit positions. + var dbURI string + pollingDS := b.NewDatastore(t, func(engine, uri string) datastore.Datastore { + dbURI = uri + ds, err := newPostgresDatastore( + t.Context(), uri, primaryInstanceID, + RevisionQuantization(0), + GCWindow(1000*time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(512), + WithRevisionHeartbeat(false), + ) + require.NoError(err) + return ds + }) + + headRevision, err := pollingDS.HeadRevision(ctx) + require.NoError(err) + legacyToken := headRevision.Revision.(postgresRevision) + require.False(legacyToken.ByteSortable(), "a revision from the polling watch carries no position") + + const historyCount = 3 + historicalXids := make([]uint64, 0, historyCount) + for i := 0; i < historyCount; i++ { + rel := tuple.MustParse(fmt.Sprintf("document:upgrade#viewer@user:history_%d", i)) + revision, err := pollingDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + historicalXids = append(historicalXids, txid.Uint64) + } + require.NoError(pollingDS.Close()) + + // The upgrade: the cursor watch is enabled, which provisions the ledger and + // records the genesis snapshot that separates the history above from + // everything the ledger will see. + upgradedDS, err := newPostgresDatastore( + t.Context(), dbURI, primaryInstanceID, + RevisionQuantization(0), + GCWindow(1000*time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(512), + WithRevisionHeartbeat(false), + WithLogicalWatch(true), + ) + require.NoError(err) + t.Cleanup(func() { _ = upgradedDS.Close() }) + + // Writes drive the flushes the backfill rides on; once it reports itself + // finished, the reachable history carries positions. + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + require.EventuallyWithT(func(collect *assert.CollectT) { + _, err := upgradedDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:upgrade#viewer@user:driver"))}) + }) + assert.NoError(collect, err) + + var complete bool + assert.NoError(collect, conn.QueryRow(ctx, "SELECT backfill_complete FROM ledger_state;").Scan(&complete)) + assert.True(collect, complete, "the pre-ledger backfill did not finish") + }, cursorWatchTestTimeout, 100*time.Millisecond) + + postUpgradeRevision, err := upgradedDS.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:upgrade#viewer@user:after_upgrade"))}) + }) + require.NoError(err) + + postUpgradeXid, ok := postUpgradeRevision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + changes, errchan := upgradedDS.Watch(ctx, legacyToken, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + awaited := setOfXids(append(historicalXids, postUpgradeXid.Uint64)...) + events := collectChangesUntilXids(t, changes, errchan, awaited) + + positionedByXid := make(map[uint64]bool, len(awaited)) + observedOrder := make([]uint64, 0, len(awaited)) + seenPositioned := false + for _, event := range events { + revision := event.Revision.(postgresRevision) + if revision.ByteSortable() { + seenPositioned = true + } else { + require.False(seenPositioned, "an unpositioned event arrived after a positioned one") + } + + if event.IsCheckpoint { + continue + } + txid, ok := revision.OptionalTransactionID() + require.True(ok) + positionedByXid[txid.Uint64] = revision.ByteSortable() + observedOrder = append(observedOrder, txid.Uint64) + } + + for _, xid := range historicalXids { + require.True(positionedByXid[xid], + "transaction %d predates the ledger but is inside the collection window, so the backfill must have positioned it", xid) + } + require.True(positionedByXid[postUpgradeXid.Uint64], + "a transaction committed after the ledger was provisioned must carry a recorded position") + + // The writes that drove the backfill are delivered too, so the boundary is + // checked as a subsequence rather than the whole stream. + expected := make([]uint64, 0, len(historicalXids)+1) + expected = append(expected, historicalXids...) + expected = append(expected, postUpgradeXid.Uint64) + next := 0 + for _, xid := range observedOrder { + if next < len(expected) && xid == expected[next] { + next++ + } + } + require.Equal(len(expected), next, + "delivery must stay in commit order across the boundary; saw %v, wanted %v in order", observedOrder, expected) +} + +// testLedgerUnrecordedTransactionFailsLoudly covers a transaction that postdates +// the ledger's genesis yet has no recorded position, which is what an invalidated +// and recreated slot leaves behind. Its position is unrecoverable, so the watch +// refuses rather than delivering something it cannot order. +func testLedgerUnrecordedTransactionFailsLoudly(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds, dbURI := newCursorWatchTestDatastore(t, b) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:gap#viewer@user:lost"))}) + }) + require.NoError(err) + + lostXid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + // Wait for the position to be recorded, then erase it: the ledger has + // already confirmed past this transaction, so nothing will record it again. + // This is the state a slot recreation leaves for the transactions it skipped. + require.EventuallyWithT(func(collect *assert.CollectT) { + assert.Contains(collect, recordedCommitLSNs(t, ctx, conn), lostXid.Uint64) + }, cursorWatchTestTimeout, 20*time.Millisecond) + + _, err = conn.Exec(ctx, "DELETE FROM ledger_xid_lsn WHERE xid = $1;", lostXid) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + + select { + case err := <-errchan: + require.ErrorContains(err, "no recorded commit LSN") + require.ErrorContains(err, "unrecoverable") + case change := <-changes: + require.Fail("the watch delivered an event across an unrecoverable gap", "%v", change) + case <-time.After(cursorWatchTestTimeout): + require.Fail("the watch neither failed nor delivered across an unrecoverable gap") + } +} + +// testLedgerWritesAreInvisibleToWatchers asserts that the ledger's own writes +// never reach a watcher. They update the transaction table, and the watch keys +// off inserts there, so they carry no transaction row of their own and nothing +// would identify them as a revision. +func testLedgerWritesAreInvisibleToWatchers(t *testing.T, newDatastore newDatastoreFunc) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds := newDatastore(t, 0, 1000*time.Second, 512, true) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchSchema | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:ledgerecho#viewer@user:only_write"))}) + }) + require.NoError(err) + + writtenXid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + + events := collectChangesUntilXids(t, changes, errchan, setOfXids(writtenXid.Uint64)) + for _, event := range events { + if event.IsCheckpoint { + continue + } + txid, ok := event.Revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + require.Equal(writtenXid.Uint64, txid.Uint64, "an unexpected transaction reached the watcher") + } + + // The ledger records this write moments later, well within the window below. + // Nothing it does may surface as a change; checkpoints are the watch's own. + drainDeadline := time.After(2 * time.Second) + for { + select { + case change, ok := <-changes: + require.True(ok, "the watch closed unexpectedly") + require.True(change.IsCheckpoint, "the ledger's write surfaced as a change event: %v", change) + case err := <-errchan: + require.NoError(err) + case <-drainDeadline: + return + } + } +} + +// testLedgerDisabledFeatureDetectsAbandonedSlot covers switching the feature off +// on a database where it ran. The durable slot is deliberately left in place, +// because dropping database objects on a configuration change is not this +// datastore's call, so startup has to be able to point at it: an unattended slot +// retains WAL until the disk fills. +func testLedgerDisabledFeatureDetectsAbandonedSlot(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + enabledDS, dbURI := newCursorWatchTestDatastore(t, b) + + enabledPGDS, ok := enabledDS.(*pgDatastore) + require.True(ok) + slotName := enabledPGDS.ledgerSlotName + require.NoError(enabledDS.Close()) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + // The slot outlives the datastore that created it. + var exists bool + require.NoError(conn.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = $1);", slotName).Scan(&exists)) + require.True(exists, "the ledger slot must survive a datastore shutdown") + + disabledDS, err := newPostgresDatastore( + t.Context(), dbURI, primaryInstanceID, + RevisionQuantization(0), + GCWindow(1000*time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(512), + WithRevisionHeartbeat(false), + WithLogicalWatch(false), + ) + require.NoError(err) + t.Cleanup(func() { _ = disabledDS.Close() }) + + disabledPGDS, ok := disabledDS.(*pgDatastore) + require.True(ok) + + // Startup resolved the slot's name even with the feature off, which is what + // lets it report the abandoned slot, and observes it unattached. + require.Equal(slotName, disabledPGDS.ledgerSlotName) + + state, err := disabledPGDS.readLedgerSlotState(ctx) + require.NoError(err) + require.True(state.exists) + require.False(state.active, "nothing may be consuming the ledger slot with the feature disabled") +} + +// testLedgerStorageShape asserts where commit positions live and what recording +// one costs. +// +// Positions are kept in their own table precisely so that recording one is an +// append rather than an update of the transaction row: the watch needs positions +// indexed, so such an update could never be heap-only, and every write would +// rewrite a ~100-byte row and re-enter all four of that table's indexes. This +// pins both halves — the column is absent, and writes leave the transaction +// table's update counter untouched. +func testLedgerStorageShape(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds, dbURI := newCursorWatchTestDatastore(t, b) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + var gapCount int + require.NoError(conn.QueryRow(ctx, "SELECT count(*) FROM ledger_gap;").Scan(&gapCount)) + require.Zero(gapCount, "a healthy database must have no recorded ledger gaps") + + var hasColumn bool + require.NoError(conn.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'relation_tuple_transaction' AND column_name = 'commit_lsn');`).Scan(&hasColumn)) + require.False(hasColumn, "commit positions must live in their own table, not on the transaction row") + + // The baseline is taken after migration, which does backfill some rows of + // its own; what matters is that serving writes adds nothing to it. + transactionUpdates := func() int64 { + var updates int64 + require.NoError(conn.QueryRow(ctx, `SELECT coalesce(n_tup_upd, 0) + FROM pg_stat_user_tables WHERE relname = 'relation_tuple_transaction';`).Scan(&updates)) + return updates + } + baseline := transactionUpdates() + + writtenXids := make([]uint64, 0, 3) + for i := 0; i < 3; i++ { + rel := tuple.MustParse(fmt.Sprintf("document:shape#viewer@user:subject_%d", i)) + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + writtenXids = append(writtenXids, txid.Uint64) + } + + require.EventuallyWithT(func(collect *assert.CollectT) { + recorded := recordedCommitLSNs(t, ctx, conn) + for _, xid := range writtenXids { + assert.Contains(collect, recorded, xid) + } + }, cursorWatchTestTimeout, 20*time.Millisecond) + + require.Equal(baseline, transactionUpdates(), + "recording commit positions must not update the transaction table") +} + +// testLedgerPositionsAreGarbageCollected asserts that recorded positions are +// collected along with the transactions they describe, and that the watch keeps +// working across a collection pass. +// +// The delete order is what is really under test. Positions are removed *after* +// the transaction rows, because the reverse would briefly leave transactions +// that look unrecorded, which is indistinguishable from a transaction lost to a +// slot recreation and would fail watches for no reason. +func testLedgerPositionsAreGarbageCollected(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // A tiny GC window makes everything written here immediately collectable. + ds, dbURI := newCursorWatchTestDatastore(t, b, GCWindow(time.Millisecond)) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + for i := 0; i < 3; i++ { + rel := tuple.MustParse(fmt.Sprintf("document:collected#viewer@user:subject_%d", i)) + _, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + } + + require.EventuallyWithT(func(collect *assert.CollectT) { + assert.NotEmpty(collect, recordedCommitLSNs(t, ctx, conn), "the ledger recorded nothing to collect") + }, cursorWatchTestTimeout, 20*time.Millisecond) + + pgds, ok := ds.(*pgDatastore) + require.True(ok) + + gc, err := pgds.BuildGarbageCollector(ctx) + require.NoError(err) + defer gc.Close() + + now, err := gc.Now(ctx) + require.NoError(err) + collectBefore, err := gc.TxIDBefore(ctx, now) + require.NoError(err) + _, err = gc.DeleteBeforeTx(ctx, collectBefore) + require.NoError(err) + + // No position may outlive the horizon: whatever transactions were collected, + // their positions went with them. + collectedXid, ok := collectBefore.(postgresRevision).OptionalTransactionID() + require.True(ok) + + var stragglers int + require.NoError(conn.QueryRow(ctx, + "SELECT count(*) FROM ledger_xid_lsn WHERE xid < $1;", collectedXid).Scan(&stragglers)) + require.Zero(stragglers, "collected transactions left their positions behind") + + // And a watch started afterwards still works: collection below a consumer's + // position is not a gap, and must not be reported as one. + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + afterRevision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(tuple.MustParse("document:collected#viewer@user:after_gc"))}) + }) + require.NoError(err) + + events := collectChangesUntilXids(t, changes, errchan, revisionXids(t, afterRevision)) + _, _ = requireChangeForSubject(t, events, "after_gc") +} + +// testLedgerOrphanPositionsAreIgnored asserts that a position whose transaction +// row is gone is inert. The two are deleted by separate statements, so the state +// is reachable whenever collection is interrupted, and it must not surface as an +// event, an error, or a cursor that skips ahead. +func testLedgerOrphanPositionsAreIgnored(t *testing.T, b testdatastore.RunningEngineForTest) { + require := require.New(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ds, dbURI := newCursorWatchTestDatastore(t, b) + + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + // An orphan: a position for a transaction that has no row. Its xid is far + // above anything real, so it also sits above the watch's cursor, where a + // naive implementation would trip over it. + var currentLSN string + require.NoError(conn.QueryRow(ctx, "SELECT pg_current_wal_lsn()::text;").Scan(¤tLSN)) + _, err = conn.Exec(ctx, + "INSERT INTO ledger_xid_lsn (xid, commit_lsn) VALUES ($1::xid8, $2::pg_lsn);", + NewXid8(1<<40), currentLSN) + require.NoError(err) + + rel := tuple.MustParse("document:orphan#viewer@user:real") + revision, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(rel)}) + }) + require.NoError(err) + + events := collectChangesUntilXids(t, changes, errchan, revisionXids(t, revision)) + for _, event := range events { + if event.IsCheckpoint { + continue + } + txid, ok := event.Revision.(postgresRevision).OptionalTransactionID() + require.True(ok) + require.NotEqual(uint64(1<<40), txid.Uint64, "an orphan position was delivered as a change") + } + _, _ = requireChangeForSubject(t, events, "real") +} + +// TestPostgresCursorWatchRequiresLogicalWALLevel asserts that the startup +// preflight refuses to bring up the cursor watch on a server that cannot support +// the commit LSN ledger. +func TestPostgresCursorWatchRequiresLogicalWALLevel(t *testing.T) { + // The default test server runs with wal_level=replica, not logical. + b := testdatastore.RunPostgresForTesting(t, postgresTestVersion(), false) + + var constructErr error + _ = b.NewDatastore(t, func(engine, uri string) datastore.Datastore { + _, constructErr = newPostgresDatastore( + t.Context(), uri, primaryInstanceID, + RevisionQuantization(0), + GCWindow(1000*time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(16), + WithRevisionHeartbeat(false), + WithLogicalWatch(true), + ) + return nil + }) + + require.Error(t, constructErr, "constructing a cursor-watch datastore on a non-logical server must fail") + require.ErrorContains(t, constructErr, "wal_level") +} + +// TestPostgresCursorWatchRequiresCommitTimestamps asserts that the startup +// preflight refuses a server without commit timestamps, rather than provisioning +// a WAL-retaining slot for a watch that could not replay a gap. +func TestPostgresCursorWatchRequiresCommitTimestamps(t *testing.T) { + b := testdatastore.RunPostgresForTestingWithLogicalReplication( + t, postgresTestVersion(), + testcontainers.WithCmdArgs("-c", "track_commit_timestamp=off"), + ) + + var constructErr error + _ = b.NewDatastore(t, func(engine, uri string) datastore.Datastore { + _, constructErr = newPostgresDatastore( + t.Context(), uri, primaryInstanceID, + RevisionQuantization(0), + GCWindow(1000*time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(16), + WithRevisionHeartbeat(false), + WithLogicalWatch(true), + ) + return nil + }) + + require.Error(t, constructErr, "constructing a cursor-watch datastore without commit timestamps must fail") + require.ErrorContains(t, constructErr, "track_commit_timestamp") +} + +// TestPostgresCursorWatchNonDefaultServerConfig runs the cursor watch against a +// server with a non-UTC timezone, validating that the ledger's decoding and the +// watch's timestamps are pinned by their own session settings rather than the +// server defaults. +func TestPostgresCursorWatchNonDefaultServerConfig(t *testing.T) { + require := require.New(t) + + b := testdatastore.RunPostgresForTestingWithLogicalReplication( + t, postgresTestVersion(), + // Command-line settings take precedence over the mounted config file. + testcontainers.WithCmdArgs("-c", "track_commit_timestamp=on", "-c", "TimeZone=America/New_York"), + ) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + var dbURI string + ds := b.NewDatastore(t, func(engine, uri string) datastore.Datastore { + dbURI = uri + ds, err := newPostgresDatastore( + t.Context(), uri, primaryInstanceID, + RevisionQuantization(0), + GCWindow(1000*time.Second), + GCInterval(veryLargeGCInterval), + WatchBufferLength(512), + WithRevisionHeartbeat(false), + WithLogicalWatch(true), + ) + require.NoError(err) + return ds + }) + + // The server renders timestamps in a timezone neither the ledger nor the + // watch may inherit. + conn, err := pgx.Connect(ctx, dbURI) + require.NoError(err) + defer func() { _ = conn.Close(ctx) }() + + var setting string + require.NoError(conn.QueryRow(ctx, "SHOW TimeZone;").Scan(&setting)) + require.Equal("America/New_York", setting) + + headRevision, err := ds.HeadRevision(ctx) + require.NoError(err) + + expiration := time.Now().Add(time.Hour).UTC().Truncate(time.Microsecond) + + // Committed before the watch: delivered by the backfill phase. + backfillRel := tuple.MustParse(`document:tzdoc#viewer@user:backfill[somecaveat:{"origin":"backfill"}]`) + backfillRel.OptionalExpiration = &expiration + revBackfill, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(backfillRel)}) + }) + require.NoError(err) + + changes, errchan := ds.Watch(ctx, headRevision.Revision, datastore.WatchOptions{ + Content: datastore.WatchRelationships | datastore.WatchCheckpoints, + CheckpointInterval: 100 * time.Millisecond, + }) + require.Empty(errchan) + + backfillEvents := collectChangesUntilXids(t, changes, errchan, revisionXids(t, revBackfill)) + _, backfillEvent := requireChangeForSubject(t, backfillEvents, "backfill") + backfillRevision := backfillEvent.Revision.(postgresRevision) + require.True(backfillRevision.ByteSortable(), "the backfill must carry a recorded commit position") + + // And one committed while the watch runs, whose position the ledger decodes + // out of the WAL under the non-UTC server default. + laterRel := tuple.MustParse(`document:tzdoc#viewer@user:later[somecaveat:{"origin":"later"}]`) + laterRel.OptionalExpiration = &expiration + revLater, err := ds.ReadWriteTx(ctx, func(ctx context.Context, rwt datastore.ReadWriteTransaction) error { + return rwt.WriteRelationships(ctx, []tuple.RelationshipUpdate{tuple.Touch(laterRel)}) + }) + require.NoError(err) + + laterEvents := collectChangesUntilXids(t, changes, errchan, revisionXids(t, revLater)) + _, laterEvent := requireChangeForSubject(t, laterEvents, "later") + laterRevision := laterEvent.Revision.(postgresRevision) + require.True(laterRevision.ByteSortable()) + require.Greater(laterRevision.optionalCommitLSN, backfillRevision.optionalCommitLSN) + + // Both must decode the timestamptz expiration to the exact instant written, + // regardless of the server timezone, along with the caveat context. + for _, entry := range []struct { + name string + event datastore.RevisionChanges + want string + }{ + {"backfill", backfillEvent, "backfill"}, + {"later", laterEvent, "later"}, + } { + require.Len(entry.event.RelationshipChanges, 1, "%s event must carry exactly one relationship change", entry.name) + relationship := entry.event.RelationshipChanges[0].Relationship + require.NotNil(relationship.OptionalExpiration, "%s delivery lost the expiration", entry.name) + require.True(relationship.OptionalExpiration.Equal(expiration), + "%s delivery decoded expiration %s, want %s", entry.name, relationship.OptionalExpiration, expiration) + require.NotNil(relationship.OptionalCaveat, "%s delivery lost the caveat", entry.name) + require.Equal("somecaveat", relationship.OptionalCaveat.CaveatName) + require.Equal(entry.want, relationship.OptionalCaveat.Context.AsMap()["origin"]) + } +} + +// collectChangesUntilXids reads from the watch channel until every awaited +// transaction ID has been observed on a non-checkpoint change, returning all +// events (checkpoints included) seen along the way. +func collectChangesUntilXids(t *testing.T, changes <-chan datastore.RevisionChanges, errchan <-chan error, awaited map[uint64]struct{}) []datastore.RevisionChanges { + t.Helper() + + remaining := make(map[uint64]struct{}, len(awaited)) + for xid := range awaited { + remaining[xid] = struct{}{} + } + + var collected []datastore.RevisionChanges + timeout := time.After(cursorWatchTestTimeout) + for len(remaining) > 0 { + select { + case change, ok := <-changes: + require.True(t, ok, "watch channel closed while awaiting transactions") + collected = append(collected, change) + if !change.IsCheckpoint { + revision, ok := change.Revision.(postgresRevision) + require.True(t, ok) + if txid, ok := revision.OptionalTransactionID(); ok { + delete(remaining, txid.Uint64) + } + } + case err := <-errchan: + require.NoError(t, err, "unexpected watch error") + case <-timeout: + require.Fail(t, "timed out waiting for watched transactions", "%d transactions still unobserved", len(remaining)) + } + } + + return collected +} + +// collectChangesUntilRevision reads from the watch channel until a change or +// checkpoint at or beyond the given revision is observed, returning all +// non-checkpoint changes seen. +func collectChangesUntilRevision(t *testing.T, changes <-chan datastore.RevisionChanges, errchan <-chan error, untilRevision datastore.Revision) []datastore.RevisionChanges { + t.Helper() + + var collected []datastore.RevisionChanges + timeout := time.After(cursorWatchTestTimeout) + for { + select { + case change, ok := <-changes: + require.True(t, ok, "watch channel closed while waiting for revision %s", untilRevision) + if !change.IsCheckpoint { + collected = append(collected, change) + } + if change.Revision != nil && (change.Revision.Equal(untilRevision) || change.Revision.GreaterThan(untilRevision)) { + return collected + } + case err := <-errchan: + require.NoError(t, err, "unexpected watch error") + case <-timeout: + require.Fail(t, "timed out waiting for watch to reach revision", "revision: %s", untilRevision) + } + } +} + +// collectTokensByXid reads until every awaited transaction has been delivered, +// returning the string form of the revision each was delivered at. A transaction +// delivered more than once within one stream must carry the same token every +// time. +func collectTokensByXid(t *testing.T, changes <-chan datastore.RevisionChanges, errchan <-chan error, awaited map[uint64]struct{}) map[uint64]string { + t.Helper() + + tokens := make(map[uint64]string, len(awaited)) + for _, event := range collectChangesUntilXids(t, changes, errchan, awaited) { + if event.IsCheckpoint { + continue + } + + revision, ok := event.Revision.(postgresRevision) + require.True(t, ok) + txid, ok := revision.OptionalTransactionID() + require.True(t, ok, "change revision missing transaction ID") + if _, isAwaited := awaited[txid.Uint64]; !isAwaited { + continue + } + + token := revision.String() + if existing, seen := tokens[txid.Uint64]; seen { + require.Equal(t, existing, token, "transaction %d delivered at two different tokens in one stream", txid.Uint64) + continue + } + tokens[txid.Uint64] = token + } + + for xid := range awaited { + require.Contains(t, tokens, xid, "transaction %d was never delivered", xid) + } + + return tokens +} + +// setOfXids collects transaction IDs into the set form the watch test helpers await. +func setOfXids(xids ...uint64) map[uint64]struct{} { + set := make(map[uint64]struct{}, len(xids)) + for _, xid := range xids { + set[xid] = struct{}{} + } + return set +} + +// positionPrefixOf returns the fixed-width position prefix of a revision token, +// which is the portion of the string form that carries commit order. +func positionPrefixOf(t *testing.T, token string) string { + t.Helper() + + prefix, _, found := strings.Cut(token, string(lsnRevisionSeparator)) + require.True(t, found, "token %q carries no position prefix", token) + return prefix +} + +// revisionXids returns the set of transaction IDs of the given write revisions. +func revisionXids(t *testing.T, revisions ...datastore.Revision) map[uint64]struct{} { + t.Helper() + + xids := make(map[uint64]struct{}, len(revisions)) + for _, revision := range revisions { + txid, ok := revision.(postgresRevision).OptionalTransactionID() + require.True(t, ok, "write revision missing its transaction ID") + xids[txid.Uint64] = struct{}{} + } + return xids +} + +// requireChangeForSubject returns the position and the change event carrying a +// relationship change for the given subject object ID. +func requireChangeForSubject(t *testing.T, events []datastore.RevisionChanges, subjectObjectID string) (int, datastore.RevisionChanges) { + t.Helper() + + for index, change := range events { + if change.IsCheckpoint { + continue + } + for _, relChange := range change.RelationshipChanges { + if relChange.Relationship.Subject.ObjectID == subjectObjectID { + return index, change + } + } + } + + require.Failf(t, "change not found", "no change observed for subject %s", subjectObjectID) + return 0, datastore.RevisionChanges{} +} + +// readSubjectIDs reads all relationships of the given resource type at the given +// revision and returns their subject object IDs. +func readSubjectIDs(t *testing.T, ctx context.Context, ds datastore.Datastore, revision datastore.Revision, resourceType string) []string { + t.Helper() + + iterator, err := ds.SnapshotReader(revision).QueryRelationships(ctx, datastore.RelationshipsFilter{ + OptionalResourceType: resourceType, + }) + require.NoError(t, err) + + subjectIDs := make([]string, 0, 8) + for rel, err := range iterator { + require.NoError(t, err) + subjectIDs = append(subjectIDs, rel.Subject.ObjectID) + } + return subjectIDs +} + +// normalizedTransactionChanges is the order-insensitive, per-transaction view of +// watch output used to compare the two implementations. +type normalizedTransactionChanges struct { + RelationshipChanges []string + ChangedDefinitions []string + DeletedNamespaces []string + DeletedCaveats []string + Metadata []string +} + +func normalizeChangesByTransaction(t *testing.T, changes []datastore.RevisionChanges) map[uint64]normalizedTransactionChanges { + t.Helper() + + normalized := make(map[uint64]normalizedTransactionChanges) + for _, change := range changes { + revision, ok := change.Revision.(postgresRevision) + require.True(t, ok) + txid, ok := revision.OptionalTransactionID() + require.True(t, ok, "change revision missing transaction ID") + + entry := normalized[txid.Uint64] + for _, relChange := range change.RelationshipChanges { + entry.RelationshipChanges = append(entry.RelationshipChanges, relChange.DebugString()) + } + for _, def := range change.ChangedDefinitions { + entry.ChangedDefinitions = append(entry.ChangedDefinitions, fmt.Sprintf("%T:%s", def, def.GetName())) + } + entry.DeletedNamespaces = append(entry.DeletedNamespaces, change.DeletedNamespaces...) + entry.DeletedCaveats = append(entry.DeletedCaveats, change.DeletedCaveats...) + for _, metadata := range change.Metadatas { + entry.Metadata = append(entry.Metadata, fmt.Sprintf("%v", metadata.AsMap())) + } + + sort.Strings(entry.RelationshipChanges) + sort.Strings(entry.ChangedDefinitions) + sort.Strings(entry.DeletedNamespaces) + sort.Strings(entry.DeletedCaveats) + sort.Strings(entry.Metadata) + normalized[txid.Uint64] = entry + } + + return normalized +} diff --git a/internal/testserver/datastore/config/postgres-logical-replication.conf b/internal/testserver/datastore/config/postgres-logical-replication.conf new file mode 100644 index 0000000000..fbc4948b19 --- /dev/null +++ b/internal/testserver/datastore/config/postgres-logical-replication.conf @@ -0,0 +1,10 @@ +listen_addresses = '*' +max_connections = 3000 +track_commit_timestamp = 1 +wal_level = logical +# The commit LSN ledger's slot is durable, and every test database creates its +# own, so slots accumulate for the lifetime of the container rather than being +# released when a datastore closes. The limit therefore has to cover every +# database a test binary creates, not just the ones live at any moment. +max_replication_slots = 400 +max_wal_senders = 100 diff --git a/internal/testserver/datastore/postgres.go b/internal/testserver/datastore/postgres.go index de8150f920..f80e6ac948 100644 --- a/internal/testserver/datastore/postgres.go +++ b/internal/testserver/datastore/postgres.go @@ -57,6 +57,18 @@ func RunPostgresForTestingWithCommitTimestamps(t testing.TB, withCommitTimestamp return builder } +// RunPostgresForTestingWithLogicalReplication returns a RunningEngineForTest +// for postgres configured with wal_level=logical (plus commit timestamps), as +// required by the logical-replication watch. pgbouncer is not supported: +// replication connections cannot be proxied through it. +func RunPostgresForTestingWithLogicalReplication(t testing.TB, pgVersion string, opts ...testcontainers.ContainerCustomizer) RunningEngineForTest { + t.Helper() + + builder := &postgresTester{} + builder.runPostgresForTestingWithConfig(t, pgVersion, postgresLogicalReplicationConf, opts...) + return builder +} + func (b *postgresTester) NewDatabase(t testing.TB) string { t.Helper() uri, err := b.newDatabase(t.Context()) @@ -168,7 +180,8 @@ func (b *postgresTester) runPgbouncerForTesting(t testing.TB, pgVersion string, } postgresOptions := make([]testcontainers.ContainerCustomizer, 0, len(opts)+4) - postgresOptions = append(postgresOptions, + postgresOptions = append( + postgresOptions, testcontainers.WithEnv(map[string]string{ // use md5 auth to align postgres and pgbouncer auth methods "POSTGRES_HOST_AUTH_METHOD": "md5", @@ -184,7 +197,8 @@ func (b *postgresTester) runPgbouncerForTesting(t testing.TB, pgVersion string, postgresOptions = append(postgresOptions, opts...) image := "mirror.gcr.io/library/postgres:" + pgVersion - pgContainer, err := postgres.Run(ctx, image, + pgContainer, err := postgres.Run( + ctx, image, postgresOptions..., ) require.NoError(t, err) @@ -194,7 +208,8 @@ func (b *postgresTester) runPgbouncerForTesting(t testing.TB, pgVersion string, // set up the bouncer container // NOTE: this is a bit of a bodge; a pgbouncer container is not the same as a postgres container, // so we have to undo some of what the postgres container wrapper is doing. - bouncerContainer, err := postgres.Run(ctx, "mirror.gcr.io/edoburu/pgbouncer:latest", + bouncerContainer, err := postgres.Run( + ctx, "mirror.gcr.io/edoburu/pgbouncer:latest", network.WithNetwork([]string{"pgbouncer"}, testNetwork), testcontainers.WithLogger(log.TestLogger(t)), postgres.WithUsername(PgTestUser), @@ -238,6 +253,9 @@ var postgresConf []byte //go:embed config/postgres-with-timestamps.conf var postgresWithTimestampsConf []byte +//go:embed config/postgres-logical-replication.conf +var postgresLogicalReplicationConf []byte + // postgresConfOption is basically postgres.WithConfigFile but using the `Reader` // interface on ContainerFile instead of `HostFilePath`, which is difficult to use // when this code is invoked from different places. @@ -258,15 +276,22 @@ func postgresConfOption(confBytes []byte) testcontainers.CustomizeRequestOption func (b *postgresTester) runPostgresForTesting(t testing.TB, pgVersion string, withCommitTimestamps bool, opts ...testcontainers.ContainerCustomizer) { t.Helper() - ctx := t.Context() - logger := log.TestLogger(t) configBytes := postgresConf if withCommitTimestamps { configBytes = postgresWithTimestampsConf } + b.runPostgresForTestingWithConfig(t, pgVersion, configBytes, opts...) +} + +func (b *postgresTester) runPostgresForTestingWithConfig(t testing.TB, pgVersion string, configBytes []byte, opts ...testcontainers.ContainerCustomizer) { + t.Helper() + ctx := t.Context() + logger := log.TestLogger(t) + options := make([]testcontainers.ContainerCustomizer, 0, len(opts)+5) - options = append(options, + options = append( + options, testcontainers.WithLogger(logger), // contains the config for commit timestamps and max conns postgresConfOption(configBytes), @@ -277,7 +302,8 @@ func (b *postgresTester) runPostgresForTesting(t testing.TB, pgVersion string, w options = append(options, opts...) image := "mirror.gcr.io/library/postgres:" + pgVersion - container, err := postgres.Run(ctx, image, + container, err := postgres.Run( + ctx, image, options..., ) testcontainers.CleanupContainer(t, container) diff --git a/pkg/cmd/datastore/datastore.go b/pkg/cmd/datastore/datastore.go index 90631f372d..465d267de1 100644 --- a/pkg/cmd/datastore/datastore.go +++ b/pkg/cmd/datastore/datastore.go @@ -158,9 +158,10 @@ type Config struct { WriteAcquisitionTimeout time.Duration `debugmap:"visible" default:"30ms"` // Postgres - GCInterval time.Duration `debugmap:"visible" default:"3m"` - GCMaxOperationTime time.Duration `debugmap:"visible" default:"1m"` - RelaxedIsolationLevel bool `debugmap:"visible"` + GCInterval time.Duration `debugmap:"visible" default:"3m"` + GCMaxOperationTime time.Duration `debugmap:"visible" default:"1m"` + RelaxedIsolationLevel bool `debugmap:"visible"` + EnableLogicalReplicationWatch bool `debugmap:"visible"` // Spanner // SpannerCredentialsFile is a filename reference to a file containing @@ -378,6 +379,7 @@ func RegisterDatastoreFlagsWithPrefix(flagSet *pflag.FlagSet, prefix string, opt flagSet.DurationVar(&opts.WatchBufferWriteTimeout, flagName("datastore-watch-buffer-write-timeout"), 1*time.Second, "how long the watch buffer should queue before forcefully disconnecting the reader") flagSet.DurationVar(&opts.WatchConnectTimeout, flagName("datastore-watch-connect-timeout"), 1*time.Second, "how long the watch connection to the underlying datastore should wait before timing out (CockroachDB driver only)") flagSet.BoolVar(&opts.DisableWatchSupport, flagName("datastore-disable-watch-support"), false, "disable watch support (only enable if you absolutely do not need watch)") + flagSet.BoolVar(&opts.EnableLogicalReplicationWatch, flagName("datastore-logical-replication-watch"), false, "serve the Watch API in true commit order, keyed by commit LSNs recorded from a logical replication slot; requires wal_level=logical and the REPLICATION privilege (Postgres driver only)") flagSet.BoolVar(&opts.IncludeQueryParametersInTraces, flagName("datastore-include-query-parameters-in-traces"), false, "include query parameters in traces (Postgres and CockroachDB drivers only)") flagSet.DurationVar(&opts.WriteAcquisitionTimeout, flagName("write-conn-acquisition-timeout"), defaults.WriteAcquisitionTimeout, "amount of time that the server will wait for a connection to the datastore to become available when performing a write operation before throwing a ResourceExhausted error. 0 means wait indefinitely. (CockroachDB driver only)") @@ -437,6 +439,7 @@ func DefaultDatastoreConfig() *Config { WatchBufferWriteTimeout: 1 * time.Second, WatchConnectTimeout: 1 * time.Second, DisableWatchSupport: false, + EnableLogicalReplicationWatch: false, EnableDatastoreMetrics: true, DisableStats: false, BootstrapFiles: []string{}, @@ -763,6 +766,7 @@ func newPostgresPrimaryDatastore(ctx context.Context, opts Config) (datastore.Da postgres.AllowedMigrations(opts.AllowedMigrations), postgres.WithRevisionHeartbeat(opts.EnableRevisionHeartbeat), postgres.WithRelaxedIsolationLevel(opts.RelaxedIsolationLevel), + postgres.WithLogicalWatch(opts.EnableLogicalReplicationWatch), } commonOptions, err := commonPostgresDatastoreOptions(opts) diff --git a/pkg/cmd/datastore/zz_generated.options.go b/pkg/cmd/datastore/zz_generated.options.go index 07db957a75..6f184babb5 100644 --- a/pkg/cmd/datastore/zz_generated.options.go +++ b/pkg/cmd/datastore/zz_generated.options.go @@ -71,6 +71,7 @@ func (c *Config) ToOption() ConfigOption { to.GCInterval = c.GCInterval to.GCMaxOperationTime = c.GCMaxOperationTime to.RelaxedIsolationLevel = c.RelaxedIsolationLevel + to.EnableLogicalReplicationWatch = c.EnableLogicalReplicationWatch to.SpannerCredentialsFile = c.SpannerCredentialsFile to.SpannerCredentialsJSON = c.SpannerCredentialsJSON to.SpannerEmulatorHost = c.SpannerEmulatorHost @@ -255,6 +256,7 @@ func (c *Config) DebugMap() map[string]any { debugMap["GCMaxOperationTime"] = c.GCMaxOperationTime } debugMap["RelaxedIsolationLevel"] = c.RelaxedIsolationLevel + debugMap["EnableLogicalReplicationWatch"] = c.EnableLogicalReplicationWatch if c.SpannerCredentialsFile == "" { debugMap["SpannerCredentialsFile"] = "(empty)" } else { @@ -650,6 +652,13 @@ func WithRelaxedIsolationLevel(relaxedIsolationLevel bool) ConfigOption { } } +// WithEnableLogicalReplicationWatch returns an option that can set EnableLogicalReplicationWatch on a Config +func WithEnableLogicalReplicationWatch(enableLogicalReplicationWatch bool) ConfigOption { + return func(c *Config) { + c.EnableLogicalReplicationWatch = enableLogicalReplicationWatch + } +} + // WithSpannerCredentialsFile returns an option that can set SpannerCredentialsFile on a Config func WithSpannerCredentialsFile(spannerCredentialsFile string) ConfigOption { return func(c *Config) {