diff --git a/.changeset/breezy-sites-mix.md b/.changeset/breezy-sites-mix.md new file mode 100644 index 000000000..800f96a82 --- /dev/null +++ b/.changeset/breezy-sites-mix.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-image': minor +--- + +Added initial support for Convex replication. diff --git a/.changeset/bright-facts-nail.md b/.changeset/bright-facts-nail.md new file mode 100644 index 000000000..a4ef6eda5 --- /dev/null +++ b/.changeset/bright-facts-nail.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-module-convex': patch +--- + +Initial alpha release diff --git a/.changeset/twelve-poets-sort.md b/.changeset/twelve-poets-sort.md new file mode 100644 index 000000000..21e70c3fe --- /dev/null +++ b/.changeset/twelve-poets-sort.md @@ -0,0 +1,5 @@ +--- +'@powersync/lib-services-framework': patch +--- + +Add bigint support for codec validations diff --git a/.github/workflows/development_image_release.yaml b/.github/workflows/development_image_release.yaml index 92cc5ef7e..18013a142 100644 --- a/.github/workflows/development_image_release.yaml +++ b/.github/workflows/development_image_release.yaml @@ -49,7 +49,7 @@ jobs: # If no changesets are available the status check will fail # We should not continue if there are no changesets pnpm changeset status - pnpm changeset version --no-git-tag --snapshot dev + pnpm changeset version --snapshot dev # This uses the service's package.json version for the Docker Image tag # The changeset command above should change this to a dev package diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c4e697f5..14725a73f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -258,6 +258,63 @@ jobs: - name: Test Replication run: pnpm --filter='./modules/module-mongodb' test + run-convex-replication-tests: + name: Convex Replication Test + runs-on: ubuntu-latest + needs: run-core-tests + env: + CONVEX_DEPLOYMENT: anonymous:anonymous-module-convex + CONVEX_URL: http://127.0.0.1:3210 + CONVEX_SITE_URL: http://127.0.0.1:3211 + PG_STORAGE_TEST_URL: postgres://postgres:postgres@localhost:5431/powersync_storage_test + + steps: + - uses: actions/checkout@v5 + + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Start MongoDB + uses: ./.github/actions/start-mongodb + + - name: Start PostgreSQL (Storage) + uses: ./.github/actions/start-postgres-storage + + - name: Setup Node and build + uses: ./.github/actions/setup-node-build + + - name: Start Convex + working-directory: modules/module-convex + run: | + # Start the local Convex backend in the background so the next test step can use it. + pnpm run dev:convex > convex-dev.log 2>&1 & + + # Wait until Convex has generated its local admin key and the schema API is responding. + timeout 120 bash -c ' + until node --input-type=module -e '"'"' + import fs from "node:fs"; + const config = JSON.parse(fs.readFileSync(".convex/local/default/config.json", "utf8")); + const response = await fetch("http://127.0.0.1:3210/api/json_schemas?format=json", { + headers: { Authorization: `Convex ${config.adminKey}` } + }); + if (!response.ok) process.exit(1); + '"'"'; do + sleep 2 + done + ' + + - name: Test Replication + run: pnpm --filter='./modules/module-convex' test + + - name: Print Convex logs + if: failure() + working-directory: modules/module-convex + run: cat convex-dev.log + run-mongodb-storage-tests: name: MongoDB Storage Test runs-on: ubuntu-latest diff --git a/.prettierignore b/.prettierignore index aeeb3baa6..4cbdb8700 100644 --- a/.prettierignore +++ b/.prettierignore @@ -8,4 +8,5 @@ pnpm-lock.yaml **/*.sql # Generated files packages/schema/json-schema -packages/sync-rules/schema \ No newline at end of file +packages/sync-rules/schema +modules/module-convex/convex/_generated \ No newline at end of file diff --git a/README.md b/README.md index 292e74561..50ef39eac 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ The service can be started using the public Docker image. See the image [notes]( - MySQL replication module. +- [modules/module-convex](./modules/module-convex/README.md) + + - Convex replication module. See the module README for local integration testing with the `dev:convex` backend. + - [modules/module-postgres](./modules/module-postgres/README.md) - Postgres replication module. diff --git a/docs/convex/convex-write-checkpoints.md b/docs/convex/convex-write-checkpoints.md new file mode 100644 index 000000000..7bd46c508 --- /dev/null +++ b/docs/convex/convex-write-checkpoints.md @@ -0,0 +1,259 @@ +# Convex write checkpoints + +This note looks at whether the Convex implementation needs to write to the +`powersync_checkpoints` collection from +`createWriteCheckpointMarker(options?: { signal?: AbortSignal }): Promise`. + +TLDR: Yes, we need a `powersync_checkpoints` collection in the source database. + +## Background + +Write checkpoints let a client wait until its own uploaded write has been +replicated back through PowerSync before applying the corresponding synced data. +This avoids flicker and avoids accepting a local write as durable before the +backend-side effects of that write are visible in the sync stream. + +The managed write checkpoint flow is: + +1. The client or backend writes to the source database. +2. The PowerSync API creates a managed write checkpoint for the user/client. +3. The source connector records a source replication position that should include + the write. +4. PowerSync stores a generated write checkpoint id together with that source + position. +5. When replication commits or keeps alive at or past that source position, + sync can expose the write checkpoint id to the client. + +For Convex, the source position is the Convex replication cursor. The current +implementation in `createReplicationHead`: + +1. calls `getHeadCursor()` to read the current global Convex head, +2. invokes the callback with the original head cursor so PowerSync stores the + managed write checkpoint mapping, +3. calls `createWriteCheckpointMarker()` to run the + `powersync_checkpoints:createCheckpoint` Convex mutation. + +The callback stores the managed write checkpoint in bucket storage with the +original head as the replication head. The marker write is intentionally not the +write checkpoint position. It is a later Convex mutation whose job is to advance +the Convex delta stream beyond the stored head after the managed mapping exists. + +The key invariant is that PowerSync must observe a checkpoint update at or past +the stored head after the managed write checkpoint mapping exists. Other source +connectors make this ordering explicit by creating the mapping in the callback +and then triggering the stream. For Convex, the marker is still the mechanism +that creates the later source event, but the implementation should preserve the +same effective ordering: the client must not be left with a stored write +checkpoint id and no later observable checkpoint update. + +## How the marker is used by replication + +Convex `document_deltas` only advances when there is a source-side change. The +stream implementation ignores rows from the `powersync_checkpoints` table as +user data, but still treats marker-only delta pages as meaningful replication +progress: + +- normal user-data delta pages are committed with `batch.commit(pageLsn)`, +- marker-only pages call `batch.keepalive(pageLsn)` immediately, +- the marker table is excluded from source schema discovery and row application. + +That keepalive is enough to advance the stored checkpoint LSN. Once the stored +checkpoint LSN is at or past the managed write checkpoint head, the sync stream +can acknowledge the write checkpoint to the client. + +## Case 1: no checkpoint collection + +If there is no `powersync_checkpoints` collection, `createReplicationHead` can +still read the current Convex head and create a managed write checkpoint in +PowerSync storage. The problem is that nothing guarantees the Convex delta stream +will advance beyond that head. + +### Case 1.1: no replication lag + +If there is no replication lag, the latest Convex head may already include the +client's source write by the time `/write-checkpoint2.json` runs. PowerSync can +therefore associate the generated write checkpoint id with a correct source +head. + +However, correctness of the association is not enough. The connected sync client +only learns about acknowledged write checkpoints when PowerSync sends a later +sync checkpoint whose source LSN is at or beyond the stored write checkpoint +head. + +If the replicator has already processed past that head before the write +checkpoint mapping is created, there may be no future replication event to cause +a commit or keepalive. On an idle Convex app, repeated `document_deltas` calls can +return the same cursor, so the Convex head does not progress just because time +passes. The client can then wait indefinitely for a checkpoint acknowledgement +that is logically already true but is never published through a newer sync +checkpoint. If the backend mutation made additional changes, the client may also +not observe those changes as the confirmed result of its write. + +### Case 1.2: manual write checkpoint creation + +Manually calling `/write-checkpoint2.json` has the same shape. The write +checkpoint is associated with the current Convex head, but if no later source +change occurs, the stream does not advance. The connected client will not see the +write checkpoint acknowledgement until an unrelated external change moves the +Convex cursor and replication commits or keeps alive at that newer cursor. + +## Case 2: checkpoint collection exists + +With the `powersync_checkpoints` collection and mutation deployed, +`createWriteCheckpointMarker()` creates a small source-side write immediately +after the head is captured. + +### Case 2.1: marker advances past the managed write checkpoint head + +The managed write checkpoint is associated with the pre-marker head. The marker +mutation is committed after that head, so its delta cursor is greater than or +equal to the point needed to acknowledge the managed write checkpoint. + +When the replicator sees the marker row in `document_deltas`, it ignores the row +as replicated data but calls `keepalive(pageLsn)`. That advances the PowerSync +checkpoint LSN past the managed write checkpoint head, allowing the generated +write checkpoint id to be sent to the client. + +This works regardless of whether the normal user-data write was already +replicated before `/write-checkpoint2.json` ran, as long as the marker-driven +checkpoint update is observed after the managed write checkpoint mapping exists. +That is the event that makes the stored write checkpoint visible to sync. + +## Comparison to other sources + +This is the same role played by source-specific keepalive mechanisms elsewhere: + +- Postgres stores the current WAL LSN and emits a logical replication message so + replication proceeds past that LSN. +- MongoDB stores the current `clusterTime` and writes to + `_powersync_checkpoints` so the change stream proceeds past that time. +- Convex stores the current cursor and writes to `powersync_checkpoints` so + `document_deltas` proceeds past that cursor. + +The common requirement is not "write checkpoint data must be replicated as user +data". The requirement is that, after creating the managed write checkpoint +mapping, the source stream must produce an ordered event beyond the stored +position. + +## Filtered replication streams + +There is an additional failure mode for source databases with filtered +replication streams. For example, imagine a Postgres setup where the logical +replication stream only includes table `A`. + +1. The client writes to table `A`. +2. Some backend logic writes to table `B`. +3. The client then requests a write checkpoint. +4. PowerSync reads the current source head after the table `B` write. + +That source head is a valid database LSN, but the filtered replication stream may +never emit the table `B` change. If no later table `A` change occurs, replication +will not observe an event at or beyond the stored write checkpoint LSN, even +though the source database head has advanced. The write checkpoint can then be +stuck waiting for a position that is real in the source database but not +reachable through the filtered stream. + +That is why filtered sources need a marker that is guaranteed to appear in the +replication stream. In Postgres this can be a logical replication message; in +MongoDB this is a write to a collection included in the change stream. The marker +bridges the gap between "the database head advanced" and "the replication stream +observed progress". + +This particular filtered-stream issue does not appear to apply to Convex in the +same way. Convex `document_deltas` is not filtered by table at the API level for +PowerSync; table filtering happens inside the replicator after it receives the +delta page. A write to another Convex table should still advance the Convex delta +cursor visible to the replicator, even if PowerSync later ignores that row +because it is not included in sync rules. + +So for Convex, the checkpoint marker is less about overcoming source-side stream +filtering and more about guaranteeing source-side progress when there are no +other writes after the managed checkpoint mapping is created. The marker is still +required because an idle Convex deployment may otherwise keep returning the same +delta cursor, but the reason is different from a filtered Postgres slot that +cannot observe some source LSNs at all. + +## Test results + +Several manual tests were run to check whether Convex can safely omit the +`powersync_checkpoints` marker collection. + +### Test 1: direct write checkpoint on an idle source + +When `/write-checkpoint2.json` was called directly without any other client +mutation: + +- without writing to `powersync_checkpoints`, the write checkpoint did not show + up immediately in the client logs or PowerSync service sync logs, +- with the `powersync_checkpoints` marker write enabled, the write checkpoint did + show up immediately. + +This confirms the idle-source failure mode. A write checkpoint can be stored in +PowerSync bucket storage, but without a later Convex delta there may be no sync +checkpoint diff that exposes it to the connected client. + +### Test 2: replication lag + +When replication lag was introduced, the write checkpoint only appeared after +replication caught up. + +This is the desired behavior. The client should not validate a write checkpoint +until PowerSync has replicated to a source position at or beyond the head stored +for that write checkpoint. + +### Test 3: transaction boundaries and unfiltered deltas + +Testing also indicates that `document_deltas` pages contain entire Convex +transactions, or groups of smaller complete transactions. The current replicator +commits after each page that contains changes, so the marker collection does not +appear to be needed for transaction-boundary correctness. + +Testing also supports the expectation that Convex `document_deltas` is not +filtered per table at the API level. PowerSync filters rows inside the +replicator after receiving the delta page. This means Convex should not have the +same source-side filtered-stream problem described above for Postgres table +filters. + +### Test 4: delayed write checkpoint association + +Another test disabled the `powersync_checkpoints` marker write and added a +2-second delay in the Convex API handler before creating the managed write +checkpoint. + +In that setup, the client did not receive a checkpoint associated with the write +checkpoint. The likely sequence is: + +1. the client mutation committed in Convex, +2. the replicator saw the corresponding Convex cursor, +3. PowerSync sent the sync checkpoint to the client before the managed write + checkpoint mapping existed, +4. the API handler created the managed write checkpoint after the delay, +5. no further Convex write occurred, so no later checkpoint diff was sent. + +This reproduces the key race. The source head can be correct and the data can be +replicated, but the connected client can still miss the write checkpoint +acknowledgement if the write checkpoint mapping is created after the relevant +sync checkpoint has already been sent. + +## Conclusion + +With the current Convex replication API and managed write checkpoint design, the +write to `powersync_checkpoints` is required. + +The marker collection is not required because PowerSync needs to sync the marker +document itself. It is required because Convex cursors only advance on source +writes, and managed write checkpoints are only acknowledged after bucket storage +commits or keeps alive at a cursor at or beyond the stored head. Without the +marker write, a write checkpoint can be associated with a correct Convex head but +never become visible to the connected client on an idle source, especially when +replication has already processed past that head before the write checkpoint +mapping was stored. + +Avoiding source writes would require a different Convex capability or a different +PowerSync design, for example an API that can produce an ordered replication +barrier without a mutation, or storage logic that can safely publish a newly +created managed write checkpoint against an already-committed source head. Until +then, `CONVEX_CHECKPOINT_TABLE` is the mechanism that makes managed write +checkpoints reliably observable. The implementation should also preserve the +ordering invariant above; otherwise the marker can be replicated before the +managed mapping exists, recreating the same idle-source stall in a narrower race. diff --git a/docs/convex/schema-change-handling.md b/docs/convex/schema-change-handling.md new file mode 100644 index 000000000..9e7299b64 --- /dev/null +++ b/docs/convex/schema-change-handling.md @@ -0,0 +1,148 @@ +# Convex schema change handling + +This note explains why the Convex replicator does not implement the same +schema-change detection flow used by relational and CDC-based replicators. + +## What conventional schema change handling is for + +In source modules such as Postgres, MySQL, MongoDB, and SQL Server, schema or +relation change handling is used to keep PowerSync's source table metadata in +step with the source database. Typical reasons include: + +1. detecting changed replica identity columns, +2. refreshing cached table or relation metadata, +3. handling table rename, drop, and truncate events, +4. detecting source object identity changes, +5. deciding whether a table must be dropped, truncated, or re-snapshotted, +6. updating column metadata used by row decoding. + +Those concerns matter when the replication stream contains DDL, relation +messages, capture-instance changes, table ids, or column metadata that affect how +PowerSync should interpret subsequent row changes. + +## Why Convex is different + +Convex replication uses the Streaming Export APIs: + +- `json_schemas` for table-name discovery and schema preview, +- `list_snapshot` for initial table snapshots, +- `document_deltas` for ongoing changes. + +The replicated row shape comes from the actual JSON document returned by +`list_snapshot` and `document_deltas`. It does not come from `json_schemas`. +The Convex row converter iterates over the document payload, skips PowerSync's +internal Convex metadata fields, and converts the JSON-compatible values into +Sync Streams input values. + +That means ordinary Convex document shape changes flow through as data changes: + +- if a field is added, later snapshot/delta payloads include the field, +- if a field is removed, later payloads omit the field, +- if a field changes type, later payloads carry the new JSON value, +- if a migration updates existing documents, those updates are normal Convex + writes and should appear in `document_deltas`. + +Convex tables also always use `_id` as the replication identity in this module. +There is no source-specific primary key or replica identity definition to track, +so the usual "replica identity changed" path does not apply. + +## How table discovery works + +Exact table patterns in Sync Streams rules do not need `json_schemas` for +replication. The table name is already present in the sync rule, and the stream +can resolve that table directly. + +Wildcard table patterns do need a source of table names during initial +replication. For those patterns, the Convex replicator uses `json_schemas` to +expand the wildcard into concrete table names before snapshotting. + +An integration test verifies the key assumption behind that behavior: +`json_schemas` lists schema-defined tables even when those tables contain no +documents. This means initial wildcard expansion can discover empty tables +without waiting for a first row write. + +## New tables while streaming + +When `document_deltas` contains a row for a table that is not in the relation +cache, the replicator checks whether that table is selected by the current Sync +Streams rules. If it is selected, the table is resolved in storage and marked as +snapshot-complete at the current delta cursor. + +The replicator does not run an inline snapshot for that newly observed table. +The reasoning is: + +1. existing wildcard-selected tables were already discovered through + `json_schemas` during initial replication, +2. a table that appears later in `document_deltas` has appeared because a source + write occurred, +3. the delta payload contains the full document state needed for replication, +4. an empty newly-created table has no data to snapshot. + +So the delta stream itself is the source of truth for newly observed table data +after the initial snapshot boundary. + +## When a re-snapshot is still needed + +Convex does not need a re-snapshot merely because a field was added, removed, or +changed type. Those are document changes and should replicate through normal +deltas. + +A re-snapshot is still required for cases that change the selected data set or +invalidate the source cursor model: + +- initial replication of tables selected by the current Sync Streams rules, +- a Sync Streams deployment that selects new existing data, +- restarting initial replication from an incomplete snapshot, +- a lost or expired Convex cursor that requires restarting from a fresh + snapshot boundary. + +These are selection or consistency-boundary events, not schema-metadata drift +events. + +## Table drops + +The Convex stream does not continuously diff `json_schemas` to detect that a +table has disappeared from the source schema. + +Validation against the Convex dashboard showed that deleting a table from the +dashboard does not emit per-document `_deleted` rows in `document_deltas`. That +means PowerSync will not automatically remove previously replicated rows for that +table, and clients can continue to see stale synced data. + +If a table needs to be decommissioned while preserving replication correctness, +clear the table before deleting it. In the Convex dashboard, use the "Clear +Table" action first, then delete the table after those document removals have +replicated. Deleting documents through Convex mutations is also valid when that +path emits document delete deltas. Otherwise, treat dashboard table deletion or +schema-only table removal as a sync-rule/deployment state change: review the +affected rules and re-replicate or clear affected PowerSync state as needed. + +## Role of `json_schemas` + +`json_schemas` remains useful for: + +- validating that the Convex deployment is reachable, +- checking that required support tables such as `powersync_checkpoints` exist, +- presenting database schema information through API/debug routes, +- expanding wildcard table patterns at initial replication time. + +It is intentionally not used for runtime row decoding or schema drift detection. +Convex `json_schemas` can omit field detail until data exists, and using it for +type coercion would make replicated values depend on metadata availability +rather than on the source JSON payload. + +## Summary + +For Convex, schema changes are best understood as document-shape changes rather +than DDL events. Since PowerSync replicates the JSON payloads directly and `_id` +is always the replica identity, conventional schema-change detection has little +value for correctness. + +The important guarantees are instead: + +1. initial wildcard expansion can discover all schema-defined tables, +2. snapshot reads use a pinned Convex snapshot cursor, +3. streaming reads the unfiltered global `document_deltas` cursor in order, +4. new selected tables observed in deltas are resolved without an inline + snapshot, +5. migrations that update data are replicated as normal document changes. diff --git a/docs/convex/snapshot-consistency.md b/docs/convex/snapshot-consistency.md new file mode 100644 index 000000000..734122807 --- /dev/null +++ b/docs/convex/snapshot-consistency.md @@ -0,0 +1,136 @@ +# Convex snapshot consistency + +The current approach should be consistent: + +1. pin one global Convex snapshot cursor, +2. snapshot each selected table at that same cursor, +3. persist that cursor as the replication resume point, +4. stream `document_deltas` starting from that original cursor. + +The important property is that every table snapshot is read at the same Convex +snapshot cursor, not that all tables are fetched in one HTTP response. Since +`list_snapshot` accepts a `snapshot` parameter, we can snapshot tables +individually while still reading the same database state. + +Filtering specific tables should also be consistent as long as filtering happens +inside the PowerSync replicator after reading the unfiltered Convex delta stream. +It would become more delicate if Convex exposed a table-filtered delta stream and +PowerSync used that as the source of truth, because then some source cursor +positions could be skipped by the stream. + +## Current initial replication flow + +The Convex replicator does not take independent "latest" snapshots per table. +Instead, it resolves a single snapshot boundary and reuses it for every table: + +1. `getGlobalSnapshotCursor()` calls `list_snapshot` without a table filter to + get the latest global snapshot cursor. +2. The replicator stores that cursor as the resume LSN with `setResumeLsn`. +3. Each selected source table is snapshotted with `list_snapshot({ tableName, snapshot: snapshotCursor })`. +4. Each page returned by `list_snapshot` is checked to ensure the returned + `snapshot` still matches the original `snapshotCursor`. +5. After all selected tables are snapshotted, the replicator marks the snapshot + done at that same LSN and commits. +6. Streaming replication starts from the same cursor using `document_deltas`. + +This is the usual snapshot-then-stream consistency pattern. Rows changed while +the snapshot is being read are allowed to be missed by the snapshot because they +will appear in the delta stream after the pinned cursor. + +## Why individual table snapshots are okay + +Fetching each table separately would be unsafe if each request implicitly used +"whatever the latest database state is at request time". In that model, table +`A` could be read at time `T1`, table `B` could be read at time `T2`, and the +combined snapshot could represent no real database state. + +Convex avoids that problem by allowing a fixed snapshot cursor to be supplied to +`list_snapshot`. Once the replicator has pinned cursor `S`, every table snapshot +request is for the state of that table at `S`. + +That means a full "snapshot all tables at once" API is not required for +consistency. It may be convenient, but the consistency boundary is already the +shared snapshot cursor. + +## Relationship to document deltas + +The second half of the consistency guarantee is the relationship between the +snapshot cursor and `document_deltas`. + +For the current design to be correct, the cursor returned by `list_snapshot` +must be usable as the cursor passed to `document_deltas`, and `document_deltas` +must then return all committed changes after that cursor in order. Based on the +behavior tested so far, Convex's streaming export APIs appear to provide this: + +- backend mutations are reflected in the snapshot head after the mutation + succeeds, +- delta pages appear to contain complete Convex transactions, or groups of + complete transactions, +- the delta cursor is a global source position rather than a per-table position. + +With those properties, the snapshot and stream compose cleanly. The snapshot +captures the selected tables at cursor `S`; the delta stream then advances the +replica from `S` onward. + +## Table filtering + +PowerSync can still filter which tables it stores and applies to sync rules. +The key distinction is where filtering happens. + +The safe shape is: + +1. read the global Convex delta stream, +2. observe every Convex cursor in order, +3. ignore rows for tables not selected by sync rules, +4. commit or keepalive based on the cursor that was observed. + +This is the current model: Convex `document_deltas` is not filtered per table at +the API level for PowerSync. The replicator receives delta pages first and then +decides whether each row matches a selected source table. + +That means an unrelated table write can still move the cursor seen by the +replicator, even if PowerSync ignores the row. This is important for checkpoint +progress and avoids the filtered-stream issue seen in sources where the +replication stream itself only includes selected tables. + +## What would be risky + +Consistency would become trickier if PowerSync used a Convex API mode that +filtered deltas before the replicator saw them. + +For example, suppose the source cursor advances for writes to tables `A` and +`B`, but PowerSync only subscribes to a stream of table `A` deltas. If a write to +table `B` advances the global cursor, the source head may move to a position +that the filtered stream never emits. That can make resume points and write +checkpoint heads harder to reason about. + +This does not appear to be the current Convex setup. The current setup filters +after receiving the delta page, so the replicator can still observe cursor +movement caused by ignored tables. + +## Resume behavior + +Because the snapshot cursor is persisted before table snapshotting begins, the +initial snapshot can be resumed against the same global snapshot boundary. Table +progress stores the table page cursor and whether that table's snapshot is +finished. + +On restart, the replicator should continue using the original snapshot cursor +for unfinished tables. Completed tables do not need to be snapshotted again. Once +all tables are marked complete, streaming resumes from the same global cursor and +replays any later deltas. + +This is why resumable per-table snapshots are compatible with consistency: the +resume point is not a new per-table "latest" snapshot. It is the original global +snapshot cursor. + +## Conclusion + +There does not appear to be a consistency issue with snapshotting Convex tables +individually, as long as every table is snapshotted at the same pinned +`list_snapshot` cursor and `document_deltas` starts from that cursor. + +Likewise, filtering selected tables should be safe when filtering happens in the +PowerSync replicator after reading the unfiltered Convex delta stream. The risky +case would be source-side table-filtered deltas, where the global source cursor +can advance for writes the replication stream never exposes. diff --git a/libs/lib-services/src/schema/json-schema/keywords.ts b/libs/lib-services/src/schema/json-schema/keywords.ts index 06ee104a1..5a9b91824 100644 --- a/libs/lib-services/src/schema/json-schema/keywords.ts +++ b/libs/lib-services/src/schema/json-schema/keywords.ts @@ -4,7 +4,7 @@ export const BufferNodeType: ajv.KeywordDefinition = { keyword: 'nodeType', metaSchema: { type: 'string', - enum: ['buffer', 'date'] + enum: ['bigint', 'buffer', 'date'] }, error: { message: ({ schemaCode }) => { @@ -13,6 +13,9 @@ export const BufferNodeType: ajv.KeywordDefinition = { }, code(context) { switch (context.schema) { + case 'bigint': { + return context.fail(ajv._`typeof ${context.data} != 'bigint'`); + } case 'buffer': { return context.fail(ajv._`!Buffer.isBuffer(${context.data})`); } diff --git a/modules/module-convex/.gitignore b/modules/module-convex/.gitignore new file mode 100644 index 000000000..8c5fbb9ce --- /dev/null +++ b/modules/module-convex/.gitignore @@ -0,0 +1,2 @@ + +.env.local diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md new file mode 100644 index 000000000..512585e1c --- /dev/null +++ b/modules/module-convex/README.md @@ -0,0 +1,186 @@ +# PowerSync Convex Module + +Convex replication module for PowerSync. + +> [!WARNING] +> The Convex replicator is currently released as an alpha feature. APIs, configuration, metrics, schema-change handling, +> and replication behavior may change before it is considered stable. Test carefully before using it with production +> workloads. + +## Configuration + +```yaml +replication: + connections: + - type: convex + deployment_url: https://.convex.cloud + deploy_key: + polling_interval_ms: 1000 + request_timeout_ms: 30000 +``` + +## Manual smoke test + +1. Simplest is to run the react-convex-todolist demo in the powersync-js[repo](https://github.com/powersync-ja/powersync-js) + +## Development + +Run the `dev:convex` script to start the local Convex development backend used by the module tests. + +```bash +# In the modules/module-convex folder +pnpm run dev:convex + +# OR +# From the repo root +pnpm run -C modules/module-convex dev:convex +``` + +The local backend listens on `http://127.0.0.1:3210` by default. The integration tests read the local deploy key from +`modules/module-convex/.convex/local/default/config.json`, which is created by `dev:convex`. + +To run the Convex module tests locally: + +```bash +# Terminal 1 +pnpm run -C modules/module-convex dev:convex + +# Terminal 2, from the repo root +pnpm --filter='./modules/module-convex' test +``` + +Some integration tests are gated behind `CI=true` or `SLOW_TESTS=true`. To run them locally, keep `dev:convex` running +and start the required storage backends (MongoDB and Postgres storage), then run: + +```bash +CI=true pnpm --filter='./modules/module-convex' test +``` + +## Technical notes + +The content below is written in an agents.md style describing the behavior of `module-convex`. + +## 1) Scope + +- This module replicates Convex data into PowerSync bucket storage. +- Source APIs used are Convex [Streaming Export](https://docs.convex.dev/streaming-export-api): (`json_schemas`, `list_snapshot`, `document_deltas`). +- Initial scope is default Convex component only, but we could consider support for custom components in the future if we can figure out consistency. +- Deploy keys grant root access (read/write on all tables), components could address this later. + +## 2) Canonical Behavior + +- Initial replication: + - Initial replication pins a global Convex snapshot boundary using `list_snapshot`. If this is omitted, it provides the global snapshot boundary [ref](https://docs.convex.dev/streaming-export-api#get-apilist_snapshot). + - Snapshot each selected Sync Streams table with that fixed `snapshot`. + - First per-table snapshot call omits `cursor`; pagination cursor is only for later pages in the same run. + - Commit snapshot LSN, then switch to deltas. +- Streaming replication: + - Start from persisted resume LSN. + - Poll `document_deltas` using frequency configured in `polling_interval_ms` + - Always stream globally (no `tableName` filter), then filter locally by selected Sync Streams tables. + - If a table is first seen in a `document_deltas` page and matches Sync Streams, resolve it and apply the delta row directly. Do not snapshot it inline; initial wildcard expansion already discovers schema-defined tables through `json_schemas`, and the delta payload is the source of truth for later writes. + +## 3) Hard Invariants (Do Not Break) + +- `snapshot` is the consistency boundary; page `cursor` is pagination state. +- All table snapshots in a run must use the same pinned `snapshot`; if response snapshot differs, fail fast. +- On restart during initial replication: + - Reuse persisted snapshot LSN boundary. + - Resume table page walk from the persisted per-table `lastKey` cursor when available. + - If the last page was already flushed before interruption, mark the table snapshot done without re-reading rows. +- Delta streaming starts from resume LSN (snapshot boundary), not from table page cursor. +- `tablePattern.connectionTag` and schema must match before table selection. +- Source table replica identity is `_id`. +- The overall system must ensure causal consistency of replicated data in bucket storage. + +## 4) LSN and Cursor Rules + +- Convex snapshot and delta cursors are always `i64` timestamps (serialized as decimal numeric strings in JSON). +- The `list_snapshot` pagination cursor is a separate JSON-serialized `{table, id}` string — it is pagination state, not a replication cursor. +- Persisted Convex LSNs must be canonical 19-digit numeric cursor strings. `ZERO_LSN = "0"` remains the internal sentinel. + +## 5) API Client Contract + +- Auth header: `Authorization: Convex `. +- Always request `format=json`. +- Parse large numeric JSON using `JSONBig`. +- Convex API response shape validation is disabled by default. Set `POWERSYNC_DEV_CHECK_CONVEX_RESPONSES` before service startup to validate responses with the shared ts-codec JSON schema validator while debugging API compatibility. +- Retry classification: + - retryable: network, timeout, 429, 5xx. + - non-retryable: malformed responses, auth/config issues. + +## 6) Schema Changes + +- Conventional schema change handling is mainly used to detect changed replica identity columns, update cached table metadata, drop/rename tables, and trigger a table re-snapshot when DDL changed storage semantics. +- Convex tables always use `_id` as the replication identity, so there is no equivalent replica identity drift to detect. +- Stream row conversion uses the JSON document returned by `list_snapshot` and `document_deltas`, not Convex `json_schemas` metadata. Added fields, removed fields, and type changes are therefore replicated through normal document mutations. +- Convex data migrations are expected to run as writes/mutations over live documents. Those updates should appear in `document_deltas` and be replicated without a schema-driven re-snapshot. +- Exact table patterns are resolved directly from Sync Streams rules. `json_schemas` is only used for initial wildcard table expansion and API/debug schema previews. +- A re-snapshot is still required for initial replication, a sync-rule deployment that selects new existing data, or a lost/expired cursor, but not merely because a Convex field was added, removed, or changed type. +- Table drops are not (yet) detected by continuously diffing `json_schemas`. Validation showed that deleting a table from the Convex dashboard does not emit per-document `_deleted` rows in `document_deltas`, so previously replicated rows can remain synced to clients. Use the dashboard "Clear Table" action before deleting a table, or delete documents through mutation paths that emit document deltas. Otherwise, handle dashboard/schema-only table removal as a sync-rule/deployment state change and clear/re-replicate affected PowerSync state. +- See [Convex schema change handling](../../docs/convex/schema-change-handling.md) for the detailed rationale and limitations. + +## 7) Datatype Mapping + +- Current runtime mapping in stream writer: + +| Convex Type | JSON wire / Sync Streams value | SQLite type | +| ----------- | ------------------------------ | ----------- | +| Id | string | text | +| Null | null | null | +| Int64 | base10 string | text | +| Float64 | number | real | +| Boolean | boolean | integer | +| String | string | text | +| Bytes | base64 string | text | +| Array | Array | text | +| Object | Object | text | +| Record | Record | text | + +- Convex does not expose a native `Date` wire type; timestamps arrive as `number` or `string`. +- Value conversion flow: + 1. PowerSync requests snapshots or document deltas from Convex's Streaming Export APIs. + 2. The JSON response is parsed into raw Convex documents, where row columns are represented as JSON object fields. + 3. PowerSync converts the raw JSON-compatible values to SQLite-compatible values without using `json_schemas` metadata for datatype coercion. +- Convex's `json_schemas` endpoint appears to infer table schemas from table summaries instead of using the TypeScript schema as a complete source of truth. This means fields can be absent from `json_schemas` until populated data exists. To avoid changing replicated value types based on whether schema metadata was available, Sync Streams see the stable JSON wire representation. For `Int64` columns, users should explicitly cast those values in Sync Streams rules using `CAST(value AS INTEGER)`. + +## 8) Checkpointing and Consistency + +- `createReplicationHead` must: + 1. resolve global head cursor, + 2. pass that head to the callback so PowerSync stores the managed write checkpoint mapping, + 3. then write a Convex checkpoint marker via `POST /api/mutation` (calls `powersync_checkpoints:createCheckpoint`). +- The marker must be written after the callback. If the marker is replicated before the managed write checkpoint mapping exists, an idle source can still leave the client waiting for a later observable checkpoint update. +- Source marker table: `powersync_checkpoints` + - Convex rejects table names starting with `_`, so no leading-underscore variant is used. + - The table has a single `last_updated` field; the mutation upserts one row (bounded to one row total). + - The developer must deploy the `powersync_checkpoints` schema and mutation to their Convex project. +- Stream handling requirement: + - checkpoint marker tables must always be excluded from replicated source tables and ignored in delta row application. + - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). + +## 9) Other Convex-specific notes + +- The default schema is `convex` +- On an idle system, multiple successive calls to `/api/document_deltas` will return the same cursor value i.e. the cursor is not wall clock based. + +- **Mutation Transaction Atomicity in** `document_deltas` + + - The `cursor` in `/api/document_deltas` is a Convex commit **timestamp** (`i64`), not a per-operation counter. + - Every Convex mutation is an ACID transaction that commits with a single timestamp; all writes within that mutation share the same `_ts` value in the delta stream. + - Therefore, the cursor advances **once per mutation**, not once per individual CRUD operation inside it. + - Example: a mutation that deletes 5 documents and updates 3 produces 8 entries in `document_deltas`, all with identical `_ts`. + - The Convex backend enforces this by never splitting a page mid-timestamp: when the row limit is reached mid-transaction, the page extends until all rows at that `_ts` are included before stopping. + - Consequence for replication: all writes from a single mutation always appear in the same `document_deltas` page and are committed to bucket storage atomically as one batch. + - `TRANSACTIONS_REPLICATED` is counted from distinct `_ts` values among replicated changes, not from `document_deltas` pages. A single page can contain multiple Convex mutations, so a committed page may increase the transaction metric by more than one. + - The stream relies on Convex returning `document_deltas` in mutation order by `_ts`. Row order within the same `_ts` is not significant: those rows belong to the same Convex mutation and are committed atomically. + - The stream asserts that observed `_ts` values are non-decreasing across pages. Equal `_ts` values are allowed because they represent rows from the same Convex mutation. + +- **Replication metrics** + - Implemented: + - `ROWS_REPLICATED`: incremented for each row written from snapshots and deltas. + - `TRANSACTIONS_REPLICATED`: incremented by the number of distinct Convex `_ts` mutation timestamps replicated from `document_deltas`. + - Not implemented yet: + - `DATA_REPLICATED_BYTES`: Convex does not currently report source bytes replicated into PowerSync. This would need explicit accounting in the Convex replication/client path. + - `CHUNKS_REPLICATED`: Convex does not currently report replication chunks. + - Bucket storage size gauges (`REPLICATION_SIZE_BYTES`, `OPERATION_SIZE_BYTES`, `PARAMETER_SIZE_BYTES`) are reported by the configured bucket storage backend, not by this replication module. diff --git a/modules/module-convex/convex/README.md b/modules/module-convex/convex/README.md new file mode 100644 index 000000000..373f9f488 --- /dev/null +++ b/modules/module-convex/convex/README.md @@ -0,0 +1 @@ +Note that the \_generated/ folder is intentionally tracked with Git. See https://docs.convex.dev/understanding/best-practices/other-recommendations#check-generated-code-into-version-control. diff --git a/modules/module-convex/convex/_generated/api.d.ts b/modules/module-convex/convex/_generated/api.d.ts new file mode 100644 index 000000000..8abbb03e8 --- /dev/null +++ b/modules/module-convex/convex/_generated/api.d.ts @@ -0,0 +1,57 @@ +/* eslint-disable */ +/** + * Generated `api` utility. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import type * as benchmark from "../benchmark.js"; +import type * as lists from "../lists.js"; +import type * as powersync_checkpoints from "../powersync_checkpoints.js"; +import type * as todos from "../todos.js"; +import type * as utils_collections from "../utils/collections.js"; + +import type { + ApiFromModules, + FilterApi, + FunctionReference, +} from "convex/server"; + +declare const fullApi: ApiFromModules<{ + benchmark: typeof benchmark; + lists: typeof lists; + powersync_checkpoints: typeof powersync_checkpoints; + todos: typeof todos; + "utils/collections": typeof utils_collections; +}>; + +/** + * A utility for referencing Convex functions in your app's public API. + * + * Usage: + * ```js + * const myFunctionReference = api.myModule.myFunction; + * ``` + */ +export declare const api: FilterApi< + typeof fullApi, + FunctionReference +>; + +/** + * A utility for referencing Convex functions in your app's internal API. + * + * Usage: + * ```js + * const myFunctionReference = internal.myModule.myFunction; + * ``` + */ +export declare const internal: FilterApi< + typeof fullApi, + FunctionReference +>; + +export declare const components: {}; diff --git a/modules/module-convex/convex/_generated/api.js b/modules/module-convex/convex/_generated/api.js new file mode 100644 index 000000000..44bf98583 --- /dev/null +++ b/modules/module-convex/convex/_generated/api.js @@ -0,0 +1,23 @@ +/* eslint-disable */ +/** + * Generated `api` utility. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import { anyApi, componentsGeneric } from "convex/server"; + +/** + * A utility for referencing Convex functions in your app's API. + * + * Usage: + * ```js + * const myFunctionReference = api.myModule.myFunction; + * ``` + */ +export const api = anyApi; +export const internal = anyApi; +export const components = componentsGeneric(); diff --git a/modules/module-convex/convex/_generated/dataModel.d.ts b/modules/module-convex/convex/_generated/dataModel.d.ts new file mode 100644 index 000000000..f97fd1942 --- /dev/null +++ b/modules/module-convex/convex/_generated/dataModel.d.ts @@ -0,0 +1,60 @@ +/* eslint-disable */ +/** + * Generated data model types. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import type { + DataModelFromSchemaDefinition, + DocumentByName, + TableNamesInDataModel, + SystemTableNames, +} from "convex/server"; +import type { GenericId } from "convex/values"; +import schema from "../schema.js"; + +/** + * The names of all of your Convex tables. + */ +export type TableNames = TableNamesInDataModel; + +/** + * The type of a document stored in Convex. + * + * @typeParam TableName - A string literal type of the table name (like "users"). + */ +export type Doc = DocumentByName< + DataModel, + TableName +>; + +/** + * An identifier for a document in Convex. + * + * Convex documents are uniquely identified by their `Id`, which is accessible + * on the `_id` field. To learn more, see [Document IDs](https://docs.convex.dev/using/document-ids). + * + * Documents can be loaded using `db.get(tableName, id)` in query and mutation functions. + * + * IDs are just strings at runtime, but this type can be used to distinguish them from other + * strings when type checking. + * + * @typeParam TableName - A string literal type of the table name (like "users"). + */ +export type Id = + GenericId; + +/** + * A type describing your Convex data model. + * + * This type includes information about what tables you have, the type of + * documents stored in those tables, and the indexes defined on them. + * + * This type is used to parameterize methods like `queryGeneric` and + * `mutationGeneric` to make them type-safe. + */ +export type DataModel = DataModelFromSchemaDefinition; diff --git a/modules/module-convex/convex/_generated/server.d.ts b/modules/module-convex/convex/_generated/server.d.ts new file mode 100644 index 000000000..bec05e681 --- /dev/null +++ b/modules/module-convex/convex/_generated/server.d.ts @@ -0,0 +1,143 @@ +/* eslint-disable */ +/** + * Generated utilities for implementing server-side Convex query and mutation functions. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import { + ActionBuilder, + HttpActionBuilder, + MutationBuilder, + QueryBuilder, + GenericActionCtx, + GenericMutationCtx, + GenericQueryCtx, + GenericDatabaseReader, + GenericDatabaseWriter, +} from "convex/server"; +import type { DataModel } from "./dataModel.js"; + +/** + * Define a query in this Convex app's public API. + * + * This function will be allowed to read your Convex database and will be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export declare const query: QueryBuilder; + +/** + * Define a query that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to read from your Convex database. It will not be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export declare const internalQuery: QueryBuilder; + +/** + * Define a mutation in this Convex app's public API. + * + * This function will be allowed to modify your Convex database and will be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export declare const mutation: MutationBuilder; + +/** + * Define a mutation that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to modify your Convex database. It will not be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export declare const internalMutation: MutationBuilder; + +/** + * Define an action in this Convex app's public API. + * + * An action is a function which can execute any JavaScript code, including non-deterministic + * code and code with side-effects, like calling third-party services. + * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. + * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. + * + * @param func - The action. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped action. Include this as an `export` to name it and make it accessible. + */ +export declare const action: ActionBuilder; + +/** + * Define an action that is only accessible from other Convex functions (but not from the client). + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped function. Include this as an `export` to name it and make it accessible. + */ +export declare const internalAction: ActionBuilder; + +/** + * Define an HTTP action. + * + * The wrapped function will be used to respond to HTTP requests received + * by a Convex deployment if the requests matches the path and method where + * this action is routed. Be sure to route your httpAction in `convex/http.js`. + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument + * and a Fetch API `Request` object as its second. + * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. + */ +export declare const httpAction: HttpActionBuilder; + +/** + * A set of services for use within Convex query functions. + * + * The query context is passed as the first argument to any Convex query + * function run on the server. + * + * This differs from the {@link MutationCtx} because all of the services are + * read-only. + */ +export type QueryCtx = GenericQueryCtx; + +/** + * A set of services for use within Convex mutation functions. + * + * The mutation context is passed as the first argument to any Convex mutation + * function run on the server. + */ +export type MutationCtx = GenericMutationCtx; + +/** + * A set of services for use within Convex action functions. + * + * The action context is passed as the first argument to any Convex action + * function run on the server. + */ +export type ActionCtx = GenericActionCtx; + +/** + * An interface to read from the database within Convex query functions. + * + * The two entry points are {@link DatabaseReader.get}, which fetches a single + * document by its {@link Id}, or {@link DatabaseReader.query}, which starts + * building a query. + */ +export type DatabaseReader = GenericDatabaseReader; + +/** + * An interface to read from and write to the database within Convex mutation + * functions. + * + * Convex guarantees that all writes within a single mutation are + * executed atomically, so you never have to worry about partial writes leaving + * your data in an inconsistent state. See [the Convex Guide](https://docs.convex.dev/understanding/convex-fundamentals/functions#atomicity-and-optimistic-concurrency-control) + * for the guarantees Convex provides your functions. + */ +export type DatabaseWriter = GenericDatabaseWriter; diff --git a/modules/module-convex/convex/_generated/server.js b/modules/module-convex/convex/_generated/server.js new file mode 100644 index 000000000..bf3d25ad3 --- /dev/null +++ b/modules/module-convex/convex/_generated/server.js @@ -0,0 +1,93 @@ +/* eslint-disable */ +/** + * Generated utilities for implementing server-side Convex query and mutation functions. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import { + actionGeneric, + httpActionGeneric, + queryGeneric, + mutationGeneric, + internalActionGeneric, + internalMutationGeneric, + internalQueryGeneric, +} from "convex/server"; + +/** + * Define a query in this Convex app's public API. + * + * This function will be allowed to read your Convex database and will be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export const query = queryGeneric; + +/** + * Define a query that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to read from your Convex database. It will not be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export const internalQuery = internalQueryGeneric; + +/** + * Define a mutation in this Convex app's public API. + * + * This function will be allowed to modify your Convex database and will be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export const mutation = mutationGeneric; + +/** + * Define a mutation that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to modify your Convex database. It will not be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export const internalMutation = internalMutationGeneric; + +/** + * Define an action in this Convex app's public API. + * + * An action is a function which can execute any JavaScript code, including non-deterministic + * code and code with side-effects, like calling third-party services. + * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. + * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. + * + * @param func - The action. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped action. Include this as an `export` to name it and make it accessible. + */ +export const action = actionGeneric; + +/** + * Define an action that is only accessible from other Convex functions (but not from the client). + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped function. Include this as an `export` to name it and make it accessible. + */ +export const internalAction = internalActionGeneric; + +/** + * Define an HTTP action. + * + * The wrapped function will be used to respond to HTTP requests received + * by a Convex deployment if the requests matches the path and method where + * this action is routed. Be sure to route your httpAction in `convex/http.js`. + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument + * and a Fetch API `Request` object as its second. + * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. + */ +export const httpAction = httpActionGeneric; diff --git a/modules/module-convex/convex/benchmark.ts b/modules/module-convex/convex/benchmark.ts new file mode 100644 index 000000000..576d693e6 --- /dev/null +++ b/modules/module-convex/convex/benchmark.ts @@ -0,0 +1,45 @@ +import { v } from 'convex/values'; +import { mutation } from './_generated/server.js'; + +export const BENCHMARK_LIST_PAYLOAD = { + uuid: 'benchmark-list', + name: 'Benchmark list' +} as const; + +export const BENCHMARK_TODO_PAYLOAD = { + uuid: 'benchmark-todo', + description: 'Benchmark todo', + list_uuid: BENCHMARK_LIST_PAYLOAD.uuid +} as const; + +/** + * Creates seed data for sync replication benchmarking purposes. + * + * Logical payload estimate for the current row shape is calculated in the benchmark test from + * these payload constants. Each loop creates one list and one todo, so the estimate is: + * rowsPerTable * (encoded JSON list bytes + encoded JSON todo bytes + representative list_id bytes). + * Convex write-limit accounting can be higher because it also includes document metadata, + * indexes and storage internals. + */ +export const seedInitialReplicationBatch = mutation({ + args: { + rowsPerTable: v.number() + }, + handler: async (ctx, args) => { + for (let i = 0; i < args.rowsPerTable; i++) { + const listId = await ctx.db.insert('lists', { + ...BENCHMARK_LIST_PAYLOAD + }); + + await ctx.db.insert('todos', { + ...BENCHMARK_TODO_PAYLOAD, + list_id: listId + }); + } + + return { + lists: args.rowsPerTable, + todos: args.rowsPerTable + }; + } +}); diff --git a/modules/module-convex/convex/lists.ts b/modules/module-convex/convex/lists.ts new file mode 100644 index 000000000..520b841c0 --- /dev/null +++ b/modules/module-convex/convex/lists.ts @@ -0,0 +1,122 @@ +import { v } from 'convex/values'; +import { mutation, query } from './_generated/server.js'; +import schema from './schema.js'; +import { findListByUuid } from './utils/collections.js'; + +const BATCH_SIZE = 2000; + +export const get = query({ + args: {}, + handler: async (ctx) => { + return await ctx.db.query('lists').collect(); + } +}); + +export const deleteItem = mutation({ + args: { + uuid: v.string() + }, + handler: async ({ db }, { uuid }) => { + const found = await findListByUuid({ db: db, uuid }); + if (!found) { + throw new Error('Not found'); + } + await db.delete('lists', found._id); + } +}); + +export const updateName = mutation({ + args: { + uuid: v.string(), + name: v.string() + }, + handler: async ({ db }, { uuid, name }) => { + const found = await findListByUuid({ db, uuid }); + if (!found) { + throw new Error('Not found'); + } + await db.patch(found._id, { name }); + } +}); + +/** + * Deletes a batch of items. + * Convex limits the number of ops in a mutation/transaction to 16_000. + * This will return the number of items deleted. + * Rerun this till completion if all items should be deleted. + */ +export const deleteBatch = mutation({ + args: { + batch_size: v.optional(v.number()) + }, + handler: async (ctx, args) => { + const batchSize = args.batch_size ?? BATCH_SIZE; + const lists = await ctx.db.query('lists').take(batchSize); + for (const list of lists) { + await ctx.db.delete(list._id); + } + return lists.length; + } +}); + +export const createBatch = mutation({ + args: { + lists: v.array(schema.tables.lists.validator) + }, + handler: async (ctx, args) => { + const ids = []; + for (const list of args.lists) { + const id = await ctx.db.insert('lists', list); + ids.push(id); + } + return ids; + } +}); + +/** + * A test mutation which creates and updates multiple lists multiple times. + * This is used to ensure the order of ops is safely replicated. + */ +export const testUpdateMultipleTimes = mutation({ + args: {}, + handler: async (ctx) => { + const listCount = 5; + const createdListIds = []; + for (let i = 0; i < listCount; i++) { + const id = await ctx.db.insert('lists', { + name: `list-${i}`, + uuid: 'fake-uuid' + }); + createdListIds.push(id); + } + + // Update all the lists + for (let i = 0; i < listCount; i++) { + await ctx.db.patch('lists', createdListIds[i]!, { + name: `list-${i}-a`, + uuid: createdListIds[i]! // keep this for later + }); + } + + // Do a second update, but operate on the items in reverse order + for (let i = listCount - 1; i >= 0; i--) { + await ctx.db.patch('lists', createdListIds[i]!, { + name: `list-${i}-a-b` + }); + } + + for (let i = 0; i < listCount; i++) { + await ctx.db.patch('lists', createdListIds[i]!, { + name: `list-${i}-a-b-c` + }); + } + + // delete a random list + const [deleteId] = createdListIds.splice(Math.floor(Math.random() * createdListIds.length), 1); + await ctx.db.delete('lists', deleteId); + + return { + listIds: createdListIds + }; + } +}); diff --git a/modules/module-convex/convex/powersync_checkpoints.ts b/modules/module-convex/convex/powersync_checkpoints.ts new file mode 100644 index 000000000..7ad660756 --- /dev/null +++ b/modules/module-convex/convex/powersync_checkpoints.ts @@ -0,0 +1,14 @@ +import { mutation } from './_generated/server.js'; + +export const createCheckpoint = mutation({ + args: {}, + handler: async (ctx) => { + const existing = await ctx.db.query('powersync_checkpoints').first(); + + if (existing) { + await ctx.db.patch(existing._id, { last_updated: Date.now() }); + } else { + await ctx.db.insert('powersync_checkpoints', { last_updated: Date.now() }); + } + } +}); diff --git a/modules/module-convex/convex/schema.ts b/modules/module-convex/convex/schema.ts new file mode 100644 index 000000000..fa4209f25 --- /dev/null +++ b/modules/module-convex/convex/schema.ts @@ -0,0 +1,117 @@ +import { authTables } from '@convex-dev/auth/server'; +import { defineSchema, defineTable } from 'convex/server'; +import { v } from 'convex/values'; + +export default defineSchema({ + ...authTables, + lists: defineTable({ + created_at: v.optional(v.string()), + name: v.string(), + owner_id: v.optional(v.string()), + tags: v.optional(v.array(v.string())), + attributes: v.optional(v.record(v.string(), v.string())), + settings: v.optional( + v.object({ + theme: v.string(), + color: v.string(), + is_public: v.boolean() + }) + ), + + // External stable key from PowerSync writes. + uuid: v.string(), + // Legacy fields kept for backwards compatibility with existing data. + owner: v.optional(v.string()), + archived: v.optional(v.number()) + }).index('by_uuid', ['uuid']), + + todos: defineTable({ + // Basic fields + /** + * Due to ID mapping, we cant require a strict Convex `id` ID field - which + * correlates to a matching todo record here. + * This value will be the local-first UUID. + */ + uuid: v.string(), + created_at: v.optional(v.string()), + completed_at: v.optional(v.union(v.null(), v.string())), + description: v.string(), + list_id: v.id('lists'), + /** + * Local-first version of list_id + */ + list_uuid: v.string(), + + // All Convex datatypes for stress testing + // String types + title: v.optional(v.string()), + notes: v.optional(v.string()), + category: v.optional(v.string()), + + // Number types + priority: v.optional(v.number()), + points: v.optional(v.int64()), + estimated_hours: v.optional(v.float64()), + progress_percentage: v.optional(v.float64()), + + // Boolean types + is_urgent: v.optional(v.boolean()), + is_private: v.optional(v.boolean()), + has_attachments: v.optional(v.boolean()), + attachment_data: v.optional(v.bytes()), + + // Array types + tags: v.optional(v.array(v.string())), + dependencies: v.optional(v.array(v.id('todos'))), + assigned_users: v.optional(v.array(v.string())), + + // Object types + details: v.optional( + v.object({ + label: v.string(), + count: v.number(), + nested: v.object({ + enabled: v.boolean() + }) + }) + ), + metadata: v.optional(v.record(v.string(), v.any())), + custom_fields: v.optional(v.record(v.string(), v.union(v.string(), v.number(), v.boolean()))), + + // ID references + parent_task_id: v.optional(v.id('todos')), + project_id: v.optional(v.id('lists')), + + // Union types + status: v.optional( + v.union(v.literal('pending'), v.literal('in_progress'), v.literal('completed'), v.literal('cancelled')) + ), + difficulty: v.optional(v.union(v.literal('easy'), v.literal('medium'), v.literal('hard'))), + + // Null handling + explicit_null: v.optional(v.null()), + archived_at: v.optional(v.union(v.null(), v.string())), + deleted_by: v.optional(v.union(v.null(), v.string())), + + // Legacy fields kept for backwards compatibility + completed: v.optional(v.number()), + created_by: v.optional(v.union(v.null(), v.string())), + completed_by: v.optional(v.union(v.null(), v.string())), + photo_id: v.optional(v.union(v.null(), v.string())), + owner_id: v.optional(v.string()) + }) + .index('by_uuid', ['uuid']) + .index('by_list_id', ['list_id']) + .index('by_status', ['status']) + .index('by_priority', ['priority']) + .index('by_project', ['project_id']), + + powersync_checkpoints: defineTable({ + last_updated: v.float64() + }), + + // An empty table which only is used to check if the json-schemas endpoint returns data for empty tables. + schema_only_probe: defineTable({ + marker: v.optional(v.string()) + }) +}); diff --git a/modules/module-convex/convex/todos.ts b/modules/module-convex/convex/todos.ts new file mode 100644 index 000000000..2474ef409 --- /dev/null +++ b/modules/module-convex/convex/todos.ts @@ -0,0 +1,58 @@ +import { v } from 'convex/values'; +import { mutation, query } from './_generated/server.js'; +import schema from './schema.js'; +import { findListByUuid } from './utils/collections.js'; + +const BATCH_SIZE = 2000; + +export const get = query({ + args: {}, + handler: async (ctx) => { + return await ctx.db.query('todos').collect(); + } +}); + +/** + * Deletes a batch of items. + * Convex limits the number of ops in a mutation/transaction to 16_000. + * This will return the number of items deleted. + * Rerun this till completion if all items should be deleted. + */ +export const deleteBatch = mutation({ + args: { + batch_size: v.optional(v.number()) + }, + handler: async (ctx, args) => { + const batchSize = args.batch_size ?? BATCH_SIZE; + const todos = await ctx.db.query('todos').take(batchSize); + for (const todo of todos) { + await ctx.db.delete(todo._id); + } + return todos.length; + } +}); + +export const createBatch = mutation({ + args: { + // We don't require clients usually to track the Convex list_id, they work with list_uuid + todos: v.array(schema.tables.todos.validator.omit('list_id')) + }, + handler: async (ctx, args) => { + const { db } = ctx; + const ids = []; + for (const todo of args.todos) { + const list = await findListByUuid({ db, uuid: todo.list_uuid }); + if (!list) { + // Continue for simple testing purposes + continue; + } + const id = await ctx.db.insert('todos', { + ...todo, + list_id: list._id + }); + + ids.push(id); + } + return ids; + } +}); diff --git a/modules/module-convex/convex/utils/collections.ts b/modules/module-convex/convex/utils/collections.ts new file mode 100644 index 000000000..fc7bbc8d9 --- /dev/null +++ b/modules/module-convex/convex/utils/collections.ts @@ -0,0 +1,17 @@ +import { DatabaseReader } from '../_generated/server.js'; + +export const findListByUuid = async (params: { db: DatabaseReader; uuid: string }) => { + const { db, uuid } = params; + return db + .query('lists') + .withIndex('by_uuid', (q) => q.eq('uuid', uuid)) + .first(); +}; + +export const findTodoByUuid = async (params: { db: DatabaseReader; uuid: string }) => { + const { db, uuid } = params; + return db + .query('todos') + .withIndex('by_uuid', (q) => q.eq('uuid', uuid)) + .first(); +}; diff --git a/modules/module-convex/development/scripts/convex-sanity-check.ts b/modules/module-convex/development/scripts/convex-sanity-check.ts new file mode 100644 index 000000000..c0fb52a4c --- /dev/null +++ b/modules/module-convex/development/scripts/convex-sanity-check.ts @@ -0,0 +1,12 @@ +import { ConvexHttpClient } from 'convex/browser'; +import * as dotenv from 'dotenv'; +import { api } from '../../convex/_generated/api.js'; + +dotenv.config({ path: '.env.local' }); + +const client = new ConvexHttpClient(process.env['CONVEX_URL']!); + +// List all lists +const lists = await client.query(api.lists.get); +console.log('Fetched the following lists from Convex'); +console.log(JSON.stringify(lists, null, '\t')); diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json new file mode 100644 index 000000000..8a8327fe7 --- /dev/null +++ b/modules/module-convex/package.json @@ -0,0 +1,50 @@ +{ + "name": "@powersync/service-module-convex", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/index.d.ts", + "version": "0.1.0", + "license": "FSL-1.1-ALv2", + "main": "dist/index.js", + "type": "module", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc -b", + "build:tests": "pnpm build:convex && tsc -b test/tsconfig.json", + "build:convex": "convex codegen", + "clean": "rm -rf ./dist && tsc -b --clean", + "test": "vitest", + "dev:convex": "convex dev" + }, + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./types": { + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js" + } + }, + "dependencies": { + "@powersync/lib-services-framework": "workspace:*", + "@powersync/service-core": "workspace:*", + "@powersync/service-jsonbig": "workspace:*", + "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "bson": "^6.10.4", + "node-fetch": "^3.3.2", + "ts-codec": "^1.3.0" + }, + "devDependencies": { + "@powersync/service-core-tests": "workspace:*", + "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-postgres-storage": "workspace:*", + "@convex-dev/auth": "^0.0.92", + "convex": "^1.38.0", + "dotenv": "^16.4.5" + } +} diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts new file mode 100644 index 000000000..5c0dccaa7 --- /dev/null +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -0,0 +1,214 @@ +import { api, ParseSyncRulesOptions, ReplicationHeadCallback, ReplicationLagOptions } from '@powersync/service-core'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as service_types from '@powersync/service-types'; +import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; +import { parseConvexLsn } from '../common/ConvexLSN.js'; +import { extractProperties, jsonSchemaToSQLiteType, readConvexFieldJsonType } from '../common/convex-to-sqlite.js'; +import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; +import { checkSourceConfiguration } from '../replication/check-source-configuration.js'; +import * as types from '../types/types.js'; + +export class ConvexRouteAPIAdapter implements api.RouteAPI { + protected connectionManager: ConvexConnectionManager; + + constructor(protected config: types.ResolvedConvexConnectionConfig) { + this.connectionManager = new ConvexConnectionManager(config); + } + + async getSourceConfig(): Promise { + return this.config; + } + + async getConnectionStatus(): Promise { + const base = { + id: this.config.id, + uri: types.baseUri(this.config) + }; + + try { + const { connected, errors } = await checkSourceConfiguration(this.config, { readOnly: true }); + return { + ...base, + connected, + errors: errors.map((e) => ({ level: 'fatal', message: e })) + }; + } catch (error) { + return { + ...base, + connected: false, + errors: [{ level: 'fatal', message: error instanceof Error ? error.message : `${error}` }] + }; + } + } + + async getDebugTablesInfo( + tablePatterns: sync_rules.TablePattern[], + sqlSyncRules: sync_rules.SyncConfig + ): Promise { + const schema = await this.connectionManager.client.getJsonSchemas(); + const tablesByName = new Map(schema.tables.map((table) => [table.tableName, table])); + + const result: api.PatternResult[] = []; + + for (const tablePattern of tablePatterns) { + const patternResult: api.PatternResult = { + schema: tablePattern.schema, + pattern: tablePattern.tablePattern, + wildcard: tablePattern.isWildcard + }; + + result.push(patternResult); + + if (tablePattern.connectionTag != this.connectionManager.connectionTag) { + if (tablePattern.isWildcard) { + patternResult.tables = []; + } else { + patternResult.table = createTableInfo({ + tablePattern, + connectionTag: this.connectionManager.connectionTag, + syncRules: sqlSyncRules, + errors: [{ level: 'warning', message: 'Skipped: connection tag does not match Convex connection tag' }] + }); + } + continue; + } + + const matchedTableNames = [...tablesByName.keys()] + .filter((name) => { + //Convex doesn't support user-defined schemas, so this is more of a forwards compatibility check for when multiple connections are supported + if (tablePattern.schema != this.connectionManager.schema) { + return false; + } + if (isConvexCheckpointTable(name)) { + return false; + } + if (tablePattern.isWildcard) { + return name.startsWith(tablePattern.tablePrefix); + } + return name == tablePattern.name; + }) + .sort(); + + if (tablePattern.isWildcard) { + patternResult.tables = matchedTableNames.map((tableName) => + createTableInfo({ + tablePattern, + connectionTag: this.connectionManager.connectionTag, + syncRules: sqlSyncRules, + tableName + }) + ); + } else { + const tableName = matchedTableNames[0] ?? tablePattern.name; + patternResult.table = createTableInfo({ + tablePattern, + connectionTag: this.connectionManager.connectionTag, + syncRules: sqlSyncRules, + tableName, + errors: + matchedTableNames.length == 0 + ? [{ level: 'warning', message: `Table ${tablePattern.schema}.${tablePattern.name} not found` }] + : [] + }); + } + } + + return result; + } + + //for convex we can calculate time-based lag, but not byte-based lag + async getReplicationLagBytes(options: ReplicationLagOptions): Promise { + return undefined; + } + + async createReplicationHead(callback: ReplicationHeadCallback): Promise { + const head = await this.connectionManager.client.getHeadCursor(); + const result = await callback(parseConvexLsn(head)); + await this.connectionManager.client.createWriteCheckpointMarker(); + return result; + } + + async getConnectionSchema(): Promise { + const schema = await this.connectionManager.client.getJsonSchemas(); + + return [ + { + name: this.connectionManager.schema, + tables: schema.tables + .filter((table) => !isConvexCheckpointTable(table.tableName)) + .map((table) => ({ + name: table.tableName, + columns: Object.entries({ + ...extractProperties(table.schema), + _id: { type: 'id' } + }) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([columnName, jsonSchemaProperty]) => { + const jsonType = readConvexFieldJsonType(jsonSchemaProperty); + const sqliteType = jsonSchemaToSQLiteType(jsonType); + + return { + name: columnName, + type: jsonSchemaProperty.type ?? 'unknown', + sqlite_type: sqliteType.typeFlags, + internal_type: jsonType, + pg_type: jsonType + }; + }) + })) + } + ]; + } + + async executeQuery(query: string, params: any[]): Promise { + return service_types.internal_routes.ExecuteSqlResponse.encode({ + results: { + columns: [], + rows: [] + }, + success: false, + error: 'SQL querying is not supported for Convex' + }); + } + + async shutdown(): Promise { + await this.connectionManager.end(); + } + + async [Symbol.asyncDispose]() { + await this.shutdown(); + } + + getParseSyncRulesOptions(): ParseSyncRulesOptions { + return { + defaultSchema: this.connectionManager.schema + }; + } +} + +function createTableInfo(options: { + tablePattern: sync_rules.TablePattern; + connectionTag: string; + syncRules: sync_rules.SqlSyncRules; + tableName?: string; + errors?: service_types.ReplicationError[]; +}) { + const tableName = + options.tableName ?? + (options.tablePattern.isWildcard ? options.tablePattern.tablePrefix : options.tablePattern.name); + const ref = { + connectionTag: options.connectionTag, + schema: options.tablePattern.schema, + name: tableName + } satisfies sync_rules.SourceTableRef; + + return { + schema: options.tablePattern.schema, + name: tableName, + pattern: options.tablePattern.isWildcard ? options.tablePattern.tablePattern : undefined, + replication_id: ['_id'], + data_queries: options.syncRules.tableSyncsData(ref), + parameter_queries: options.syncRules.tableSyncsParameters(ref), + errors: options.errors ?? [] + }; +} diff --git a/modules/module-convex/src/client/ConvexAPITypes.ts b/modules/module-convex/src/client/ConvexAPITypes.ts new file mode 100644 index 000000000..d24ab151b --- /dev/null +++ b/modules/module-convex/src/client/ConvexAPITypes.ts @@ -0,0 +1,132 @@ +import { schema } from '@powersync/lib-services-framework'; +import * as t from 'ts-codec'; + +export const ConvexRawDocument = t + .object({ + _id: t.string.optional(), + _table: t.string.optional(), + _deleted: t.boolean.optional() + }) + .and(t.record(t.any)); + +export type ConvexRawDocument = t.Encoded; + +// Placeholder to represent bigints +export const bigint = t.codec( + 'bigint', + (decoded: bigint) => decoded, + (encoded: bigint) => encoded +); + +// Placeholder which allows validating bigint payloads +export const bigintParser = t.createParser(bigint._tag, () => ({ + nodeType: 'bigint' +})); + +export const ConvexDocumentDelta = ConvexRawDocument.and( + t.object({ + _ts: bigint + }) +); + +export type ConvexDocumentDelta = t.Encoded; + +export const ConvexTableSchema = t.object({ + tableName: t.string, + // JSON schema object + schema: t.record(t.any) +}); + +export type ConvexTableSchema = t.Encoded; + +export const ConvexJsonSchemasResult = t.object({ + tables: t.array(ConvexTableSchema) +}); + +export type ConvexJsonSchemasResult = t.Encoded; + +export const ConvexListSnapshotResult = t.object({ + snapshot: bigint, + cursor: t.string.or(t.Null), + hasMore: t.boolean, + values: t.array(ConvexRawDocument) +}); + +export type ConvexListSnapshotResult = t.Encoded; + +const CHECK_CONVEX_RESPONSES_ENV = 'POWERSYNC_DEV_CHECK_CONVEX_RESPONSES'; + +// These validators optionally assert the API response shape in development. +export const ensureConvexListSnapshotResult = ensureResponseFormatValidator(ConvexListSnapshotResult); + +export const ConvexDocumentDeltasResult = t.object({ + cursor: bigint, + hasMore: t.boolean, + values: t.array(ConvexDocumentDelta) +}); + +export type ConvexDocumentDeltasResult = t.Encoded; + +export const ensureConvexDocumentDeltasResult = ensureResponseFormatValidator(ConvexDocumentDeltasResult); + +export interface ConvexListSnapshotOptions { + snapshot?: string; + cursor?: string; + tableName?: string; + signal?: AbortSignal; +} + +export interface ConvexDocumentDeltasOptions { + cursor?: string; + signal?: AbortSignal; +} + +export const RawJsonSchemaResponse = t.record( + t.object({ + type: t.string, + properties: t.record(t.any) + }) +); + +export type RawJsonSchemaResponse = t.Encoded; + +export const ensureRawJsonSchemaResponse = ensureResponseFormatValidator(RawJsonSchemaResponse); + +/** + * Optionally validates that Convex API response data matches the codec specification. + * + * These checks were originally added because earlier iterations of this client + * handled several Convex API route and field-name permutations, such as falling + * back between `has_more` and `hasMore`. That suggested there may have been + * response differences between Convex Cloud and self-hosted Convex deployments. + * + * The current response typings have been verified with cloud and self-hosted + * integration tests, so we do not need to pay this validation cost in + * production. The checks are still useful while developing against Convex API + * changes, so they are kept as an opt-in guardrail. + * + * Set POWERSYNC_DEV_CHECK_CONVEX_RESPONSES to enable this development safety net. + */ +export function ensureResponseFormatValidator( + codec: Codec +): (data: unknown) => t.Encoded { + if (!process.env[CHECK_CONVEX_RESPONSES_ENV]) { + return (data: unknown) => data as t.Encoded; + } + + const validator = schema.createTsCodecValidator(codec, { + parsers: [bigintParser], + allowAdditional: true + }); + + return (data: unknown) => { + const result = validator.validate(data); + if (!result.valid) { + // This does not result in leaking failed data, it only logs the keys which failed validation + throw new Error( + `Invalid data received. Got parsing errors when checking data format. Errors: ${result.errors.join(', ')}` + ); + } + return data as t.Encoded; + }; +} diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts new file mode 100644 index 000000000..1c22945bb --- /dev/null +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -0,0 +1,293 @@ +import { JSONBig } from '@powersync/service-jsonbig'; +// Using node-fetch in order to supply a custom agent +import fetch from 'node-fetch'; +import * as http from 'node:http'; +import * as https from 'node:https'; +import { setTimeout as delay } from 'timers/promises'; +import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; +import { NormalizedConvexConnectionConfig } from '../types/types.js'; +import { + ConvexDocumentDeltasOptions, + ConvexDocumentDeltasResult, + ConvexJsonSchemasResult, + ConvexListSnapshotOptions, + ConvexListSnapshotResult, + ensureConvexDocumentDeltasResult, + ensureConvexListSnapshotResult, + ensureRawJsonSchemaResponse +} from './ConvexAPITypes.js'; + +type GetRequestParams = { + path: string; + params?: Record; + signal?: AbortSignal; + extraHeaders?: Record; + includeJsonFormat?: boolean; +}; + +export class ConvexApiError extends Error { + readonly status?: number; + readonly retryable: boolean; + readonly body?: unknown; + + constructor(options: { message: string; status?: number; retryable: boolean; body?: unknown; cause?: unknown }) { + super(options.message, options.cause !== undefined ? { cause: options.cause } : undefined); + this.name = 'ConvexApiError'; + this.status = options.status; + this.retryable = options.retryable; + this.body = options.body; + } +} + +export class ConvexApiClient { + private readonly agent: http.Agent | https.Agent; + + constructor(private readonly config: NormalizedConvexConnectionConfig) { + this.agent = this.resolveAgent(); + } + + async getJsonSchemas(options?: { signal?: AbortSignal }): Promise { + const raw = await this.performTypedGetRequest( + { + path: '/api/json_schemas', + signal: options?.signal + }, + ensureRawJsonSchemaResponse + ); + // Convex returns this as a map of {[tableName]: JSONSchema} for tables + return { + tables: Object.entries(raw).map(([tableName, schema]) => ({ + tableName, + schema + })) + }; + } + + async listSnapshot(options: ConvexListSnapshotOptions): Promise { + return await this.performTypedGetRequest( + { + path: '/api/list_snapshot', + params: { + snapshot: options.snapshot, + cursor: options.cursor, + table_name: options.tableName + }, + signal: options.signal + }, + ensureConvexListSnapshotResult + ); + } + + async documentDeltas(options: ConvexDocumentDeltasOptions): Promise { + return await this.performTypedGetRequest( + { + path: '/api/document_deltas', + params: { + cursor: options.cursor + }, + signal: options.signal + }, + ensureConvexDocumentDeltasResult + ); + } + + async getGlobalSnapshotCursor(options?: { signal?: AbortSignal }): Promise { + const page = await this.listSnapshot({ + signal: options?.signal + }); + return page.snapshot.toString(); + } + + async getHeadCursor(options?: { signal?: AbortSignal }): Promise { + return this.getGlobalSnapshotCursor({ signal: options?.signal }); + } + + async createWriteCheckpointMarker(options?: { signal?: AbortSignal }): Promise { + await this.performRequest({ + method: 'POST', + url: new URL('/api/mutation', this.config.deployment_url), + body: { + path: `${CONVEX_CHECKPOINT_TABLE}:createCheckpoint`, + args: {}, + format: 'json' + }, + signal: options?.signal, + extraHeaders: { + 'Content-Type': 'application/json' + } + }); + } + + private async performGetRequest(options: GetRequestParams) { + const { path, params = {}, signal, extraHeaders, includeJsonFormat } = options; + + // `params` are mapped to url search params for GET requests + const url = new URL(path, this.config.deployment_url); + if (includeJsonFormat ?? true) { + url.searchParams.set('format', 'json'); + } + + for (const [key, value] of Object.entries(params)) { + if (value == null) { + continue; + } + url.searchParams.set(key, `${value}`); + } + + return this.performRequest({ + method: 'GET', + url, + extraHeaders, + signal + }); + } + + private async performTypedGetRequest( + options: GetRequestParams, + validator: (data: unknown) => ResponseType + ) { + const rawResponse = await this.performGetRequest(options); + try { + return validator(rawResponse); + } catch (ex) { + throw new ConvexApiError({ + message: `Failed to validate Convex API request response format`, + retryable: false, + cause: ex + }); + } + } + + /** + * Performs a request which expects a JSON response + */ + private async performRequest(options: { + method: 'GET' | 'POST'; + url: URL; + body?: unknown; + signal?: AbortSignal; + extraHeaders?: Record; + }): Promise> { + const { method, url, body, extraHeaders = {}, signal: requestSignal } = options; + + const timeout = new AbortController(); + const timeoutPromise = delay(this.config.request_timeout_ms, undefined, { + signal: timeout.signal + }).then(() => { + timeout.abort(new Error(`Convex API request timed out after ${this.config.request_timeout_ms}ms`)); + }); + + const signals = [timeout.signal]; + if (requestSignal) { + signals.push(requestSignal); + } + + const signal = AbortSignal.any(signals); + + try { + const response = await fetch(url.toString(), { + method, + headers: { + Authorization: `Convex ${this.config.deploy_key}`, + Accept: 'application/json', + ...extraHeaders + }, + body: body == null ? undefined : JSON.stringify(options.body), + signal, + agent: this.agent + }); + + const text = await response.text(); + let json: unknown; + try { + json = JSONBig.parse(text); + } catch (ex) { + // The response could not be json, this should only happen for !ok responses. + } + + if (!response.ok) { + const retryable = response.status == 429 || response.status >= 500; + throw new ConvexApiError({ + message: `Convex API request failed (${response.status}) for ${url.pathname}`, + status: response.status, + retryable, + body: json, + cause: new Error(text) + }); + } + + if (!isRecord(json)) { + throw new ConvexApiError({ + message: `Convex API response was not an object for ${url.pathname}`, + retryable: false, + body: json + }); + } + + return json; + } catch (error) { + if (error instanceof ConvexApiError) { + throw error; + } + + const message = error instanceof Error ? error.message : `${error}`; + const retryable = + message.includes('timed out') || + message.includes('fetch failed') || + message.includes('ECONNREFUSED') || + message.includes('ECONNRESET') || + message.includes('ENOTFOUND') || + message.includes('EAI_AGAIN') || + message.includes('ETIMEDOUT') || + error instanceof DOMException; + throw new ConvexApiError({ + message: `Convex API request failed for ${url.pathname}: ${message}`, + retryable, + cause: error + }); + } finally { + timeout.abort(); + await timeoutPromise.catch(() => { + // no-op + }); + } + } + + private resolveAgent(): http.Agent | https.Agent { + const deploymentUrl = new URL(this.config.deployment_url); + const options: http.AgentOptions = {}; + + if (this.config.lookup) { + options.lookup = this.config.lookup; + } + + switch (deploymentUrl.protocol) { + case 'http:': + return new http.Agent(options); + case 'https:': + return new https.Agent(options); + } + + throw new ConvexApiError({ + message: `Convex deployment_url must use http or https, got ${JSON.stringify(deploymentUrl.protocol)}`, + retryable: false + }); + } +} + +export function isCursorExpiredError(error: unknown): boolean { + if (!(error instanceof ConvexApiError)) { + return false; + } + + const asString = `${error.message} ${JSON.stringify(error.body ?? {})}`.toLowerCase(); + + return ( + (asString.includes('cursor') && (asString.includes('expired') || asString.includes('invalid'))) || + (asString.includes('snapshot') && asString.includes('expired')) + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value == 'object' && value != null && !Array.isArray(value); +} diff --git a/modules/module-convex/src/common/ConvexCheckpoints.ts b/modules/module-convex/src/common/ConvexCheckpoints.ts new file mode 100644 index 000000000..d797bceca --- /dev/null +++ b/modules/module-convex/src/common/ConvexCheckpoints.ts @@ -0,0 +1,13 @@ +/** + * Table name used by PowerSync to write checkpoint markers into Convex. + * + * Convex rejects table names starting with `_`, so we use a plain prefix. + */ +export const CONVEX_CHECKPOINT_TABLE = 'powersync_checkpoints'; + +/** + * Returns true if `tableName` is the PowerSync checkpoint marker table. + */ +export function isConvexCheckpointTable(tableName: string): boolean { + return tableName === CONVEX_CHECKPOINT_TABLE; +} diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts new file mode 100644 index 000000000..cd7810ef7 --- /dev/null +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -0,0 +1,48 @@ +// Convex replication cursors are fixed-width decimal timestamps. +const CONVEX_TIMESTAMP_DIGITS = 19; + +export const ZERO_LSN = '0'; + +/** + * Converts the decimal timestamp LSN to a JS Date. + */ +export function lsnCursorToDate(cursor: bigint): Date { + return new Date(Number(cursor / 1_000_000n)); +} + +/** + * Parse and validate an incoming LSN. + * Sources for input are usually the `list-snapshot` response - which returns a bigint + * or the current stored LSN - which is stored as a string. + */ +export function parseConvexLsn(input: string | bigint): string { + return toConvexLsn(input); +} + +export function toConvexLsn(input: string | bigint): string { + const asString = `${input}`; + assertValidConvexLsn(asString); + return asString; +} + +function assertValidConvexLsn(cursor: string) { + if (cursor.length == 0) { + throw new Error('Convex cursor cannot be empty'); + } + + if (!/^[0-9]+$/.test(cursor)) { + throw new Error(`Convex cursor is not a valid numeric timestamp: ${cursor}`); + } + + if (cursor == ZERO_LSN) { + return; + } + + if (cursor.startsWith('0')) { + throw new Error(`Convex cursor is not a canonical numeric timestamp: ${cursor}`); + } + + if (cursor.length != CONVEX_TIMESTAMP_DIGITS) { + throw new Error(`Convex cursor is not a 19-digit numeric timestamp: ${cursor}`); + } +} diff --git a/modules/module-convex/src/common/convex-to-sqlite.ts b/modules/module-convex/src/common/convex-to-sqlite.ts new file mode 100644 index 000000000..eb109b746 --- /dev/null +++ b/modules/module-convex/src/common/convex-to-sqlite.ts @@ -0,0 +1,127 @@ +import { JSONBig } from '@powersync/service-jsonbig'; +import { + DatabaseInputRow, + DatabaseInputValue, + ExpressionType, + SqliteInputRow, + toSyncRulesRow +} from '@powersync/service-sync-rules'; +import { ConvexRawDocument } from '../client/ConvexAPITypes.js'; + +export enum SupportedJSONSchemaPropertyType { + ID = 'id', + STRING = 'string', + BYTES = 'bytes', + ARRAY = 'array', + OBJECT = 'object', + NULL = 'null', + INT64 = 'int64', + FLOAT64 = 'float64', + BOOLEAN = 'boolean', + UNKNOWN = 'unknown' +} + +export const CONVEX_TO_SQLITE_TYPE_MAP: Record = { + [SupportedJSONSchemaPropertyType.ID]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.STRING]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.BYTES]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.ARRAY]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.OBJECT]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.NULL]: ExpressionType.NONE, + [SupportedJSONSchemaPropertyType.INT64]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.FLOAT64]: ExpressionType.REAL, + [SupportedJSONSchemaPropertyType.BOOLEAN]: ExpressionType.INTEGER, + [SupportedJSONSchemaPropertyType.UNKNOWN]: ExpressionType.TEXT +} as const; + +const INTERNAL_KEYS = new Set(['_table', '_deleted', '_ts', '_component']); + +export function jsonSchemaToSQLiteType(jsonType: SupportedJSONSchemaPropertyType): ExpressionType { + return CONVEX_TO_SQLITE_TYPE_MAP[jsonType]; +} + +export function toSqliteInputRow(change: ConvexRawDocument): SqliteInputRow { + const row: DatabaseInputRow = {}; + + for (const [key, value] of Object.entries(change)) { + if (INTERNAL_KEYS.has(key)) { + continue; + } + + row[key] = toDatabaseValue(value); + } + + return toSyncRulesRow(row); +} + +export function extractProperties(schema: Record): Record { + // Convex returns each table schema as a standard JSON Schema object. + // For example: { "type": "object", "properties": { "name": { "type": "string" } } }. + // The table columns live under `properties`; top-level keys such as `type` + // describe the schema itself and must not be treated as columns. + return isRecord(schema.properties) ? schema.properties : {}; +} + +/** + * Converts a Convex row value to a DatabaseInputValue. + * This intentionally ignores Convex json_schemas metadata so the same source column keeps a stable + * representation regardless of whether Convex reported that field in json_schemas. + */ +function toDatabaseValue(value: unknown): DatabaseInputValue { + if (value == null) { + return null; + } else if ( + typeof value == 'string' || + typeof value == 'boolean' || + typeof value == 'number' || + typeof value == 'bigint' + ) { + return value; + } else if (Array.isArray(value) || typeof value == 'object') { + return JSONBig.stringify(value); + } else { + return null; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value == 'object' && value != null && !Array.isArray(value); +} + +export function readConvexFieldJsonType(jsonSchemaProperty: unknown): SupportedJSONSchemaPropertyType { + if (!isRecord(jsonSchemaProperty)) { + // Invalid schema property entry received + return SupportedJSONSchemaPropertyType.UNKNOWN; + } + + const description = jsonSchemaProperty['$description']; + const type = jsonSchemaProperty['type']; + /** + * An Int64 example + * "points": {"$description": "int64 represented as base10 string", "type": "string"}, + */ + if (description == 'int64 represented as base10 string' && type == 'string') { + return SupportedJSONSchemaPropertyType.INT64; + } else if (description == 'base64 bytes' && type == 'string') { + /** + * Buffer example + * "attachment_data": {"$description": "base64 bytes", "type": "string"}, + */ + return SupportedJSONSchemaPropertyType.BYTES; + } else if (type == 'string' || type == 'id') { + return SupportedJSONSchemaPropertyType.STRING; + } else if (type == 'boolean') { + return SupportedJSONSchemaPropertyType.BOOLEAN; + } else if (type == 'number') { + // number and float64 seem to both be represented as 'number' in the reported JSON schema + return SupportedJSONSchemaPropertyType.FLOAT64; + } else if (type == 'array') { + return SupportedJSONSchemaPropertyType.ARRAY; + } else if (type == 'object') { + return SupportedJSONSchemaPropertyType.OBJECT; + } else if (type == 'null') { + return SupportedJSONSchemaPropertyType.NULL; + } + + return SupportedJSONSchemaPropertyType.UNKNOWN; +} diff --git a/modules/module-convex/src/index.ts b/modules/module-convex/src/index.ts new file mode 100644 index 000000000..90ee6204f --- /dev/null +++ b/modules/module-convex/src/index.ts @@ -0,0 +1,4 @@ +export * from './api/ConvexRouteAPIAdapter.js'; +export * from './module/ConvexModule.js'; +export * from './replication/replication-index.js'; +export * from './types/types.js'; diff --git a/modules/module-convex/src/module/ConvexModule.ts b/modules/module-convex/src/module/ConvexModule.ts new file mode 100644 index 000000000..67e606726 --- /dev/null +++ b/modules/module-convex/src/module/ConvexModule.ts @@ -0,0 +1,77 @@ +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; +import { + api, + ConfigurationFileSyncRulesProvider, + ConnectionTestResult, + replication, + system, + TearDownOptions +} from '@powersync/service-core'; +import { ConvexRouteAPIAdapter } from '../api/ConvexRouteAPIAdapter.js'; +import { checkSourceConfiguration } from '../replication/check-source-configuration.js'; +import { ConvexConnectionManagerFactory } from '../replication/ConvexConnectionManagerFactory.js'; +import { ConvexErrorRateLimiter } from '../replication/ConvexErrorRateLimiter.js'; +import { ConvexReplicator } from '../replication/ConvexReplicator.js'; +import * as types from '../types/types.js'; + +export class ConvexModule extends replication.ReplicationModule { + constructor() { + super({ + name: 'Convex', + type: types.CONVEX_CONNECTION_TYPE, + configSchema: types.ConvexConnectionConfig + }); + } + + async onInitialized(context: system.ServiceContextContainer): Promise {} + + protected createRouteAPIAdapter(): api.RouteAPI { + return new ConvexRouteAPIAdapter(this.resolveConfig(this.decodedConfig!)); + } + + protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator { + const normalizedConfig = this.resolveConfig(this.decodedConfig!); + const syncRuleProvider = new ConfigurationFileSyncRulesProvider(context.configuration.sync_rules); + const connectionFactory = new ConvexConnectionManagerFactory(normalizedConfig); + + return new ConvexReplicator({ + id: this.getDefaultId(normalizedConfig.deployment_url), + syncRuleProvider, + storageEngine: context.storageEngine, + metricsEngine: context.metricsEngine, + connectionFactory, + rateLimiter: new ConvexErrorRateLimiter() + }); + } + + private resolveConfig(config: types.ConvexConnectionConfig): types.ResolvedConvexConnectionConfig { + return types.resolveConvexConnectionConfig(config); + } + + async teardown(options: TearDownOptions): Promise { + // No source-side teardown required. + } + + async testConnection(config: types.ConvexConnectionConfig) { + this.decodeConfig(config); + const normalizedConfig = this.resolveConfig(this.decodedConfig!); + return await ConvexModule.testConnection(normalizedConfig); + } + + static async testConnection(normalizedConfig: types.ResolvedConvexConnectionConfig): Promise { + const { connected, errors } = await checkSourceConfiguration(normalizedConfig); + /** + * The mutation check can report configuration errors even after a successful + * schema fetch, so treat either disconnected or errored states as failures. + * */ + if (!connected || errors.length > 0) { + throw new ServiceError({ + code: ErrorCode.PSYNC_R0001, + description: errors.join('\n') + }); + } + return { + connectionDescription: normalizedConfig.deployment_url + }; + } +} diff --git a/modules/module-convex/src/replication/ConvexConnectionManager.ts b/modules/module-convex/src/replication/ConvexConnectionManager.ts new file mode 100644 index 000000000..369b83d32 --- /dev/null +++ b/modules/module-convex/src/replication/ConvexConnectionManager.ts @@ -0,0 +1,28 @@ +import { BaseObserver } from '@powersync/lib-services-framework'; +import { DEFAULT_TAG } from '@powersync/service-sync-rules'; +import { ConvexApiClient } from '../client/ConvexApiClient.js'; +import { NormalizedConvexConnectionConfig } from '../types/types.js'; + +export interface ConvexConnectionManagerListener { + onEnded?: () => void; +} + +export class ConvexConnectionManager extends BaseObserver { + readonly client: ConvexApiClient; + readonly schema = 'convex'; + readonly connectionTag: string; + readonly connectionId: string; + + constructor(public readonly config: NormalizedConvexConnectionConfig) { + super(); + this.client = new ConvexApiClient(config); + this.connectionTag = config.tag ?? DEFAULT_TAG; + this.connectionId = config.id ?? 'default'; + } + + async end(): Promise { + this.iterateListeners((listener) => { + listener.onEnded?.(); + }); + } +} diff --git a/modules/module-convex/src/replication/ConvexConnectionManagerFactory.ts b/modules/module-convex/src/replication/ConvexConnectionManagerFactory.ts new file mode 100644 index 000000000..aef23df3e --- /dev/null +++ b/modules/module-convex/src/replication/ConvexConnectionManagerFactory.ts @@ -0,0 +1,28 @@ +import { logger } from '@powersync/lib-services-framework'; +import { ResolvedConvexConnectionConfig } from '../types/types.js'; +import { ConvexConnectionManager } from './ConvexConnectionManager.js'; + +export class ConvexConnectionManagerFactory { + private readonly connectionManagers = new Set(); + + constructor(public readonly connectionConfig: ResolvedConvexConnectionConfig) {} + + create() { + const manager = new ConvexConnectionManager(this.connectionConfig); + manager.registerListener({ + onEnded: () => { + this.connectionManagers.delete(manager); + } + }); + this.connectionManagers.add(manager); + return manager; + } + + async shutdown() { + logger.info('Shutting down Convex connection managers...'); + for (const manager of [...this.connectionManagers]) { + await manager.end(); + } + logger.info('Convex connection managers shutdown completed.'); + } +} diff --git a/modules/module-convex/src/replication/ConvexErrorRateLimiter.ts b/modules/module-convex/src/replication/ConvexErrorRateLimiter.ts new file mode 100644 index 000000000..110510170 --- /dev/null +++ b/modules/module-convex/src/replication/ConvexErrorRateLimiter.ts @@ -0,0 +1,47 @@ +import { ErrorRateLimiter } from '@powersync/service-core'; +import { setTimeout } from 'timers/promises'; +import { ConvexApiError } from '../client/ConvexApiClient.js'; + +export class ConvexErrorRateLimiter implements ErrorRateLimiter { + private nextAllowed = Date.now(); + + async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise { + const delay = Math.max(0, this.nextAllowed - Date.now()); + this.setDelay(500); + await setTimeout(delay, undefined, { signal: options?.signal }); + } + + mayPing(): boolean { + return Date.now() >= this.nextAllowed; + } + + reportError(error: any): void { + if (error instanceof ConvexApiError) { + if (error.status == 401 || error.status == 403) { + this.setDelay(120_000); + return; + } + if (error.status == 429) { + this.setDelay(15_000); + return; + } + if (error.retryable) { + this.setDelay(10_000); + return; + } + this.setDelay(45_000); + return; + } + + const message = (error?.message as string | undefined) ?? ''; + if (message.includes('ENOTFOUND') || message.includes('ECONNREFUSED')) { + this.setDelay(120_000); + } else { + this.setDelay(20_000); + } + } + + private setDelay(delay: number) { + this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay); + } +} diff --git a/modules/module-convex/src/replication/ConvexReplicationJob.ts b/modules/module-convex/src/replication/ConvexReplicationJob.ts new file mode 100644 index 000000000..e446eb349 --- /dev/null +++ b/modules/module-convex/src/replication/ConvexReplicationJob.ts @@ -0,0 +1,75 @@ +import { container } from '@powersync/lib-services-framework'; +import { replication } from '@powersync/service-core'; +import { ConvexConnectionManagerFactory } from './ConvexConnectionManagerFactory.js'; +import { ConvexCursorExpiredError, ConvexStream } from './ConvexStream.js'; + +export interface ConvexReplicationJobOptions extends replication.AbstractReplicationJobOptions { + connectionFactory: ConvexConnectionManagerFactory; +} + +export class ConvexReplicationJob extends replication.AbstractReplicationJob { + private readonly connectionFactory: ConvexConnectionManagerFactory; + private lastStream: ConvexStream | null = null; + + constructor(options: ConvexReplicationJobOptions) { + super(options); + this.connectionFactory = options.connectionFactory; + this.logger = options.storage.logger; + } + + async keepAlive() { + // streaming API is polled continuously; no dedicated keepalive operation required here + } + + async replicate() { + try { + await this.replicateOnce(); + } catch (error) { + if (!this.isStopped) { + this.logger.error('Replication error', error); + if (error?.cause != null) { + this.logger.error('cause', error.cause); + } + + container.reporter.captureException(error, { + metadata: {} + }); + + this.rateLimiter.reportError(error); + } + + if (error instanceof ConvexCursorExpiredError) { + await this.options.storage.factory.restartReplication(this.storage.group_id); + } + } finally { + this.abortController.abort(); + } + } + + async replicateOnce() { + const manager = this.connectionFactory.create(); + try { + await this.rateLimiter.waitUntilAllowed({ signal: this.abortController.signal }); + if (this.isStopped) { + return; + } + + const stream = new ConvexStream({ + abortSignal: this.abortController.signal, + connections: manager, + logger: this.logger, + metrics: this.options.metrics, + storage: this.options.storage + }); + + this.lastStream = stream; + await stream.replicate(); + } finally { + await manager.end(); + } + } + + getReplicationLagMillis(): number | undefined { + return this.lastStream?.getReplicationLagMillis(); + } +} diff --git a/modules/module-convex/src/replication/ConvexReplicator.ts b/modules/module-convex/src/replication/ConvexReplicator.ts new file mode 100644 index 000000000..b2db9135f --- /dev/null +++ b/modules/module-convex/src/replication/ConvexReplicator.ts @@ -0,0 +1,41 @@ +import { replication, storage } from '@powersync/service-core'; +import { ConvexModule } from '../module/ConvexModule.js'; +import { ConvexConnectionManagerFactory } from './ConvexConnectionManagerFactory.js'; +import { ConvexReplicationJob } from './ConvexReplicationJob.js'; + +export interface ConvexReplicatorOptions extends replication.AbstractReplicatorOptions { + connectionFactory: ConvexConnectionManagerFactory; +} + +export class ConvexReplicator extends replication.AbstractReplicator { + private readonly connectionFactory: ConvexConnectionManagerFactory; + + constructor(options: ConvexReplicatorOptions) { + super(options); + this.connectionFactory = options.connectionFactory; + } + + createJob(options: replication.CreateJobOptions): ConvexReplicationJob { + return new ConvexReplicationJob({ + id: this.createJobId(options.storage.group_id), + storage: options.storage, + metrics: this.metrics, + lock: options.lock, + connectionFactory: this.connectionFactory, + rateLimiter: this.rateLimiter + }); + } + + async cleanUp(syncRulesStorage: storage.SyncRulesBucketStorage): Promise { + // No source-side cleanup needed for Convex. + } + + async stop(): Promise { + await super.stop(); + await this.connectionFactory.shutdown(); + } + + async testConnection() { + return await ConvexModule.testConnection(this.connectionFactory.connectionConfig); + } +} diff --git a/modules/module-convex/src/replication/ConvexSnapshotProgressCursor.ts b/modules/module-convex/src/replication/ConvexSnapshotProgressCursor.ts new file mode 100644 index 000000000..9f6483667 --- /dev/null +++ b/modules/module-convex/src/replication/ConvexSnapshotProgressCursor.ts @@ -0,0 +1,40 @@ +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import * as bson from 'bson'; +import * as t from 'ts-codec'; + +export const ConvexSnapshotProgressCursor = t.object({ + cursor: t.string.or(t.Null), + finished: t.boolean +}); + +export type ConvexSnapshotProgressCursor = t.Encoded; + +export const DEFAULT_CONVEX_SNAPSHOT_CURSOR: ConvexSnapshotProgressCursor = { + cursor: null, + finished: false +}; + +export const BinaryConvexSnapshotProgressCursor = t.codec( + 'ConvexSnapshotProgressCursor', + (decoded: ConvexSnapshotProgressCursor) => bson.serialize(ConvexSnapshotProgressCursor.encode(decoded)), + (encoded) => ConvexSnapshotProgressCursor.decode(bson.deserialize(encoded) as any) +); + +/** + * Decodes an (optional) encoded ConvexSnapshotProgressCursor. + * @default {DEFAULT_CONVEX_SNAPSHOT_CURSOR} + * @throws {ReplicationAssertionError} if the value could not be decoded + */ +export function decodeSnapshotProgressCursor(value: Uint8Array | null | undefined): ConvexSnapshotProgressCursor { + if (value == null) { + return DEFAULT_CONVEX_SNAPSHOT_CURSOR; + } + + try { + return BinaryConvexSnapshotProgressCursor.decode(value); + } catch (error) { + throw new ReplicationAssertionError( + `Convex snapshot progress cursor is not valid BSON: ${error instanceof Error ? error.message : `${error}`}` + ); + } +} diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts new file mode 100644 index 000000000..a33b448b7 --- /dev/null +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -0,0 +1,703 @@ +import { + container, + DatabaseConnectionError, + logger as defaultLogger, + ErrorCode, + Logger, + ReplicationAbortedError, + ReplicationAssertionError +} from '@powersync/lib-services-framework'; +import { + MetricsEngine, + RelationCache, + ReplicationLagTracker, + SaveOperationTag, + SourceEntityDescriptor, + SourceTable, + storage +} from '@powersync/service-core'; +import { HydratedSyncConfig, TablePattern } from '@powersync/service-sync-rules'; +import { ReplicationMetric } from '@powersync/service-types'; +import { setTimeout as delay } from 'timers/promises'; +import { ConvexListSnapshotResult, ConvexRawDocument } from '../client/ConvexAPITypes.js'; +import { isCursorExpiredError } from '../client/ConvexApiClient.js'; +import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; +import { lsnCursorToDate, parseConvexLsn, ZERO_LSN } from '../common/ConvexLSN.js'; +import { toSqliteInputRow } from '../common/convex-to-sqlite.js'; +import { ConvexConnectionManager } from './ConvexConnectionManager.js'; +import { BinaryConvexSnapshotProgressCursor, decodeSnapshotProgressCursor } from './ConvexSnapshotProgressCursor.js'; + +export interface ConvexStreamOptions { + connections: ConvexConnectionManager; + storage: storage.SyncRulesBucketStorage; + metrics: MetricsEngine; + abortSignal: AbortSignal; + logger?: Logger; +} + +export class ConvexCursorExpiredError extends DatabaseConnectionError { + constructor(message: string, cause: unknown) { + super(ErrorCode.PSYNC_S1500, message, cause); + } +} + +export class ConvexStream { + private readonly storage: storage.SyncRulesBucketStorage; + private readonly metrics: MetricsEngine; + private readonly syncConfig: HydratedSyncConfig; + private readonly logger: Logger; + + private readonly relationCache = new RelationCache(getCacheIdentifier); + private replicationLag = new ReplicationLagTracker(); + + private lastKeepaliveAt = 0; + private lastTouchedAt = performance.now(); + + private initialSnapshotPromise: Promise | null = null; + + constructor(private readonly options: ConvexStreamOptions) { + this.storage = options.storage; + this.metrics = options.metrics; + this.syncConfig = options.storage.getParsedSyncRules({ defaultSchema: options.connections.schema }); + this.logger = options.logger ?? defaultLogger; + } + + get isStartingReplication() { + return this.replicationLag.isStartingReplication; + } + + private get connections() { + return this.options.connections; + } + + private get abortSignal() { + return this.options.abortSignal; + } + + private get defaultSchema() { + return this.connections.schema; + } + + private get pollingIntervalMs() { + return this.connections.config.polling_interval_ms; + } + + get stopped() { + return this.abortSignal.aborted; + } + + get connectionId() { + const { connectionId } = this.connections; + // Default to 1 if not set + if (!connectionId) { + return 1; + } + /** + * This is often `"default"` (string) which will parse to `NaN` + */ + const parsed = Number.parseInt(connectionId); + if (isNaN(parsed)) { + return 1; + } + return parsed; + } + + async replicate() { + try { + this.initialSnapshotPromise = this.initReplication(); + // This pattern/member is used for tests + await this.initialSnapshotPromise; + + await this.streamChanges(); + } catch (error) { + await this.storage.reportError(error); + throw error; + } + } + + /** + * After calling replicate(), call this to wait for the initial snapshot to complete. + * + * For tests only. + */ + async waitForInitialSnapshot() { + if (this.initialSnapshotPromise == null) { + throw new ReplicationAssertionError(`Initial snapshot not started yet`); + } + return this.initialSnapshotPromise; + } + + async initReplication() { + const status = await this.initSlot(); + if (!status.needsInitialSync) { + return; + } + + if (status.snapshotLsn == null) { + await this.storage.clear({ signal: this.abortSignal }); + } + + const { lastOpId } = await this.initialReplication(status.snapshotLsn); + if (lastOpId != null) { + await this.storage.populatePersistentChecksumCache({ + signal: this.abortSignal, + maxOpId: lastOpId + }); + } + } + + async streamChanges() { + await using batch = await this.storage.createWriter({ + logger: this.logger, + zeroLSN: ZERO_LSN, + defaultSchema: this.defaultSchema, + // Convex document_deltas include the full document state after each mutation, + // so storage does not need to keep current row data to apply partial updates. + storeCurrentData: false, + skipExistingRows: false + }); + + let resumeFromLsn = batch.resumeFromLsn; + if (resumeFromLsn == null) { + throw new ReplicationAssertionError(`No LSN found to resume replication from.`); + } + + // Resolve source tables up-front to warm table metadata and sync-config matching. + await this.resolveAllSourceTables(batch); + + let cursor: string = resumeFromLsn; + let lastTransactionTimestamp: bigint | null = null; + + while (!this.abortSignal.aborted) { + const page = await this.connections.client + .documentDeltas({ + cursor: cursor, + signal: this.abortSignal + }) + .catch((error) => { + if (isCursorExpiredError(error)) { + throw new ConvexCursorExpiredError('Convex cursor expired; initial replication restart required', error); + } + throw error; + }); + + // We receive the cursor as a bigint, but we track it as a string + const nextCursor = page.cursor.toString(); + const pageLsn = parseConvexLsn(nextCursor); + + let changesInPage = 0; + const transactionTimestampsInPage = new Set(); + let sawCheckpointMarker = false; + let didMarkOldestUncommitedChange = false; + + /** + * Convex returns document_deltas in mutation order by _ts (corresponding to mutation/transaction). + * The row order inside each transaction is out-of-order. + * It looks like Convex squashes multiple mutations on rows before storing deltas. + * We currently don't sort values by their `_creationTime` value. + */ + for (const change of page.values) { + if (this.abortSignal.aborted) { + throw new ReplicationAbortedError('Replication interrupted'); + } + + const tableName = readTableName(change); + if (tableName == null) { + continue; + } + + if (isConvexCheckpointTable(tableName)) { + sawCheckpointMarker = true; + continue; + } + + const transactionTimestamp = change._ts; + if (lastTransactionTimestamp != null && transactionTimestamp < lastTransactionTimestamp) { + throw new ReplicationAssertionError( + `Convex document_deltas returned out-of-order _ts values: ${transactionTimestamp} after ${lastTransactionTimestamp}` + ); + } + lastTransactionTimestamp = transactionTimestamp; + + const tables = await this.getOrResolveTables(batch, tableName, pageLsn); + if (tables.length == 0) { + continue; + } + + /** + * This tracks the begining of a new transaction which is not yet commited. + * This uses the current page's cursor as the timestamp since this is the closes timestamp + * to the mutation. + * We should only track the first op for a new transaction. + * Note that the document-deltas aren't filtered, so we only + * mark the start after this point - which means we will have an uncommited change. + */ + if (!didMarkOldestUncommitedChange) { + this.replicationLag.trackUncommittedChange(lsnCursorToDate(page.cursor)); + didMarkOldestUncommitedChange = true; + } + + let wroteChange = false; + for (const table of tables) { + if (!table.syncAny) { + continue; + } + const changed = await this.writeChange(batch, table, change); + wroteChange = wroteChange || changed; + } + + if (wroteChange) { + // Convex assigns one _ts commit timestamp to every write in a mutation. + // document_deltas may return multiple mutations in one page, so transaction + // metrics are counted by distinct _ts values, not by delta pages. + changesInPage += 1; + transactionTimestampsInPage.add(transactionTimestamp.toString()); + } + } + + if (changesInPage > 0) { + /** + * It looks like the document-deltas api won't split transactions between pages, + * That means it should be safe to commit after each page. + * Each page could contain many smaller transactions - that should also be fine. + */ + const { checkpointBlocked } = await batch.commit(pageLsn, { + createEmptyCheckpoints: false, + oldestUncommittedChange: this.replicationLag.oldestUncommittedChange + }); + + this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(transactionTimestampsInPage.size); + if (!checkpointBlocked) { + this.replicationLag.markCommitted(); + } + } else if (sawCheckpointMarker) { + /** + * This is only reached if the checkpoint marker was the only change observed in a page. + */ + const { checkpointBlocked } = await batch.keepalive(pageLsn); + if (!checkpointBlocked) { + this.replicationLag.clearUncommittedChange(); + } + this.lastKeepaliveAt = Date.now(); + } else if (nextCursor == cursor && Date.now() - this.lastKeepaliveAt > 60_000) { + const { checkpointBlocked } = await batch.keepalive(pageLsn); + if (!checkpointBlocked) { + this.replicationLag.clearUncommittedChange(); + } + this.replicationLag.markStarted(); + this.lastKeepaliveAt = Date.now(); + } + + cursor = nextCursor; + + if (!page.hasMore) { + await delay(this.pollingIntervalMs, undefined, { signal: this.abortSignal }).catch((error) => { + if (this.abortSignal.aborted) { + return; + } + throw error; + }); + } + + this.touch(); + } + } + + getReplicationLagMillis(): number | undefined { + return this.replicationLag.getLagMillis(); + } + + private async initSlot(): Promise<{ needsInitialSync: boolean; snapshotLsn: string | null }> { + const status = await this.storage.getStatus(); + if (status.snapshot_done && status.checkpoint_lsn) { + this.logger.info('Initial replication already done'); + return { + needsInitialSync: false, + snapshotLsn: null + }; + } + + return { + needsInitialSync: true, + snapshotLsn: status.snapshot_lsn + }; + } + + private async initialReplication(snapshotLsn: string | null) { + await using batch = await this.storage.createWriter({ + logger: this.logger, + zeroLSN: ZERO_LSN, + defaultSchema: this.defaultSchema, + // Convex snapshots emit complete documents, so no current row state is needed. + storeCurrentData: false, + skipExistingRows: true + }); + + const snapshotCursor = await this.resolveSnapshotBoundary(snapshotLsn); + const snapshotLsnValue = parseConvexLsn(snapshotCursor); + await batch.setResumeLsn(snapshotLsnValue); + + const sourceTables = await this.resolveAllSourceTables(batch); + + for (const sourceTable of sourceTables) { + if (sourceTable.snapshotComplete) { + this.logger.info(`Skipping table [${sourceTable.qualifiedName}] - snapshot already done.`); + continue; + } + + const tableWithProgress = + sourceTable.snapshotStatus == null + ? await batch.updateTableProgress(sourceTable, { + totalEstimatedCount: -1, + replicatedCount: 0, + lastKey: null + }) + : sourceTable; + this.relationCache.update(tableWithProgress); + + await this.snapshotTable(batch, tableWithProgress, snapshotCursor); + } + + await batch.markAllSnapshotDone(snapshotLsnValue); + + await batch.commit(snapshotLsnValue); + + this.logger.info(`Snapshot done. Need to replicate from ${snapshotLsnValue} for consistency.`); + + return { + lastOpId: batch.last_flushed_op + }; + } + + private async snapshotTable( + batch: storage.BucketStorageBatch, + table: SourceTable, + snapshotCursor: string + ): Promise<{ table: SourceTable }> { + const snapshotProgress = decodeSnapshotProgressCursor(table.snapshotStatus?.lastKey); + let pageCursor = snapshotProgress.cursor; + let replicatedCount = table.snapshotStatus?.replicatedCount ?? 0; + let latestTable = table; + + if (snapshotProgress.finished) { + this.logger.info(`Finishing table snapshot from persisted progress for [${table.qualifiedName}]`); + } else if (pageCursor != null) { + this.logger.info(`Resuming table snapshot from persisted cursor for [${table.qualifiedName}]`); + } else { + this.logger.info(`Starting table snapshot from first page for [${table.qualifiedName}]`); + } + + if (this.abortSignal.aborted) { + throw new ReplicationAbortedError('Initial replication interrupted'); + } + + if (snapshotProgress.finished) { + return { + table: await this.markSnapshotDone(batch, latestTable, snapshotCursor) + }; + } + + while (!this.abortSignal.aborted) { + const page: ConvexListSnapshotResult = await this.connections.client + .listSnapshot({ + tableName: table.name, + snapshot: snapshotCursor, + cursor: pageCursor ?? undefined, + signal: this.abortSignal + }) + .catch((error) => { + if (isCursorExpiredError(error)) { + throw new ConvexCursorExpiredError('Convex snapshot cursor expired; restart required', error); + } + throw error; + }); + + if (snapshotCursor != page.snapshot.toString()) { + throw new ReplicationAssertionError( + `Convex snapshot cursor changed while snapshotting ${table.qualifiedName}: ${snapshotCursor} -> ${page.snapshot}` + ); + } + + for (const rawDocument of page.values) { + if (rawDocument._deleted) { + continue; + } + + const replicaId = rawDocument._id; + if (replicaId == null) { + this.logger.warn(`Skipping Convex document without _id on table ${table.qualifiedName}`); + continue; + } + + const row = this.toSqliteRow(rawDocument); + await batch.save({ + tag: SaveOperationTag.INSERT, + sourceTable: latestTable, + before: undefined, + beforeReplicaId: undefined, + after: row, + afterReplicaId: replicaId + }); + replicatedCount += 1; + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); + } + + await batch.flush(); + + pageCursor = page.cursor; + latestTable = await batch.updateTableProgress(latestTable, { + replicatedCount, + totalEstimatedCount: -1, + lastKey: BinaryConvexSnapshotProgressCursor.encode({ + cursor: pageCursor, + finished: !page.hasMore + }) + }); + this.relationCache.update(latestTable); + + if (!page.hasMore) { + break; + } + + this.touch(); + } + + if (this.abortSignal.aborted) { + throw new ReplicationAbortedError('Initial replication interrupted'); + } + + return { + table: await this.markSnapshotDone(batch, latestTable, snapshotCursor) + }; + } + + private async markSnapshotDone( + batch: storage.BucketStorageBatch, + table: SourceTable, + snapshotCursor: string + ): Promise { + const snapshotLsnValue = parseConvexLsn(snapshotCursor); + const [doneTable] = await batch.markTableSnapshotDone([table], snapshotLsnValue); + this.relationCache.update(doneTable); + return doneTable; + } + + private async resolveSnapshotBoundary(snapshotLsn: string | null): Promise { + if (snapshotLsn != null) { + const snapshotCursor = parseConvexLsn(snapshotLsn); + this.logger.info(`Using existing global snapshot ${snapshotCursor}`); + return snapshotCursor; + } + + const snapshotCursor = await this.connections.client.getGlobalSnapshotCursor({ signal: this.abortSignal }); + this.logger.info(`Pinned global snapshot ${snapshotCursor}`); + return snapshotCursor; + } + + private async resolveAllSourceTables(batch: storage.BucketStorageBatch): Promise { + const sourceTablePatterns = this.syncConfig.getSourceTables(); + const resolved: SourceTable[] = []; + const seenSourceTableIds = new Set(); + + for (const tablePattern of sourceTablePatterns) { + const tables = await this.resolveQualifiedTableNames(batch, tablePattern); + for (const table of tables) { + const id = `${table.id}`; + if (seenSourceTableIds.has(id)) { + continue; + } + seenSourceTableIds.add(id); + resolved.push(table); + } + } + + return resolved; + } + + private async resolveQualifiedTableNames( + batch: storage.BucketStorageBatch, + tablePattern: TablePattern + ): Promise { + if (tablePattern.connectionTag != this.connections.connectionTag) { + return []; + } + + if (tablePattern.schema != this.defaultSchema) { + return []; + } + + const matchedTableNames = await this.resolveTablePattern(tablePattern); + + if (!tablePattern.isWildcard && matchedTableNames.length == 0) { + this.logger.warn(`Table ${tablePattern.schema}.${tablePattern.name} not found`); + } + + const resolved: SourceTable[] = []; + for (const tableName of matchedTableNames) { + const tables = await this.processTables(batch, { + connectionTag: this.connections.connectionTag, + schema: this.defaultSchema, + name: tableName, + objectId: tableName, + replicaIdColumns: [{ name: '_id' }] + }); + resolved.push(...tables); + } + + return resolved; + } + + private async getOrResolveTables( + batch: storage.BucketStorageBatch, + tableName: string, + snapshotLSN: string + ): Promise { + if (!this.isTableSelectedBySyncConfig(tableName)) { + return []; + } + + const descriptor: SourceEntityDescriptor = { + schema: this.defaultSchema, + connectionTag: this.connections.connectionTag, + name: tableName, + objectId: tableName, + replicaIdColumns: [{ name: '_id' }] + }; + + const existing = this.relationCache.getAll(descriptor); + if (existing) { + return existing; + } + + let tables = await this.processTables(batch, descriptor); + const snapshotCandidates = tables.filter((table) => !table.snapshotComplete && table.syncAny); + if (snapshotCandidates.length > 0) { + this.logger.info( + `New table discovered while streaming: [${descriptor.schema}.${descriptor.name}], applying deltas without snapshot` + ); + const doneTables = await batch.markTableSnapshotDone(snapshotCandidates, snapshotLSN); + const doneTableById = new Map(doneTables.map((table) => [table.id, table])); + tables = tables.map((table) => doneTableById.get(table.id) ?? table); + this.relationCache.updateAll(descriptor, tables); + } + + return tables; + } + + private isTableSelectedBySyncConfig(tableName: string): boolean { + for (const sourceTablePattern of this.syncConfig.getSourceTables()) { + if (sourceTablePattern.connectionTag != this.connections.connectionTag) { + continue; + } + if (sourceTablePattern.schema != this.defaultSchema) { + continue; + } + + if (sourceTablePattern.isWildcard) { + if (tableName.startsWith(sourceTablePattern.tablePrefix)) { + return true; + } + } else if (sourceTablePattern.name == tableName) { + return true; + } + } + + return false; + } + + private async processTables( + batch: storage.BucketStorageBatch, + descriptor: SourceEntityDescriptor + ): Promise { + const resolved = await batch.resolveTables({ + connection_id: this.connectionId, + source: descriptor + }); + + if (resolved.dropTables.length > 0) { + await batch.drop(resolved.dropTables); + } + + this.relationCache.updateAll(descriptor, resolved.tables); + return resolved.tables; + } + + private async resolveTablePattern(tablePattern: TablePattern): Promise { + const schema = await this.connections.client.getJsonSchemas({ signal: this.abortSignal }); + const availableTableNames = schema.tables + .map((table) => table.tableName) + .filter((tableName) => !isConvexCheckpointTable(tableName)) + .sort(); + + if (!tablePattern.isWildcard) { + return availableTableNames.includes(tablePattern.name) ? [tablePattern.name] : []; + } + + return availableTableNames.filter((tableName) => tableName.startsWith(tablePattern.tablePrefix)); + } + + private async writeChange( + batch: storage.BucketStorageBatch, + table: SourceTable, + change: ConvexRawDocument + ): Promise { + const replicaId = change._id; + if (replicaId == null) { + this.logger.warn(`Skipping Convex change without _id for ${table.qualifiedName}`); + return false; + } + + if (change._deleted) { + await batch.save({ + tag: SaveOperationTag.DELETE, + sourceTable: table, + before: undefined, + beforeReplicaId: replicaId, + after: undefined, + afterReplicaId: undefined + }); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); + return true; + } + + const after = this.toSqliteRow(change); + await batch.save({ + tag: SaveOperationTag.UPDATE, + sourceTable: table, + before: undefined, + beforeReplicaId: undefined, + after, + afterReplicaId: replicaId + }); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); + return true; + } + + private toSqliteRow(change: ConvexRawDocument) { + return this.syncConfig.applyRowContext(toSqliteInputRow(change)); + } + + private touch() { + if (performance.now() - this.lastTouchedAt < 1_000) { + return; + } + + this.lastTouchedAt = performance.now(); + container.probes.touch().catch((error) => { + this.logger.error(`Failed to touch the container probe: ${error instanceof Error ? error.message : `${error}`}`); + }); + } +} + +function getCacheIdentifier(source: SourceEntityDescriptor | SourceTable): string { + const connectionTag = source instanceof SourceTable ? source.ref.connectionTag : source.connectionTag; + return `${connectionTag}.${source.schema}.${source.name}`; +} + +function readTableName(change: ConvexRawDocument): string | null { + const table = change._table; + if (typeof table != 'string' || table.length == 0) { + return null; + } + return table; +} diff --git a/modules/module-convex/src/replication/check-source-configuration.ts b/modules/module-convex/src/replication/check-source-configuration.ts new file mode 100644 index 000000000..7f8ed7ed6 --- /dev/null +++ b/modules/module-convex/src/replication/check-source-configuration.ts @@ -0,0 +1,113 @@ +import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; +import { NormalizedConvexConnectionConfig } from '../types/types.js'; +import { ConvexConnectionManager } from './ConvexConnectionManager.js'; + +const MISSING_MUTATOR_FRAGMENT = ` +Define a mutator for the PowerSync service to use + +convex/powersync_checkpoints.ts +\`\`\`TypeScript +import { mutation } from './_generated/server.js'; + +export const createCheckpoint = mutation({ + args: {}, + handler: async (ctx) => { + const existing = await ctx.db.query('powersync_checkpoints').first(); + + if (existing) { + await ctx.db.patch(existing._id, { last_updated: Date.now() }); + } else { + await ctx.db.insert('powersync_checkpoints', { last_updated: Date.now() }); + } + } +}); +\`\`\``.trim(); + +export type ConvexSourceConfigurationTestResult = { + connected: boolean; + errors: string[]; +}; + +export type CheckSourceConfigurationOptions = { + /** + * Avoid calling Convex mutations while checking the source configuration. + * + * This is used by diagnostics and connection-status routes, which may be + * called often. Creating write checkpoint markers there is safe, but + * unnecessary and can create noisy replication activity on every poll. + */ + readOnly?: boolean; +}; + +export async function checkSourceConfiguration( + normalizedConfig: NormalizedConvexConnectionConfig, + options: CheckSourceConfigurationOptions = {} +): Promise { + const connectionManager = new ConvexConnectionManager(normalizedConfig); + const errors: string[] = []; + let connected = false; + try { + // Check if the database is reachable by fetching the schema. + const schema = await connectionManager.client.getJsonSchemas().catch((error) => { + const message = error instanceof Error ? error.message : `${error}`; + errors.push( + `Could not fetch Convex schema for provided connection configuration. Error: ${message || 'unknown'}` + ); + }); + + // We could connect if we got a schema response + connected = !!schema; + if (!schema) { + return { + connected, + errors + }; + } + + const hasCheckpointTable = schema.tables.some((table) => table.tableName == CONVEX_CHECKPOINT_TABLE); + if (!hasCheckpointTable) { + errors.push( + ` +Could not find the ${CONVEX_CHECKPOINT_TABLE} table in the schema. + +Define the ${CONVEX_CHECKPOINT_TABLE} table in the Convex schema: + +convex/schema.ts +\`\`\`TypeScript +//... + +export default defineSchema({ + // ... your other tables + + powersync_checkpoints: defineTable({ + last_updated: v.float64() + }) +}); +\`\`\` + +${MISSING_MUTATOR_FRAGMENT} + `.trim() + ); + } + + // Check that the PowerSync checkpoint table and mutation are deployed. + // It should be safe to update this table at any point. We only use it for emitting a replication event. + if (hasCheckpointTable && !options.readOnly) { + await connectionManager.client.createWriteCheckpointMarker().catch((error) => { + const message = error instanceof Error ? error.message : `${error}`; + errors.push( + ` +Could not call the createCheckpoint mutator. Error ${message || 'unknown'} + +${MISSING_MUTATOR_FRAGMENT}`.trim() + ); + }); + } + } finally { + await connectionManager.end(); + } + return { + connected, + errors + }; +} diff --git a/modules/module-convex/src/replication/replication-index.ts b/modules/module-convex/src/replication/replication-index.ts new file mode 100644 index 000000000..427c400fd --- /dev/null +++ b/modules/module-convex/src/replication/replication-index.ts @@ -0,0 +1,6 @@ +export * from './ConvexConnectionManager.js'; +export * from './ConvexConnectionManagerFactory.js'; +export * from './ConvexErrorRateLimiter.js'; +export * from './ConvexReplicationJob.js'; +export * from './ConvexReplicator.js'; +export * from './ConvexStream.js'; diff --git a/modules/module-convex/src/types/types.ts b/modules/module-convex/src/types/types.ts new file mode 100644 index 000000000..bdd8a1457 --- /dev/null +++ b/modules/module-convex/src/types/types.ts @@ -0,0 +1,109 @@ +import { ErrorCode, makeHostnameLookupFunction, ServiceError } from '@powersync/lib-services-framework'; +import * as service_types from '@powersync/service-types'; +import { LookupFunction } from 'node:net'; +import * as t from 'ts-codec'; + +export const CONVEX_CONNECTION_TYPE = 'convex' as const; +const DEFAULT_POLLING_INTERVAL_MS = 1_000; +const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; + +export interface NormalizedConvexConnectionConfig { + id: string; + tag: string; + + deployment_url: string; + deploy_key: string; + + debug_api: boolean; + polling_interval_ms: number; + request_timeout_ms: number; + + lookup?: LookupFunction; +} + +export const ConvexConnectionConfig = service_types.configFile.DataSourceConfig.and( + t.object({ + type: t.literal(CONVEX_CONNECTION_TYPE), + deployment_url: t.string, + deploy_key: t.string, + polling_interval_ms: t.number.optional(), + request_timeout_ms: t.number.optional(), + reject_ip_ranges: t.array(t.string).optional() + }) +); + +export type ConvexConnectionConfig = t.Decoded; + +export type ResolvedConvexConnectionConfig = ConvexConnectionConfig & NormalizedConvexConnectionConfig; + +export function normalizeConnectionConfig(options: ConvexConnectionConfig): NormalizedConvexConnectionConfig { + let deploymentURL: URL; + try { + deploymentURL = new URL(options.deployment_url); + } catch (error) { + throw new ServiceError( + ErrorCode.PSYNC_S1109, + `Convex connection: invalid deployment_url ${error instanceof Error ? `- ${error.message}` : ''}` + ); + } + + if (deploymentURL.protocol != 'https:' && deploymentURL.protocol != 'http:') { + throw new ServiceError( + ErrorCode.PSYNC_S1109, + `Convex connection: deployment_url must use http or https, got ${JSON.stringify(deploymentURL.protocol)}` + ); + } + + if (deploymentURL.hostname == '') { + throw new ServiceError(ErrorCode.PSYNC_S1106, `Convex connection: hostname required`); + } + + if (options.deploy_key == '') { + throw new ServiceError(ErrorCode.PSYNC_S1108, `Convex connection: deploy_key required`); + } + + const pollingIntervalMs = options.polling_interval_ms ?? DEFAULT_POLLING_INTERVAL_MS; + if (!Number.isFinite(pollingIntervalMs) || pollingIntervalMs <= 0) { + throw new ServiceError( + ErrorCode.PSYNC_S1109, + `Convex connection: polling_interval_ms must be a positive finite number` + ); + } + + const requestTimeoutMs = options.request_timeout_ms ?? DEFAULT_REQUEST_TIMEOUT_MS; + if (!Number.isFinite(requestTimeoutMs) || requestTimeoutMs <= 0) { + throw new ServiceError( + ErrorCode.PSYNC_S1109, + `Convex connection: request_timeout_ms must be a positive finite number` + ); + } + + const lookup = makeHostnameLookupFunction(deploymentURL.hostname, { + reject_ip_ranges: options.reject_ip_ranges ?? [] + }); + + return { + id: options.id ?? 'default', + tag: options.tag ?? 'default', + + deployment_url: deploymentURL.toString().replace(/\/$/, ''), + deploy_key: options.deploy_key, + + debug_api: options.debug_api ?? false, + polling_interval_ms: pollingIntervalMs, + request_timeout_ms: requestTimeoutMs, + + lookup + }; +} + +export function resolveConvexConnectionConfig(config: ConvexConnectionConfig): ResolvedConvexConnectionConfig { + return { + ...config, + ...normalizeConnectionConfig(config) + }; +} + +export function baseUri(config: ResolvedConvexConnectionConfig) { + return config.deployment_url; +} diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts new file mode 100644 index 000000000..c96266594 --- /dev/null +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -0,0 +1,180 @@ +import { ConvexApiClient } from '@module/client/ConvexApiClient.js'; +import { ConvexListSnapshotResult, RawJsonSchemaResponse } from '@module/client/ConvexAPITypes.js'; +import { CONVEX_CHECKPOINT_TABLE } from '@module/common/ConvexCheckpoints.js'; +import { normalizeConnectionConfig } from '@module/types/types.js'; +import { JSONBig } from '@powersync/service-jsonbig'; +import nodeFetch from 'node-fetch'; +import * as https from 'node:https'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('node-fetch', () => ({ + default: vi.fn() +})); + +const baseConfig = normalizeConnectionConfig({ + type: 'convex', + deployment_url: 'https://example.convex.cloud', + deploy_key: 'test-key' +}); +const SNAPSHOT_CURSOR = 1770335566197683000n; +const fetchMock = vi.mocked(nodeFetch); + +describe('ConvexApiClient', () => { + afterEach(() => { + vi.useRealTimers(); + fetchMock.mockReset(); + vi.restoreAllMocks(); + }); + + it('sends Convex authorization header and format=json', async () => { + fetchMock.mockResolvedValue( + new Response( + JSON.stringify({ + users: { type: 'table', properties: { _id: { type: 'string' } } } + } satisfies RawJsonSchemaResponse), + { status: 200 } + ) as any + ); + + const client = new ConvexApiClient(baseConfig); + const result = await client.getJsonSchemas(); + + expect(result.tables.map((table) => table.tableName)).toEqual(['users']); + + const [url, init] = fetchMock.mock.calls[0]!; + expect(String(url)).toContain('/api/json_schemas'); + expect(String(url)).toContain('format=json'); + expect((init?.headers as Record).Authorization).toBe('Convex test-key'); + }); + + it('preserves high-precision numeric snapshot values', async () => { + fetchMock.mockResolvedValue( + new Response( + '{"values":[],"snapshot":1770335566197682922,"cursor":"{\\"tablet\\":\\"X0yj4Cm7GfuikfsSBm9QCQ\\",\\"id\\":\\"j5700000000000000000000000001qv0\\"}","hasMore":true}', + { status: 200 } + ) as any + ); + + const client = new ConvexApiClient(baseConfig); + const page = await client.listSnapshot({ tableName: 'lists' }); + + expect(page.snapshot).toBe(1770335566197682922n); + expect(page.cursor).toContain('"tablet":"X0yj4Cm7GfuikfsSBm9QCQ"'); + expect(page.hasMore).toBe(true); + }); + + it('sends table_name as snake_case query parameter in list_snapshot', async () => { + fetchMock.mockResolvedValue( + new Response( + JSONBig.stringify({ + snapshot: SNAPSHOT_CURSOR, + cursor: null, + hasMore: false, + values: [] + } satisfies ConvexListSnapshotResult), + { status: 200 } + ) as any + ); + + const client = new ConvexApiClient(baseConfig); + await client.listSnapshot({ tableName: 'lists', snapshot: SNAPSHOT_CURSOR.toString() }); + + const url = String(fetchMock.mock.calls[0]![0]); + expect(url).toContain('table_name=lists'); + expect(url).not.toContain('tableName=lists'); + }); + + it('marks network failures as retryable', async () => { + fetchMock.mockRejectedValue(new Error('fetch failed: ECONNRESET')); + + const client = new ConvexApiClient(baseConfig); + + await expect(client.getJsonSchemas()).rejects.toMatchObject({ + retryable: true + }); + }); + + it('uses the configured request timeout', async () => { + vi.useFakeTimers(); + fetchMock.mockImplementation( + (_url, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + reject((init.signal as any).reason); + }); + }) as any + ); + + const client = new ConvexApiClient({ + ...baseConfig, + request_timeout_ms: 25 + }); + + const request = expect(client.getJsonSchemas()).rejects.toMatchObject({ + message: expect.stringContaining('timed out after 25ms'), + retryable: true + }); + + await vi.advanceTimersByTimeAsync(25); + await request; + }); + + it('creates write checkpoint markers via mutation', async () => { + fetchMock.mockResolvedValue(new Response(JSON.stringify({ status: 'success' }), { status: 200 }) as any); + + const client = new ConvexApiClient(baseConfig); + await client.createWriteCheckpointMarker(); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0]!; + expect(String(url)).toContain('/api/mutation'); + expect(init?.method).toBe('POST'); + expect((init?.headers as Record).Authorization).toBe('Convex test-key'); + + const body = JSON.parse(String(init?.body)); + expect(body.path).toBe(`${CONVEX_CHECKPOINT_TABLE}:createCheckpoint`); + expect(body.args).toEqual({}); + expect(body.format).toBe('json'); + }); + + it('propagates checkpoint write errors directly (no fallback)', async () => { + fetchMock.mockResolvedValueOnce(new Response(JSON.stringify({ code: 'SomeError' }), { status: 400 }) as any); + + const client = new ConvexApiClient(baseConfig); + await expect(client.createWriteCheckpointMarker()).rejects.toMatchObject({ + status: 400, + retryable: false + }); + }); + + it('uses an agent with the configured hostname policy for Convex API requests', async () => { + fetchMock.mockResolvedValue(new Response(JSON.stringify({ status: 'success' }), { status: 200 }) as any); + const lookup = vi.fn((_hostname: string, _options: any, callback: (error: Error) => void) => { + callback(new Error('blocked by reject_ip_ranges')); + }) as unknown as import('node:net').LookupFunction; + + const client = new ConvexApiClient({ + ...baseConfig, + lookup + }); + + await client.createWriteCheckpointMarker(); + + const init = fetchMock.mock.calls[0]![1] as RequestInit & { agent: https.Agent }; + expect(init.agent).toBeInstanceOf(https.Agent); + expect(init.agent.options.lookup).toBe(lookup); + expect(lookup).not.toHaveBeenCalled(); + + await expect( + new Promise((resolve, reject) => { + init.agent.options.lookup!('example.convex.cloud', {}, (error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }) + ).rejects.toThrow('blocked by reject_ip_ranges'); + }); +}); diff --git a/modules/module-convex/test/src/ConvexCheckpoints.test.ts b/modules/module-convex/test/src/ConvexCheckpoints.test.ts new file mode 100644 index 000000000..7e2f64051 --- /dev/null +++ b/modules/module-convex/test/src/ConvexCheckpoints.test.ts @@ -0,0 +1,17 @@ +import { isConvexCheckpointTable } from '@module/common/ConvexCheckpoints.js'; +import { describe, expect, it } from 'vitest'; + +describe('ConvexCheckpoints', () => { + it('recognizes the checkpoint table name', () => { + expect(isConvexCheckpointTable('powersync_checkpoints')).toBe(true); + }); + + it('does not match underscore-prefixed variant', () => { + expect(isConvexCheckpointTable('_powersync_checkpoints')).toBe(false); + }); + + it('does not match non-checkpoint tables', () => { + expect(isConvexCheckpointTable('lists')).toBe(false); + expect(isConvexCheckpointTable('powersync_other')).toBe(false); + }); +}); diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts new file mode 100644 index 000000000..06d731512 --- /dev/null +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -0,0 +1,50 @@ +import { parseConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; +import { describe, expect, it } from 'vitest'; + +describe('Convex cursor LSN helpers', () => { + it('validates and round-trips the numeric cursor', () => { + const source = parseConvexLsn('1772817606884944136'); + const roundTrip = parseConvexLsn(source); + + expect(source).toBe('1772817606884944136'); + expect(roundTrip).toBe('1772817606884944136'); + }); + + it('sorts lexicographically after validating fixed-width timestamps', () => { + const older = parseConvexLsn('1772817606884944136'); + const newer = parseConvexLsn('1772817606884944137'); + + expect(older < newer).toBe(true); + }); + + it('handles bare numeric cursor string', () => { + const parsed = parseConvexLsn('1772817606884944136'); + expect(parsed).toBe('1772817606884944136'); + }); + + it('allows the zero sentinel', () => { + expect(parseConvexLsn(ZERO_LSN)).toBe(ZERO_LSN); + }); + + it('rejects non-19-digit numeric cursors', () => { + expect(() => parseConvexLsn('1770335566197683')).toThrow('Convex cursor is not a 19-digit numeric timestamp'); + }); + + it('rejects padded serialized payloads', () => { + expect(() => parseConvexLsn('0001772817606884944136')).toThrow( + 'Convex cursor is not a canonical numeric timestamp' + ); + }); + + it('rejects delimiter-based serialized payloads', () => { + expect(() => parseConvexLsn('1772817606884944136|1772817606884944136')).toThrow( + 'Convex cursor is not a valid numeric timestamp' + ); + }); + + it('rejects non-numeric cursors', () => { + expect(() => parseConvexLsn('{"tablet":"abc","id":"xyz"}')).toThrow( + 'Convex cursor is not a valid numeric timestamp' + ); + }); +}); diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts new file mode 100644 index 000000000..5b64ddf56 --- /dev/null +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -0,0 +1,102 @@ +import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; +import { ConvexJsonSchemasResult } from '@module/client/ConvexAPITypes.js'; +import { parseConvexLsn } from '@module/common/ConvexLSN.js'; +import { normalizeConnectionConfig } from '@module/types/types.js'; +import { ExpressionType, SqlSyncRules } from '@powersync/service-sync-rules'; +import { describe, expect, it, vi } from 'vitest'; + +const HEAD_CURSOR = '1772817606884944123'; + +function createAdapter() { + const config = normalizeConnectionConfig({ + type: 'convex', + deployment_url: 'https://example.convex.cloud', + deploy_key: 'test-key' + }); + + const adapter = new ConvexRouteAPIAdapter({ + ...config, + type: 'convex', + deployment_url: 'https://example.convex.cloud', + deploy_key: 'test-key' + }); + + (adapter as any).connectionManager.client = { + getJsonSchemas: async () => { + return { + tables: [ + { + tableName: 'users', + schema: { + type: 'object', + properties: { + _id: { type: 'string' }, + age: { type: 'integer' }, + avatar: { type: 'string', $description: 'base64 bytes' } + } + } + } + ] + } satisfies ConvexJsonSchemasResult; + }, + getHeadCursor: async () => HEAD_CURSOR, + createWriteCheckpointMarker: async () => undefined + }; + + return adapter; +} + +describe('ConvexRouteAPIAdapter', () => { + it('returns connection schema from Convex json schema', async () => { + const adapter = createAdapter(); + + const schema = await adapter.getConnectionSchema(); + expect(schema[0]?.name).toBe('convex'); + expect(schema[0]?.tables[0]?.name).toBe('users'); + expect(schema[0]?.tables[0]?.columns.find((column) => column.name == '_id')?.type).toBe('id'); + expect(schema[0]?.tables[0]?.columns.find((column) => column.name == '_creationTime')).toBeUndefined(); + expect(schema[0]?.tables[0]?.columns.find((column) => column.name == 'avatar')?.sqlite_type).toBe( + ExpressionType.TEXT.typeFlags + ); + + await adapter.shutdown(); + }); + + it('builds debug table info for matching patterns', async () => { + const adapter = createAdapter(); + + const syncRules = SqlSyncRules.fromYaml( + ` +bucket_definitions: + test: + data: + - SELECT _id AS id FROM users +`, + { + defaultSchema: 'convex' + } + ); + + const result = await adapter.getDebugTablesInfo(syncRules.config.getSourceTables(), syncRules.config); + expect(result[0]?.table?.name).toBe('users'); + + await adapter.shutdown(); + }); + + it('creates replication head from the global snapshot cursor', async () => { + const adapter = createAdapter(); + const getHeadCursor = vi.fn(async (_options?: any) => HEAD_CURSOR); + const createWriteCheckpointMarker = vi.fn(async (_options?: any) => undefined); + (adapter as any).connectionManager.client.getHeadCursor = getHeadCursor; + (adapter as any).connectionManager.client.createWriteCheckpointMarker = createWriteCheckpointMarker; + + const result = await adapter.createReplicationHead(async (head) => head); + expect(result).toBe(parseConvexLsn(HEAD_CURSOR)); + expect(getHeadCursor).toHaveBeenCalledTimes(1); + expect(getHeadCursor).toHaveBeenCalledWith(); + expect(createWriteCheckpointMarker).toHaveBeenCalledTimes(1); + expect(createWriteCheckpointMarker).toHaveBeenCalledWith(); + + await adapter.shutdown(); + }); +}); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts new file mode 100644 index 000000000..aea76e9ef --- /dev/null +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -0,0 +1,673 @@ +import { ConvexDocumentDeltasResult } from '@module/client/ConvexAPITypes.js'; +import { parseConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; +import { BinaryConvexSnapshotProgressCursor } from '@module/replication/ConvexSnapshotProgressCursor.js'; +import { ConvexStream } from '@module/replication/ConvexStream.js'; +import { SaveOperationTag, SourceTable } from '@powersync/service-core'; +import { TablePattern } from '@powersync/service-sync-rules'; +import { ReplicationMetric } from '@powersync/service-types'; +import { describe, expect, it, vi } from 'vitest'; + +const CURSOR_100 = 1772817606884944100n; +const CURSOR_101 = 1772817606884944101n; +const CURSOR_102 = 1772817606884944102n; +const CURSOR_200 = 1772817606884944200n; +const CURSOR_300 = 1772817606884944300n; +const CURSOR_301 = 1772817606884944301n; + +function createFakeStorage(options?: { + snapshotDone?: boolean; + snapshotLsn?: string | null; + resumeFromLsn?: string | null; + sourcePatterns?: TablePattern[]; + tableSnapshotStatus?: { + totalEstimatedCount?: number; + replicatedCount?: number; + lastKey?: Uint8Array | null; + }; +}) { + const saves: any[] = []; + const commits: string[] = []; + const keepalives: string[] = []; + const allSnapshotDoneLsns: string[] = []; + const resumeLsnUpdates: string[] = []; + const tableProgressUpdates: any[] = []; + + const tables = new Map(); + let nextTableId = 1; + const getOrCreateTable = ( + name: string, + tableOptions?: { + snapshotStatus?: { + totalEstimatedCount?: number; + replicatedCount?: number; + lastKey?: Uint8Array | null; + }; + snapshotComplete?: boolean; + } + ) => { + const existing = tables.get(name); + if (existing != null) { + return existing; + } + + const table = new SourceTable({ + id: `${nextTableId++}`, + ref: { + connectionTag: 'default', + schema: 'convex', + name + }, + objectId: name, + replicaIdColumns: [{ name: '_id' }], + snapshotComplete: tableOptions?.snapshotComplete ?? false, + bucketDataSources: [], + parameterLookupSources: [] + }); + if (tableOptions?.snapshotStatus) { + table.snapshotStatus = { + totalEstimatedCount: tableOptions.snapshotStatus.totalEstimatedCount ?? -1, + replicatedCount: tableOptions.snapshotStatus.replicatedCount ?? 0, + lastKey: tableOptions.snapshotStatus.lastKey ?? null + }; + } + + tables.set(name, table); + return table; + }; + + const table = getOrCreateTable('users', { + snapshotStatus: options?.tableSnapshotStatus + }); + + const batch: any = { + lastCheckpointLsn: null, + resumeFromLsn: options?.resumeFromLsn ?? null, + noCheckpointBeforeLsn: ZERO_LSN, + async [Symbol.asyncDispose]() {}, + async save(record: any) { + saves.push(record); + return null; + }, + async truncate(_tables: SourceTable[]) { + return null; + }, + async drop(_tables: SourceTable[]) { + return null; + }, + async flush() { + return null; + }, + async commit(lsn: string) { + commits.push(lsn); + this.lastCheckpointLsn = lsn; + return true; + }, + async keepalive(lsn: string) { + keepalives.push(lsn); + this.lastCheckpointLsn = lsn; + return true; + }, + async setResumeLsn(lsn: string) { + resumeLsnUpdates.push(lsn); + this.resumeFromLsn = lsn; + }, + async markAllSnapshotDone(lsn: string) { + allSnapshotDoneLsns.push(lsn); + }, + async markTableSnapshotDone(tables: SourceTable[], _lsn: string) { + for (const sourceTable of tables) { + sourceTable.snapshotComplete = true; + } + return tables; + }, + async updateTableProgress(sourceTable: SourceTable, progress: any) { + tableProgressUpdates.push({ + tableName: sourceTable.name, + ...progress + }); + sourceTable.snapshotStatus = { + totalEstimatedCount: progress.totalEstimatedCount ?? sourceTable.snapshotStatus?.totalEstimatedCount ?? -1, + replicatedCount: progress.replicatedCount ?? sourceTable.snapshotStatus?.replicatedCount ?? 0, + lastKey: progress.lastKey ?? sourceTable.snapshotStatus?.lastKey ?? null + }; + return sourceTable; + } + }; + + const syncRules = { + getSourceTables: () => options?.sourcePatterns ?? [new TablePattern('convex', 'users')], + applyRowContext: (row: Record) => row, + getMatchingSources: () => ({ + bucketDataSources: [], + parameterLookupSources: [] + }), + tableTriggersEvent: () => false + }; + + for (const sourceTable of tables.values()) { + sourceTable.syncData = true; + sourceTable.syncParameters = false; + sourceTable.syncEvent = false; + } + + batch.resolveTables = vi.fn(async ({ source }: any) => { + const resolvedTable = getOrCreateTable(source.name); + resolvedTable.syncData = true; + resolvedTable.syncParameters = false; + resolvedTable.syncEvent = false; + return { + tables: [resolvedTable], + dropTables: [] + }; + }); + + const storage = { + group_id: 1, + getParsedSyncRules: () => syncRules, + async getStatus() { + return { + active: true, + snapshot_done: options?.snapshotDone ?? false, + checkpoint_lsn: options?.snapshotDone ? parseConvexLsn(CURSOR_100) : null, + snapshot_lsn: options?.snapshotLsn ?? null + }; + } + }; + Object.assign(storage, { + clear: vi.fn(async () => undefined), + populatePersistentChecksumCache: vi.fn(async () => ({ buckets: 0 })), + createWriter: vi.fn(async (_options: any) => batch), + startBatch: vi.fn(async (_options: any, callback: (batch: any) => Promise) => { + await callback(batch); + return { flushed_op: 1n }; + }), + reportError: vi.fn(async () => undefined) + }); + + return { + storage, + batch, + table, + tables, + saves, + commits, + keepalives, + allSnapshotDoneLsns, + resumeLsnUpdates, + tableProgressUpdates + }; +} + +describe('ConvexStream', () => { + it('pins a global snapshot boundary before table hydration', async () => { + const context = createFakeStorage(); + const abortController = new AbortController(); + const snapshotCalls: any[] = []; + const getJsonSchemas = vi.fn(async () => ({ + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], + raw: {} + })); + const listSnapshot = vi.fn(async (options: any) => { + snapshotCalls.push(options ?? {}); + if (options?.tableName == null) { + return { + snapshot: CURSOR_100, + cursor: null, + hasMore: false, + values: [] + }; + } + + return { + snapshot: CURSOR_100, + cursor: null, + hasMore: false, + values: [{ _table: 'users', _id: 'u1', name: 'Alice' }] + }; + }); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas, + listSnapshot, + getGlobalSnapshotCursor: async (options?: any) => (await listSnapshot(options)).snapshot + } + } as any + }); + + await stream.initReplication(); + + expect(snapshotCalls.length).toBe(2); + expect(snapshotCalls[0]?.tableName).toBeUndefined(); + expect(snapshotCalls[0]?.cursor).toBeUndefined(); + expect(snapshotCalls[1]?.tableName).toBe('users'); + expect(snapshotCalls[1]?.cursor).toBeUndefined(); + expect(snapshotCalls[1]?.snapshot).toBe(CURSOR_100); + expect(getJsonSchemas).toHaveBeenCalledTimes(1); + expect(context.saves.length).toBe(1); + expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); + expect(context.resumeLsnUpdates.length).toBe(1); + expect(context.allSnapshotDoneLsns).toEqual([parseConvexLsn(CURSOR_100)]); + expect(context.commits.at(-1)).toBe(parseConvexLsn(CURSOR_100)); + }); + + it('does not snapshot exact tables missing from json_schemas', async () => { + const context = createFakeStorage(); + const abortController = new AbortController(); + const listSnapshot = vi.fn(async (options: any) => { + if (options?.tableName == null) { + return { + snapshot: CURSOR_100, + cursor: null, + hasMore: false, + values: [] + }; + } + + throw new Error(`Unexpected table snapshot for ${options.tableName}`); + }); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [], + raw: {} + }), + listSnapshot, + getGlobalSnapshotCursor: async (options?: any) => (await listSnapshot(options)).snapshot + } + } as any + }); + + await stream.initReplication(); + + expect(listSnapshot).toHaveBeenCalledTimes(1); + expect(context.saves).toHaveLength(0); + expect(context.allSnapshotDoneLsns).toEqual([parseConvexLsn(CURSOR_100)]); + }); + + it('keeps bytes fields as base64 strings during snapshot hydration', async () => { + const context = createFakeStorage(); + const abortController = new AbortController(); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [ + { + tableName: 'users', + schema: { + type: 'object', + properties: { + avatar: { type: 'string', $description: 'base64 bytes' } + } + } + } + ], + raw: {} + }), + listSnapshot: async (options: any) => { + if (options?.tableName == null) { + return { + snapshot: CURSOR_100, + cursor: null, + hasMore: false, + values: [] + }; + } + + return { + snapshot: CURSOR_100, + cursor: null, + hasMore: false, + values: [{ _table: 'users', _id: 'u1', avatar: 'AQID' }] + }; + }, + getGlobalSnapshotCursor: async () => CURSOR_100 + } + } as any + }); + + await stream.initReplication(); + + expect(context.saves).toHaveLength(1); + expect(context.saves[0]?.after.avatar).toBe('AQID'); + }); + + it('marks snapshot done without re-reading rows when the final page was already flushed', async () => { + const context = createFakeStorage({ + snapshotLsn: parseConvexLsn(CURSOR_200), + tableSnapshotStatus: { + replicatedCount: 2, + totalEstimatedCount: -1, + lastKey: BinaryConvexSnapshotProgressCursor.encode({ + cursor: null, + finished: true + }) + } + }); + const abortController = new AbortController(); + const listSnapshot = vi.fn(async () => ({ + snapshot: CURSOR_200, + cursor: null, + hasMore: false, + values: [] + })); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { polling_interval_ms: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], + raw: {} + }), + listSnapshot, + getGlobalSnapshotCursor: async () => 'should-not-be-called' + } + } as any + }); + + await stream.initReplication(); + + expect(listSnapshot).not.toHaveBeenCalled(); + expect(context.saves.length).toBe(0); + expect(context.table.snapshotComplete).toBe(true); + }); + + it('fails when table snapshots return a different snapshot boundary', async () => { + const context = createFakeStorage({ + snapshotLsn: parseConvexLsn(CURSOR_300) + }); + const abortController = new AbortController(); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], + raw: {} + }), + getGlobalSnapshotCursor: async () => 'should-not-be-called', + listSnapshot: async () => ({ + snapshot: CURSOR_301, + cursor: null, + hasMore: false, + values: [{ _table: 'users', _id: 'u1', name: 'Alice' }] + }) + } + } as any + }); + + await expect(stream.initReplication()).rejects.toThrow(/snapshot cursor changed while snapshotting/); + }); + + it('streams deltas and commits checkpoint', async () => { + const context = createFakeStorage({ + snapshotDone: true, + resumeFromLsn: parseConvexLsn(CURSOR_100) + }); + const abortController = new AbortController(); + + let calls = 0; + const deltaCalls: any[] = []; + const transactionCounts: number[] = []; + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: (metric: string) => ({ + add: (value: number) => { + if (metric == ReplicationMetric.TRANSACTIONS_REPLICATED) { + transactionCounts.push(value); + } + } + }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], + raw: {} + }), + documentDeltas: async (options: any) => { + calls += 1; + deltaCalls.push(options ?? {}); + setTimeout(() => abortController.abort(), 0); + return { + cursor: CURSOR_102, + hasMore: false, + values: [ + { _table: 'users', _id: 'u1', _ts: BigInt(CURSOR_101), name: 'Updated' }, + { _table: 'users', _id: 'u2', _ts: BigInt(CURSOR_101), _deleted: true }, + { _table: 'users', _id: 'u3', _ts: BigInt(CURSOR_102), name: 'Second transaction' } + ] + }; + } + } + } as any + }); + + await stream.streamChanges(); + + expect(calls).toBeGreaterThan(0); + expect(context.saves.length).toBe(3); + expect(context.saves[0]?.tag).toBe(SaveOperationTag.UPDATE); + expect(context.saves[1]?.tag).toBe(SaveOperationTag.DELETE); + expect(context.saves[2]?.tag).toBe(SaveOperationTag.UPDATE); + expect(context.commits.at(-1)).toBe(parseConvexLsn(CURSOR_102)); + expect(deltaCalls[0]?.tableName).toBeUndefined(); + expect(transactionCounts).toEqual([2]); + }); + + it('fails when document_deltas returns decreasing transaction timestamps', async () => { + // This should never happen in practice. + // We assert that _ts is increasing in ConvexStream + // This test just verifies the assertion would catch an issue if it ever happened for some reason. + const context = createFakeStorage({ + snapshotDone: true, + resumeFromLsn: parseConvexLsn(CURSOR_100) + }); + const abortController = new AbortController(); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }] + }), + documentDeltas: async () => { + return { + cursor: CURSOR_102, + hasMore: false, + values: [ + { _table: 'users', _id: 'u1', _ts: BigInt(CURSOR_102), name: 'Later' }, + { _table: 'users', _id: 'u2', _ts: BigInt(CURSOR_101), name: 'Earlier' } + ] + } satisfies ConvexDocumentDeltasResult; + } + } + } as any + }); + + await expect(stream.streamChanges()).rejects.toThrow(/out-of-order _ts values/); + }); + + it('resolves a newly discovered wildcard-matched table from document deltas without snapshotting', async () => { + const context = createFakeStorage({ + snapshotDone: true, + resumeFromLsn: parseConvexLsn(CURSOR_100), + sourcePatterns: [new TablePattern('convex', 'projects%')] + }); + const abortController = new AbortController(); + + let calls = 0; + const getJsonSchemas = vi.fn(async () => ({ + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], + raw: {} + })); + const listSnapshot = vi.fn(); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas, + listSnapshot, + documentDeltas: async () => { + calls += 1; + setTimeout(() => abortController.abort(), 0); + return { + cursor: CURSOR_101, + hasMore: false, + values: [{ _table: 'projects_archive', _id: 'p1', _ts: CURSOR_101, name: 'From delta' }] + }; + } + } + } as any + }); + + await stream.streamChanges(); + + expect(calls).toBeGreaterThan(0); + expect(getJsonSchemas).toHaveBeenCalledTimes(1); + expect(listSnapshot).not.toHaveBeenCalled(); + expect(context.saves.length).toBe(1); + expect(context.saves[0]?.tag).toBe(SaveOperationTag.UPDATE); + expect(context.saves[0]?.after.name).toBe('From delta'); + expect(context.saves[0]?.sourceTable.name).toBe('projects_archive'); + expect(context.tables.get('projects_archive')?.snapshotComplete).toBe(true); + }); + + it('keeps alive on idle startup and immediately when only checkpoint marker rows are streamed', async () => { + const context = createFakeStorage({ + snapshotDone: true, + resumeFromLsn: parseConvexLsn(CURSOR_100) + }); + const abortController = new AbortController(); + let calls = 0; + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], + raw: {} + }), + documentDeltas: async () => { + calls += 1; + if (calls == 1) { + // Convex does not advance the cursor when there are no deltas. + // Since this is the first poll after startup, the periodic + // keepalive path still runs immediately because lastKeepaliveAt + // starts at 0. + return { + cursor: CURSOR_100, + hasMore: false, + values: [] + }; + } + + setTimeout(() => abortController.abort(), 0); + // A checkpoint marker is a real Convex delta, so the cursor advances. + // The row is ignored as replicated data, but marker-only pages must + // still advance the storage checkpoint immediately so managed write + // checkpoints can become visible without waiting for the 60s throttle. + return { + cursor: CURSOR_101, + hasMore: false, + values: [{ _table: 'powersync_checkpoints', _id: 'cp1' }] + }; + } + } + } as any + }); + + await stream.streamChanges(); + + expect(context.saves.length).toBe(0); + expect(context.commits.length).toBe(0); + // The idle startup page and marker-only page both keep the checkpoint moving, + // but for different reasons: the first uses the same-cursor idle keepalive + // path, and the second uses the marker-only immediate keepalive path. + expect(context.keepalives).toEqual([parseConvexLsn(CURSOR_100), parseConvexLsn(CURSOR_101)]); + }); +}); diff --git a/modules/module-convex/test/src/bench/initial_replication.test.ts b/modules/module-convex/test/src/bench/initial_replication.test.ts new file mode 100644 index 000000000..8983715af --- /dev/null +++ b/modules/module-convex/test/src/bench/initial_replication.test.ts @@ -0,0 +1,106 @@ +import { afterAll, describe, test } from 'vitest'; +import { env } from '../env.js'; +import { ConvexStreamTestContext } from '../test-utils/ConvexStreamTestContext.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY } from '../test-utils/util.js'; + +/** + * Vitest has built in benchmarking functionality, but, it seems non-trivial to preseed the database. + * Run this by running: + * ```bash + * export SHOULD_RUN_BENCHMARK=true + * vitest initial_replication.test.ts + * ``` + */ +const SEED_BATCH_SIZE = 1_000; +const ESTIMATED_STORAGE_BYTES_PER_1K_ROWS = 350 * 1024; + +const BENCHMARK_SYNC_RULES = ` +bucket_definitions: + global: + data: + - SELECT uuid as id, name FROM "lists" + - SELECT uuid as id, description, list_uuid FROM "todos" +`; + +describe + .skipIf(!env.SHOULD_RUN_BENCHMARK) + .sequential('Convex initial replication benchmark', { timeout: Infinity }, () => { + let testResults = new Map(); + + afterAll(() => { + console.log('Test results'); + console.log('| Total Rows | Estimated Size (MiB) | Elapsed Time (ms) | Estimated Rate (MiB/s) |'); + for (const [totalRows, elapsedTime] of testResults.entries()) { + const estimatedSize = estimateStorageSize(totalRows); + const estimatedRate = estimateReplicationRate(estimatedSize.bytes, elapsedTime); + console.log( + `| ${totalRows} | ${estimatedSize.mib.toFixed(2)} | ${Math.round(elapsedTime)} | ${estimatedRate.toFixed( + 2 + )} |` + ); + } + }); + + function defineTest({ totalRows }: { totalRows: number }) { + return test(`starting initial replication for ${totalRows} total rows`, async () => { + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY.factory, { + doNotClear: false // Will clear the source (Convex) database + }); + // seed with data + await seedBenchmarkRows(context, totalRows); + + await context.updateSyncRules(BENCHMARK_SYNC_RULES); + + const startedAt = performance.now(); + await context.replicateSnapshot(); + const elapsedMs = performance.now() - startedAt; + + console.info(`Initial Convex replication benchmark replicated ${totalRows} rows in ${Math.round(elapsedMs)}ms`); + + testResults.set(totalRows, elapsedMs); + }); + } + + // Now register the tests for each dataset + // Storage estimate is based on current observed storage use: 1_000 rows ~= 350 KiB. + defineTest({ + totalRows: 10_000 + }); + + defineTest({ + totalRows: 50_000 + }); + + defineTest({ + totalRows: 100_000 + }); + }); + +function estimateStorageSize(totalRows: number) { + const bytes = (totalRows / 1_000) * ESTIMATED_STORAGE_BYTES_PER_1K_ROWS; + return { + bytes, + mib: bytes / 1024 / 1024 + }; +} + +function estimateReplicationRate(bytes: number, elapsedMs: number) { + return bytes / 1024 / 1024 / (elapsedMs / 1_000); +} + +async function seedBenchmarkRows(context: ConvexStreamTestContext, totalRows: number) { + const totalRowsPerTable = Math.floor(totalRows / 2); + for (let counter = 0; counter < totalRowsPerTable; counter += SEED_BATCH_SIZE) { + const count = Math.min(SEED_BATCH_SIZE, totalRowsPerTable - counter); + await context.backend.client.mutation(context.backend.api.benchmark.seedInitialReplicationBatch, { + rowsPerTable: count + }); + + console.info(`Seeded ${(counter + count) * 2}/${totalRows} rows.`); + /** + * Nasty workarround for default Convex write limits + * {"code":"TooManyWrites","message":"Too many writes per second. Your deployment is limited to 4 MiB bytes written per 1 second. Reduce your write rate or upgrade to a larger deployment."} + */ + await new Promise((r) => setTimeout(r, 1_000)); + } +} diff --git a/modules/module-convex/test/src/convex-to-sqlite.test.ts b/modules/module-convex/test/src/convex-to-sqlite.test.ts new file mode 100644 index 000000000..a0710f883 --- /dev/null +++ b/modules/module-convex/test/src/convex-to-sqlite.test.ts @@ -0,0 +1,71 @@ +import { jsonSchemaToSQLiteType, readConvexFieldJsonType, toSqliteInputRow } from '@module/common/convex-to-sqlite.js'; +import { + applyRowContext, + CompatibilityContext, + CompatibilityEdition, + ExpressionType +} from '@powersync/service-sync-rules'; +import { describe, expect, it } from 'vitest'; + +const context = new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }); + +describe('convex-to-sqlite', () => { + it('detects bytes and id field types from schema metadata', () => { + expect(jsonSchemaToSQLiteType(readConvexFieldJsonType({ type: 'string', format: 'id' }))).toBe(ExpressionType.TEXT); + expect(jsonSchemaToSQLiteType(readConvexFieldJsonType({ type: 'object' }))).toBe(ExpressionType.TEXT); + expect(jsonSchemaToSQLiteType(readConvexFieldJsonType({ $description: 'base64 bytes', type: 'string' }))).toBe( + ExpressionType.TEXT + ); + expect( + jsonSchemaToSQLiteType( + readConvexFieldJsonType({ $description: 'int64 represented as base10 string', type: 'string' }) + ) + ).toBe(ExpressionType.TEXT); + }); + + it('keeps bytes as base64 strings without using schema metadata for row conversion', () => { + const row = applyRowContext( + toSqliteInputRow({ + _id: 'doc1', + payload: 'AQID', + plain_text: 'AQID' + }), + context + ); + + expect(row._id).toBe('doc1'); + expect(row.payload).toBe('AQID'); + expect(row.plain_text).toBe('AQID'); + }); + + it('strips Convex metadata fields that should not be reported to storage', () => { + const row = applyRowContext( + toSqliteInputRow({ + _id: 'doc1', + _creationTime: 1772817606884, + _table: 'users', + _ts: 1772817606884944123n, + name: 'Alice' + }), + context + ); + + expect(row._id).toBe('doc1'); + expect(row._table).toBeUndefined(); + expect(row._ts).toBeUndefined(); + }); + + it('keeps raw JSON wire values instead of applying declared numeric field types', () => { + const row = applyRowContext( + toSqliteInputRow({ + int_value: '9007199254740991', + float_value: 1 + }), + context + ); + + expect(row.int_value).toBe('9007199254740991'); + expect(row.float_value).toBe(1); + expect(typeof row.float_value).toBe('number'); + }); +}); diff --git a/modules/module-convex/test/src/env.ts b/modules/module-convex/test/src/env.ts new file mode 100644 index 000000000..af7f7cbb6 --- /dev/null +++ b/modules/module-convex/test/src/env.ts @@ -0,0 +1,30 @@ +import { utils } from '@powersync/lib-services-framework'; +import fs from 'fs'; +import path from 'node:path'; +function obtainDefaultLocalConvexDeployKey(): string | null { + const localConfigPath = path.join(import.meta.dirname, '../../.convex/local/default/config.json'); + try { + if (!fs.existsSync(localConfigPath)) { + return null; + } + + const content = JSON.parse(fs.readFileSync(localConfigPath, 'utf8')); + return content.adminKey; + } catch (ex) { + console.warn(`Could not find local convex config in .convex`); + return null; + } +} + +export const env = utils.collectEnvironmentVariables({ + CONVEX_URL: utils.type.string.default('http://127.0.0.1:3210'), + CONVEX_DEPLOY_KEY: utils.type.string.default(obtainDefaultLocalConvexDeployKey() ?? ''), + CONVEX_DEPLOYMENT: utils.type.string.default('anonymous:anonymous-module-convex'), + MONGO_TEST_URL: utils.type.string.default('mongodb://127.0.0.1:27017/powersync_test?directConnection=true'), + PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_storage_test'), + CI: utils.type.boolean.default('false'), + SLOW_TESTS: utils.type.boolean.default('false'), + TEST_MONGO_STORAGE: utils.type.boolean.default('true'), + TEST_POSTGRES_STORAGE: utils.type.boolean.default('true'), + SHOULD_RUN_BENCHMARK: utils.type.boolean.default('false') +}); diff --git a/modules/module-convex/test/src/integration/ConvexModule.integration.test.ts b/modules/module-convex/test/src/integration/ConvexModule.integration.test.ts new file mode 100644 index 000000000..efd4cdc67 --- /dev/null +++ b/modules/module-convex/test/src/integration/ConvexModule.integration.test.ts @@ -0,0 +1,11 @@ +import { ConvexModule } from '@module/module/ConvexModule.js'; +import { describe, expect, test } from 'vitest'; +import { TEST_CONNECTION_OPTIONS } from '../test-utils/util.js'; + +describe('ConvexModule', () => { + test('Testing Connections should succeed for valid connections', async () => { + // It's not easy to test for in invalid Convex backend, since we only configure a valid backend. + const result = await ConvexModule.testConnection(TEST_CONNECTION_OPTIONS); + expect(result.connectionDescription).eq(TEST_CONNECTION_OPTIONS.deployment_url); + }); +}); diff --git a/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts b/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts new file mode 100644 index 000000000..fa515882d --- /dev/null +++ b/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts @@ -0,0 +1,166 @@ +import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; +import type { DatabaseSchema } from '@powersync/service-types'; +import { randomUUID } from 'crypto'; +import { describe, expect, test } from 'vitest'; +import { env } from '../env.js'; +import { ConvexStreamTestContext } from '../test-utils/ConvexStreamTestContext.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_CONNECTION_OPTIONS } from '../test-utils/util.js'; + +function normalizeSchemaForSnapshot(schema: DatabaseSchema[]): DatabaseSchema[] { + const snapshottedTables = new Set(['lists', 'todos']); + + return schema.map((database) => ({ + ...database, + tables: [...database.tables] + .filter((table) => snapshottedTables.has(table.name)) + .sort((a, b) => a.name.localeCompare(b.name)) + .map((table) => ({ + columns: [...table.columns] + .sort((a, b) => a.name.localeCompare(b.name)) + .map((column) => ({ + internal_type: column.internal_type, + name: column.name, + pg_type: column.pg_type, + sqlite_type: column.sqlite_type, + type: column.type + })), + name: table.name + })) + })); +} + +describe.skipIf(!env.CONVEX_DEPLOY_KEY)('ConvexStream ConvexRouteAPIAdapter tests', function () { + test('json_schemas lists schema-defined tables without documents', async () => { + /** + * The Convex stream uses json_schemas for initial wildcard table expansion. + * If json_schemas only listed tables that already contain documents, then a + * wildcard sync rule could miss an empty-but-schema-defined table at the + * initial snapshot boundary and would need a later inline snapshot when the + * first delta appears. This verifies that table names are available even + * before any documents exist, so new table deltas can be applied directly. + */ + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY.factory, {}); + + const schemas = await context.connectionManager.client.getJsonSchemas(); + const tableNames = schemas.tables.map((table) => table.tableName); + + expect(tableNames).toContain('schema_only_probe'); + + const page = await context.connectionManager.client.listSnapshot({ tableName: 'schema_only_probe' }); + expect(page.values).toHaveLength(0); + }); + + test('retrieves the testing Convex schema in the expected service schema format', async () => { + /** + * It seems like Convex requires the table to contain populated columns in order to report + * valuable column information. + */ + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY.factory, {}); + // Create an item + await context.backend.client.mutation(context.backend.api.lists.createBatch, { + lists: [ + { + name: 'a string name', + uuid: randomUUID(), + archived: 1, + attributes: { color: 'red' }, + created_at: new Date().toISOString(), + owner: 'an owner', + owner_id: randomUUID(), + settings: { + color: 'red', + is_public: true, + theme: 'theme' + }, + tags: ['one', 'two'] + } + ] + }); + await using adapter = new ConvexRouteAPIAdapter(TEST_CONNECTION_OPTIONS); + const schema = await adapter.getConnectionSchema(); + expect(schema).toMatchObject( + expect.arrayContaining([ + { + name: 'convex', + tables: expect.arrayContaining([ + { + name: 'lists', + columns: expect.arrayContaining([ + { + internal_type: 'string', + name: '_id', + pg_type: 'string', + sqlite_type: 2, + type: 'id' + }, + { + internal_type: 'float64', + name: 'archived', + pg_type: 'float64', + sqlite_type: 8, + type: 'number' + }, + { + internal_type: 'object', + name: 'attributes', + pg_type: 'object', + sqlite_type: 2, + type: 'object' + }, + { + internal_type: 'string', + name: 'created_at', + pg_type: 'string', + sqlite_type: 2, + type: 'string' + }, + { + internal_type: 'string', + name: 'name', + pg_type: 'string', + sqlite_type: 2, + type: 'string' + }, + { + internal_type: 'string', + name: 'owner', + pg_type: 'string', + sqlite_type: 2, + type: 'string' + }, + { + internal_type: 'string', + name: 'owner_id', + pg_type: 'string', + sqlite_type: 2, + type: 'string' + }, + { + internal_type: 'object', + name: 'settings', + pg_type: 'object', + sqlite_type: 2, + type: 'object' + }, + { + internal_type: 'array', + name: 'tags', + pg_type: 'array', + sqlite_type: 2, + type: 'array' + }, + { + internal_type: 'string', + name: 'uuid', + pg_type: 'string', + sqlite_type: 2, + type: 'string' + } + ]) + } + ]) + } + ]) + ); + }); +}); diff --git a/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts b/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts new file mode 100644 index 000000000..a6401294d --- /dev/null +++ b/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts @@ -0,0 +1,469 @@ +import { randomUUID } from 'node:crypto'; + +import { METRICS_HELPER, removeOp } from '@powersync/service-core-tests'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { ReplicationMetric } from '@powersync/service-types'; + +import { describe, expect, test, vi } from 'vitest'; +import { env } from '../env.js'; +import { ConvexStreamTestContext } from '../test-utils/ConvexStreamTestContext.js'; +import { describeWithStorage, StorageVersionTestContext } from '../test-utils/util.js'; + +import schema from '@testing-convex/schema.js'; + +type ListsData = typeof schema.tables.lists.validator.type; +type TodosData = Omit; + +const BASIC_SYNC_RULES = ` +bucket_definitions: + global: + data: + - SELECT uuid as id, name FROM "lists" +`; + +describe.skipIf(!env.CONVEX_DEPLOY_KEY)('ConvexStream integration tests', function () { + describeWithStorage({ timeout: 120_000 }, function ({ factory, storageVersion }) { + defineConvexStreamTests(factory, storageVersion); + }); +}); + +function defineConvexStreamTests( + factory: StorageVersionTestContext['factory'], + storageVersion: StorageVersionTestContext['storageVersion'] +) { + test('Initial snapshot sync', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const testData = await createList(context, { name: 'snapshot-list' }); + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + + await context.replicateSnapshot(); + + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([convexPutOp('lists', syncedId(testData), listBucketRow(testData))]); + expect(endRowCount - startRowCount).toEqual(1); + }); + + test('Replicate basic values', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + await context.replicateSnapshot(); + + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + + const testData = await createList(context, { name: 'streamed-list' }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([convexPutOp('lists', syncedId(testData), listBucketRow(testData))]); + + await vi.waitFor(async () => { + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + expect(endRowCount - startRowCount).toEqual(1); + expect(endTxCount - startTxCount).toEqual(1); + }); + }); + + test('Counts Convex batch mutations as single replicated transactions', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + await context.replicateSnapshot(); + + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + + const batchOne = await createLists(context, ['batch-one-a', 'batch-one-b']); + const single = await createList(context, { name: 'single-mutation' }); + const batchTwo = await createLists(context, ['batch-two-a', 'batch-two-b']); + + const data = await context.getBucketData('global[]'); + expect(data).toHaveLength(5); + expect(data.slice(0, 2)).toEqual(expectListOps(batchOne)); + expect(data[2]).toMatchObject(convexPutOp('lists', syncedId(single), listBucketRow(single))); + expect(data.slice(3, 5)).toEqual(expectListOps(batchTwo)); + + await vi.waitFor(async () => { + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + expect(endRowCount - startRowCount).toEqual(5); + expect(endTxCount - startTxCount).toEqual(3); + }); + }); + + test('Replicated rows in transactions are correctly ordered', async () => { + /** + * out-of-ordered operations on different rows are not an issue. + * out-of-ordered operations on the same row could be large issues. + */ + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + await context.replicateSnapshot(); + + const backendResult = await context.backend.client.mutation(context.backend.api.lists.testUpdateMultipleTimes, {}); + + const data = await context.getBucketData('global[]'); + // Convex seems to squash the deltas, so we don't get a delta for each update which happened in the backend + // The deleted row is reported as _deleted:true, which causes us not to replicate it + expect(data.length).eq(backendResult.listIds.length); + + // All the put ops should contain a value for name which ends with a-b-c + expect( + data.every((item) => { + const parsed = JSON.parse(item.data!); + return parsed.name.endsWith('a-b-c'); + }) + ).true; + }); + + test('Replicate row updates', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const testData = await createList(context, { name: 'initial-list' }); + await context.replicateSnapshot(); + + const updatedData = { + ...testData, + name: 'updated-list' + }; + await context.backend.client.mutation(context.backend.api.lists.updateName, { + uuid: testData.uuid, + name: updatedData.name + }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([ + convexPutOp('lists', syncedId(testData), listBucketRow(testData)), + convexPutOp('lists', syncedId(updatedData), listBucketRow(updatedData)) + ]); + }); + + test('Replicate row deletions', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const testData = await createList(context, { name: 'deleted-list' }); + await context.replicateSnapshot(); + + await context.backend.client.mutation(context.backend.api.lists.deleteItem, { + uuid: testData.uuid + }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([ + convexPutOp('lists', syncedId(testData), listBucketRow(testData)), + removeOp('lists', syncedId(testData)) + ]); + }); + + test('Replicate matched wildcard tables in sync rules', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT uuid as id, * FROM "%" +`); + + const list = await createList(context, { name: 'parent-list' }); + + const sampleTodo: TodosData = { + uuid: randomUUID(), + description: 'A test', + list_uuid: list.uuid, + title: 'Typed snapshot todo', + notes: 'notes', + category: 'testing', + priority: 3, + points: 9_007_199_254_740_991n, + estimated_hours: 12.5, + progress_percentage: 87.25, + is_urgent: true, + is_private: false, + has_attachments: true, + attachment_data: Uint8Array.from([0, 1, 255]).buffer, + tags: ['convex', 'sqlite'], + assigned_users: ['alice', 'bob'], + details: { label: 'detail', count: 7, nested: { enabled: true } }, + metadata: { count: 3, enabled: true, nested: { label: 'metadata' } }, + custom_fields: { score: 10, ready: true, owner: 'tester' }, + status: 'in_progress', + difficulty: 'hard', + explicit_null: null, + archived_at: '2026-05-14T00:00:00.000Z', + deleted_by: 'debugger' + }; + + const todo = await createTodo(context, sampleTodo); + + await context.replicateSnapshot(); + + const streamedList = await createList(context, { name: 'streamed-list' }); + const streamedTodo = await createTodo(context, { + ...sampleTodo, + list_uuid: streamedList.uuid, + description: 'streamed-todo' + }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([ + convexOp('PUT', 'lists', syncedId(list)), + convexOp('PUT', 'todos', syncedId(todo)), + convexOp('PUT', 'lists', syncedId(streamedList)), + convexOp('PUT', 'todos', syncedId(streamedTodo)) + ]); + + // Now verifying the to sqlite typings + const firstTodoOp = data[1]; + const parsedData = JSONBig.parse(firstTodoOp.data!) as Record; + + expect(typeof parsedData.uuid).eq('string'), + expect(typeof parsedData.description).eq('string'), + expect(typeof parsedData.list_uuid).eq('string'), + expect(typeof parsedData.description).eq('string'), + expect(typeof parsedData.list_id).eq('string'), + expect(typeof parsedData.title).eq('string'), + expect(typeof parsedData.notes).eq('string'), + expect(typeof parsedData.category).eq('string'), + expect(typeof parsedData.priority).eq('number'), // This is a regular v.number() + expect(typeof parsedData.points).eq('string'), // int64 values use the raw Convex JSON wire type + expect(typeof parsedData.estimated_hours).eq('number'), + expect(typeof parsedData.progress_percentage).eq('number'), + expect(typeof parsedData.is_urgent).eq('bigint'), // boolean + expect(typeof parsedData.is_private).eq('bigint'), // boolean + expect(typeof parsedData.has_attachments).eq('bigint'), // boolean + expect(typeof parsedData.attachment_data).eq('string'), // base64 bytes + expect(typeof parsedData.tags).eq('string'), // array + expect(typeof parsedData.assigned_users).eq('string'), //array + expect(typeof parsedData.details).eq('string'), + expect(typeof parsedData.explicit_null).eq('object'), // null + expect(typeof parsedData.archived_at).eq('string'); // '2026-05-14T00:00:00.000Z', + }); + + /** + * It seems like the json-schema's route will not return JSON schema values for table + * columns if not populated value exists for the row yet. + * This test simulates this behaviour. We do an initial replication and start streaming + * before adding any todo records. We create a todo record while streaming. + */ + test('Replicate values when initial snapshot did not include data', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT uuid as id, * FROM lists + - SELECT uuid as id, * FROM todos + +`); + + const list = await createList(context, { name: 'parent-list' }); + + // This one does not include some optional column values. + // Replicating this will cause the schema for todos to be queried and cached. + const firstTodo = await createTodo(context, { + description: 'the first one, to be recorded in initial snapshot', + list_uuid: list.uuid, + uuid: randomUUID() + }); + + // This starts the replication of the initial snapshot and also then streams + await context.replicateSnapshot(); + + await new Promise((r) => setTimeout(r, 1_000)); + + // Create a new todo record + const sampleTodo: TodosData = { + uuid: randomUUID(), + description: 'A test', + list_uuid: list.uuid, + title: 'Typed snapshot todo', + notes: 'notes', + category: 'testing', + priority: 3, + points: 9_007_199_254_740_991n, + estimated_hours: 12.5, + progress_percentage: 87.25, + is_urgent: true, + is_private: false, + has_attachments: true, + attachment_data: Uint8Array.from([0, 1, 255]).buffer, + tags: ['convex', 'sqlite'], + assigned_users: ['alice', 'bob'], + details: { label: 'detail', count: 7, nested: { enabled: true } }, + metadata: { count: 3, enabled: true, nested: { label: 'metadata' } }, + custom_fields: { score: 10, ready: true, owner: 'tester' }, + status: 'in_progress', + difficulty: 'hard', + explicit_null: null, + archived_at: '2026-05-14T00:00:00.000Z', + deleted_by: 'debugger' + }; + + const streamedList = await createList(context, { name: 'streamed-list' }); + const streamedTodo = await createTodo(context, { + ...sampleTodo, + list_uuid: streamedList.uuid, + description: 'streamed-todo' + }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([ + convexOp('PUT', 'lists', syncedId(list)), + convexOp('PUT', 'todos', syncedId(firstTodo)), + convexOp('PUT', 'lists', syncedId(streamedList)), + convexOp('PUT', 'todos', syncedId(streamedTodo)) + ]); + + // Now verifying the to sqlite typings + const firstTodoOp = data[3]; + const parsedData = JSONBig.parse(firstTodoOp.data!) as Record; + + expect(typeof parsedData.uuid).eq('string'), + expect(typeof parsedData.description).eq('string'), + expect(typeof parsedData.list_uuid).eq('string'), + expect(typeof parsedData.description).eq('string'), + expect(typeof parsedData.list_id).eq('string'), + expect(typeof parsedData.title).eq('string'), + expect(typeof parsedData.notes).eq('string'), + expect(typeof parsedData.category).eq('string'), + expect(typeof parsedData.priority).eq('number'), // This is a regular v.number() + expect(typeof parsedData.points).eq('string'), // int64 values use the raw Convex JSON wire type + expect(typeof parsedData.estimated_hours).eq('number'), + expect(typeof parsedData.progress_percentage).eq('number'), + expect(typeof parsedData.is_urgent).eq('bigint'), // boolean + expect(typeof parsedData.is_private).eq('bigint'), // boolean + expect(typeof parsedData.has_attachments).eq('bigint'), // boolean + expect(typeof parsedData.attachment_data).eq('string'), // base64 bytes + expect(typeof parsedData.tags).eq('string'), // array + expect(typeof parsedData.assigned_users).eq('string'), //array + expect(typeof parsedData.details).eq('string'), + expect(typeof parsedData.explicit_null).eq('object'), // null + expect(typeof parsedData.archived_at).eq('string'); // '2026-05-14T00:00:00.000Z', + }); + + test('Replication for tables not in the sync rules are ignored', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const list = await createList(context, { name: 'synced-parent' }); + await context.replicateSnapshot(); + + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + + // Basic sync streams here don't replicate todo rows + await createTodo(context, { + uuid: randomUUID(), + list_uuid: list.uuid, + description: 'ignored-todo' + }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([convexPutOp('lists', syncedId(list), listBucketRow(list))]); + + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + expect(endRowCount - startRowCount).toEqual(0); + expect(endTxCount - startTxCount).toEqual(0); + }); + + test('Table matching is case sensitive', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT uuid as id, name FROM "Lists" +`); + + await context.replicateSnapshot(); + + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + + await createList(context, { name: 'case-sensitive-list' }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([]); + + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + expect(endRowCount - startRowCount).toEqual(0); + expect(endTxCount - startTxCount).toEqual(0); + }); +} + +async function createList(context: ConvexStreamTestContext, options: { name: string }) { + const [list] = await createLists(context, [options.name]); + return list; +} + +async function createLists(context: ConvexStreamTestContext, names: string[]) { + const lists = names.map((name) => ({ + uuid: randomUUID(), + name + })); + + const ids = await context.backend.client.mutation(context.backend.api.lists.createBatch, { + lists + }); + + return lists.map((list, index) => ({ + id: ids[index]!, + uuid: list.uuid, + name: list.name + })); +} + +async function createTodo(context: ConvexStreamTestContext, options: TodosData) { + const [id] = await context.backend.client.mutation(context.backend.api.todos.createBatch, { + todos: [options] + }); + + return { + id, + ...options + }; +} + +function syncedId(document: { uuid: string }) { + return document.uuid; +} + +function listBucketRow(document: { uuid: string; name: string }) { + return { + id: document.uuid, + name: document.name + }; +} + +/** + * The order of ops do not match that done in a single mutation, this is due to out-of-order (internal) updates. + */ +function expectListOps(lists: Array<{ uuid: string; name: string }>) { + return expect.arrayContaining( + lists.map((list) => expect.objectContaining(convexPutOp('lists', syncedId(list), listBucketRow(list)))) + ); +} + +function convexPutOp(table: string, id: string, data: Record) { + return { + ...convexOp('PUT', table, id), + data: JSONBig.stringify(data) + }; +} + +function convexOp(op: 'PUT' | 'REMOVE', table: string, id: string) { + return { + op, + object_type: table, + object_id: id + }; +} diff --git a/modules/module-convex/test/src/integration/resuming_snapshots.integration.test.ts b/modules/module-convex/test/src/integration/resuming_snapshots.integration.test.ts new file mode 100644 index 000000000..e014502e1 --- /dev/null +++ b/modules/module-convex/test/src/integration/resuming_snapshots.integration.test.ts @@ -0,0 +1,154 @@ +import { randomUUID } from 'node:crypto'; +import { describe, expect, test, vi } from 'vitest'; +import { env } from '../env.js'; +import { ConvexStreamTestContext } from '../test-utils/ConvexStreamTestContext.js'; +import { describeWithStorage, StorageVersionTestContext } from '../test-utils/util.js'; + +describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () { + describeWithStorage({ timeout: 240_000 }, function ({ factory, storageVersion }) { + test('resuming initial replication', async () => { + // The initial replication will be split into + // 1 - The first 1001 list records + // 2 - A batch of 1000 todo records + // 3 - Another 1000 batch of todo records + // We interupt the todos after batch 2 (in the middle of 3) + await testResumingReplication(factory, storageVersion, 2500); + }); + }); +}); + +async function testResumingReplication( + factory: StorageVersionTestContext['factory'], + storageVersion: number, + stopAfter: number +) { + // This tests interrupting and then resuming initial replication. + // We interrupt replication after lists has fully replicated, and + // todos has partially replicated. + // This test relies on interval behavior that is not 100% deterministic: + // 1. We attempt to abort initial replication once a certain number of + // rows have been replicated, but this is not exact. Our only requirement + // is that we have not fully replicated todos yet. + // 2. Order of replication is not deterministic, so which specific rows + // have been / have not been replicated at that point is not deterministic. + // We do allow for some variation in the test results to account for this. + + await using context = await ConvexStreamTestContext.open(factory, { + storageVersion + }); + + await context.updateSyncRules(/* yaml */ `bucket_definitions: + global: + data: + - SELECT uuid as id, * FROM lists + - SELECT uuid as id, * FROM todos `); + + const { backend } = context; + + // Seed the database + // Max number of mutations is batch size supported is 8192 + // Maximum number of reads is 4096 in a single mutation + await backend.client.mutation(backend.api.lists.createBatch, { + lists: Array.from({ length: 1_000 }).map((_, index) => ({ + uuid: randomUUID(), + name: `list-${index}` + })) + }); + // Delay to avoid TooManyWrites error from Convex + await new Promise((r) => setTimeout(r, 1_000)); + + // create a single row to track deleted items + const deletableListId = randomUUID(); + await backend.client.mutation(backend.api.lists.createBatch, { + lists: [ + { + uuid: deletableListId, + name: 'parent' + } + ] + }); + // create a single one with a tracked uuid for relationships + const relationalListId = randomUUID(); + await backend.client.mutation(backend.api.lists.createBatch, { + lists: [ + { + uuid: relationalListId, + name: 'parent' + } + ] + }); + await backend.client.mutation(backend.api.todos.createBatch, { + todos: Array.from({ length: 2_000 }).map((_, index) => ({ + uuid: randomUUID(), + list_uuid: relationalListId, + description: `todo-${index}` + })) + }); + // Delay to avoid TooManyWrites error from Convex + await new Promise((r) => setTimeout(r, 1_000)); + // twice in order to get many todos (see limits above) + await backend.client.mutation(backend.api.todos.createBatch, { + todos: Array.from({ length: 2_000 }).map((_, index) => ({ + uuid: randomUUID(), + list_uuid: relationalListId, + description: `todo-${index}` + })) + }); + // Delay to avoid TooManyWrites error from Convex + await new Promise((r) => setTimeout(r, 1_000)); + + let stopped = new Promise((resolve) => { + context.storage!.registerListener({ + batchStarted: (batch) => { + //register a pre-emptive spy in order to halt writes + const original = batch.save; + let savedCount = 0; + vi.spyOn(batch, 'save').mockImplementation(async (param) => { + if (savedCount >= stopAfter) { + // This interrupts initial replication + // don't await this since awaiting will cause a deadlock + context.dispose(); + resolve(); + throw new Error('Stopping now'); + } + savedCount++; + return original.call(batch, param); + }); + } + }); + }); + + const replicationError = context.replicateSnapshot().catch((error) => error); + + await stopped; + await replicationError; + + // Add delete a row which has already been replicated + await backend.client.mutation(backend.api.lists.deleteItem, { + uuid: deletableListId + }); + + await using context2 = await ConvexStreamTestContext.open(factory, { + doNotClear: true, + storageVersion + }); + + // Spy on the list-snapshot endpoint + const snapshotSpy = vi.spyOn(context2.connectionManager.client, 'listSnapshot'); + + await context2.loadNextSyncRules(); + await context2.replicateSnapshot(); + + // The second replication should have called list snapshot for the todos table, at a specific cursor value (resuming) + expect(snapshotSpy).called; + const firstCall = snapshotSpy.mock.calls[0]; + expect(firstCall).toBeDefined(); + expect(firstCall[0].tableName).eq('todos'); + expect(firstCall[0].cursor).toBeDefined(); //it resumed within the table + + // Check the final bucket data + const data = await context2.getBucketData('global[]', undefined, {}); + expect(data.length).eq(5003); // Puts: 1000 + 1 + 1 + 2000 + 2000, REMOVE: 1 + expect(data.filter((item) => item.op == 'PUT').length).eq(5002); + expect(data.filter((item) => item.op == 'REMOVE').length).eq(1); +} diff --git a/modules/module-convex/test/src/setup.ts b/modules/module-convex/test/src/setup.ts new file mode 100644 index 000000000..b14ebcec9 --- /dev/null +++ b/modules/module-convex/test/src/setup.ts @@ -0,0 +1,11 @@ +import { container } from '@powersync/lib-services-framework'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; +import { beforeAll, beforeEach } from 'vitest'; + +beforeAll(async () => { + container.registerDefaults(); +}); + +beforeEach(async () => { + METRICS_HELPER.resetMetrics(); +}); diff --git a/modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts b/modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts new file mode 100644 index 000000000..9872e83e8 --- /dev/null +++ b/modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts @@ -0,0 +1,118 @@ +import { ConvexConnectionManager } from '@module/replication/ConvexConnectionManager.js'; +import { ConvexStream, ConvexStreamOptions } from '@module/replication/ConvexStream.js'; +import { logger } from '@powersync/lib-services-framework'; +import { + BucketStorageFactory, + createCoreReplicationMetrics, + initializeCoreReplicationMetrics, + LEGACY_STORAGE_VERSION, + storage +} from '@powersync/service-core'; +import { AbstractStreamTestContext, METRICS_HELPER } from '@powersync/service-core-tests'; +import { clearTestDb, connectConvex, TEST_CONNECTION_OPTIONS, TestConvexConnection } from './util.js'; + +export class ConvexStreamTestContext extends AbstractStreamTestContext { + protected _stream?: ConvexStream; + + /** + * Tests operating on the stream need to configure the stream and manage asynchronous + * replication, which gets a little tricky. + * + * This configures all the context, and tears it down afterwards. + */ + static async open( + factory: (options: storage.TestStorageOptions) => Promise, + options?: { + doNotClear?: boolean; + storageVersion?: number; + streamOptions?: Partial; + clearSource?: boolean; + } + ) { + const f = await factory({ doNotClear: options?.doNotClear }); + const connectionManager = new ConvexConnectionManager(TEST_CONNECTION_OPTIONS); + + const convexBackend = connectConvex(); + + if (options?.clearSource ?? !options?.doNotClear) { + await clearTestDb(convexBackend); + } + + const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION; + + return new ConvexStreamTestContext(f, connectionManager, convexBackend, options?.streamOptions, storageVersion); + } + + constructor( + public factory: BucketStorageFactory, + public connectionManager: ConvexConnectionManager, + public backend: TestConvexConnection, + protected streamOptions?: Partial, + protected storageVersion: number = LEGACY_STORAGE_VERSION + ) { + super(); + createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + } + + get connectionTag() { + return this.connectionManager.connectionTag; + } + + get stream() { + if (this.storage == null) { + throw new Error('updateSyncRules() first'); + } + if (this._stream) { + return this._stream; + } + const options: ConvexStreamOptions = { + storage: this.storage, + metrics: METRICS_HELPER.metricsEngine, + connections: this.connectionManager, + abortSignal: this.abortController.signal, + ...this.streamOptions + }; + this._stream = new ConvexStream(options); + return this._stream!; + } + + protected async _dispose(): Promise { + await this.connectionManager.end(); + } + + protected triggerReplication(): Promise { + return this.stream.replicate(); + } + + protected waitForInitialSnapshot(): Promise { + return this.stream.waitForInitialSnapshot(); + } + + async getClientCheckpoint(options?: { timeout?: number }): Promise { + const start = Date.now(); + + const lsn = await this.connectionManager.client.getHeadCursor(); + await this.connectionManager.client.createWriteCheckpointMarker(); + + // This old API needs a persisted checkpoint id. + // Since we don't use LSNs anymore, the only way to get that is to wait. + + const timeout = options?.timeout ?? 50_000; + + logger.info(`Waiting for LSN checkpoint: ${lsn}`); + while (Date.now() - start < timeout) { + const storage = await this.factory.getActiveStorage(); + const cp = await storage?.getCheckpoint(); + + if (cp?.lsn != null && cp.lsn >= lsn) { + logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`); + return cp.checkpoint; + } + + await new Promise((resolve) => setTimeout(resolve, 5)); + } + + throw new Error('Timeout while waiting for checkpoint'); + } +} diff --git a/modules/module-convex/test/src/test-utils/util.ts b/modules/module-convex/test/src/test-utils/util.ts new file mode 100644 index 000000000..5bc275d4b --- /dev/null +++ b/modules/module-convex/test/src/test-utils/util.ts @@ -0,0 +1,104 @@ +import * as types from '@module/types/types.js'; +import { api } from '@testing-convex/_generated/api.js'; +import { ConvexHttpClient } from 'convex/browser'; + +import { SUPPORTED_STORAGE_VERSIONS, TestStorageConfig, TestStorageFactory } from '@powersync/service-core'; +import { describe, TestOptions } from 'vitest'; +import { env } from '../env.js'; + +export type TestConvexConnection = { + client: ConvexHttpClient; + api: typeof api; +}; + +export const TEST_URI = env.CONVEX_URL; + +export const INITIALIZED_MONGO_STORAGE_FACTORY: TestStorageConfig = { + tableIdStrings: false, + factory: async (options) => { + const mongo_storage = await import('@powersync/service-module-mongodb-storage'); + const config = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ + url: env.MONGO_TEST_URL, + isCI: env.CI + }); + return config.factory(options); + } +}; + +export const INITIALIZED_POSTGRES_STORAGE_FACTORY: TestStorageConfig = { + tableIdStrings: true, + factory: async (options) => { + const postgres_storage = await import('@powersync/service-module-postgres-storage'); + const config = postgres_storage.test_utils.postgresTestSetup({ + url: env.PG_STORAGE_TEST_URL + }); + return config.factory(options); + } +}; + +const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; + +export interface StorageVersionTestContext { + factory: TestStorageFactory; + storageVersion: number; +} + +export function describeWithStorage( + options: TestOptions & { storageVersions?: number[] }, + fn: (context: StorageVersionTestContext) => void +) { + const storageVersions = options.storageVersions ?? TEST_STORAGE_VERSIONS; + const describeFactory = (storageName: string, config: TestStorageConfig) => { + describe(`${storageName} storage`, options, function () { + for (const storageVersion of storageVersions) { + describe(`storage v${storageVersion}`, function () { + fn({ + factory: config.factory, + storageVersion + }); + }); + } + }); + }; + + if (env.TEST_MONGO_STORAGE) { + describeFactory('mongodb', INITIALIZED_MONGO_STORAGE_FACTORY); + } + + if (env.TEST_POSTGRES_STORAGE) { + describeFactory('postgres', INITIALIZED_POSTGRES_STORAGE_FACTORY); + } +} + +export const RAW_TEST_CONNECTION_OPTIONS: types.ConvexConnectionConfig = { + type: 'convex', + deploy_key: env.CONVEX_DEPLOY_KEY, + deployment_url: env.CONVEX_URL +} as const; + +export const TEST_CONNECTION_OPTIONS = types.resolveConvexConnectionConfig(RAW_TEST_CONNECTION_OPTIONS); + +export function connectConvex(): TestConvexConnection { + return { + client: new ConvexHttpClient(env.CONVEX_URL), + api + }; +} + +export async function clearTestDb(connection: TestConvexConnection) { + const { api, client } = connection; + + // Delete all lists + let deletedCount = 0; + console.info(`Clearing Convex DB`); + do { + deletedCount = await client.mutation(api.lists.deleteBatch, {}); + console.info(`Cleared ${deletedCount} lists`); + } while (deletedCount > 0); + + deletedCount = 0; + do { + deletedCount = await client.mutation(api.todos.deleteBatch, {}); + console.info(`Cleared ${deletedCount} todos`); + } while (deletedCount > 0); +} diff --git a/modules/module-convex/test/src/types.test.ts b/modules/module-convex/test/src/types.test.ts new file mode 100644 index 000000000..94627e550 --- /dev/null +++ b/modules/module-convex/test/src/types.test.ts @@ -0,0 +1,71 @@ +import { CONVEX_CONNECTION_TYPE, normalizeConnectionConfig } from '@module/types/types.js'; +import { describe, expect, it } from 'vitest'; + +describe('Convex connection config', () => { + it('normalizes defaults', () => { + const config = normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'https://example.convex.cloud', + deploy_key: 'secret-key' + }); + + expect(config.id).toBe('default'); + expect(config.tag).toBe('default'); + expect(config.polling_interval_ms).toBe(1000); + expect(config.request_timeout_ms).toBe(60_000); + expect(config.deployment_url).toBe('https://example.convex.cloud'); + }); + + it('normalizes custom request timeout', () => { + const config = normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'https://example.convex.cloud', + deploy_key: 'secret-key', + request_timeout_ms: 30_000 + }); + + expect(config.request_timeout_ms).toBe(30_000); + }); + + it('throws for invalid URL', () => { + expect(() => + normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'not-a-url', + deploy_key: 'secret-key' + }) + ).toThrow(); + }); + + it('throws for empty deploy key', () => { + expect(() => + normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'https://example.convex.cloud', + deploy_key: '' + }) + ).toThrow(); + }); + + it.each([-1, 0, Number.NaN, Number.POSITIVE_INFINITY])('throws for invalid polling_interval_ms: %s', (value) => { + expect(() => + normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'https://example.convex.cloud', + deploy_key: 'secret-key', + polling_interval_ms: value + }) + ).toThrow('polling_interval_ms must be a positive finite number'); + }); + + it.each([-1, 0, Number.NaN, Number.POSITIVE_INFINITY])('throws for invalid request_timeout_ms: %s', (value) => { + expect(() => + normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'https://example.convex.cloud', + deploy_key: 'secret-key', + request_timeout_ms: value + }) + ).toThrow('request_timeout_ms must be a positive finite number'); + }); +}); diff --git a/modules/module-convex/test/tsconfig.json b/modules/module-convex/test/tsconfig.json new file mode 100644 index 000000000..1e518f4b0 --- /dev/null +++ b/modules/module-convex/test/tsconfig.json @@ -0,0 +1,31 @@ +{ + "extends": "../../../tsconfig.tests.json", + "compilerOptions": { + "rootDir": "..", + "noEmit": true, + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true, + "paths": { + "@/*": ["../../../packages/service-core/src/*"], + "@module/*": ["../src/*"], + "@core-tests/*": ["../../../packages/service-core/test/src/*"], + "@testing-convex/*": ["../convex/*"] + } + }, + "include": ["src", "../convex/**/*.ts"], + "references": [ + { + "path": "../" + }, + { + "path": "../../../packages/service-core-tests" + }, + { + "path": "../../module-mongodb-storage" + }, + { + "path": "../../module-postgres-storage" + } + ] +} diff --git a/modules/module-convex/tsconfig.json b/modules/module-convex/tsconfig.json new file mode 100644 index 000000000..91db1044f --- /dev/null +++ b/modules/module-convex/tsconfig.json @@ -0,0 +1,28 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [ + { + "path": "../../packages/types" + }, + { + "path": "../../packages/sync-rules" + }, + { + "path": "../../packages/service-core" + }, + { + "path": "../../packages/jsonbig" + }, + { + "path": "../../libs/lib-services" + } + ] +} diff --git a/modules/module-convex/vitest.config.ts b/modules/module-convex/vitest.config.ts new file mode 100644 index 000000000..9f1670135 --- /dev/null +++ b/modules/module-convex/vitest.config.ts @@ -0,0 +1,12 @@ +import path from 'node:path'; +import { serviceIntegrationTestConfig } from '../test_config'; + +const baseConfig = serviceIntegrationTestConfig(__dirname); +baseConfig.resolve = { + ...baseConfig.resolve, + alias: { + ...baseConfig.resolve?.alias, + '@testing-convex': path.resolve(__dirname, 'convex') + } +}; +export default baseConfig; diff --git a/packages/schema/package.json b/packages/schema/package.json index e2f4df09b..67e1320ae 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -32,6 +32,7 @@ "@powersync/service-module-postgres-storage": "workspace:*", "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-convex": "workspace:*", "@powersync/service-module-mysql": "workspace:*", "@powersync/service-module-mssql": "workspace:*", "@powersync/service-types": "workspace:*", diff --git a/packages/schema/src/scripts/compile-json-schema.ts b/packages/schema/src/scripts/compile-json-schema.ts index 2f810e3ac..9d68b053b 100644 --- a/packages/schema/src/scripts/compile-json-schema.ts +++ b/packages/schema/src/scripts/compile-json-schema.ts @@ -1,3 +1,4 @@ +import { ConvexConnectionConfig } from '@powersync/service-module-convex/types'; import { MongoStorageConfig } from '@powersync/service-module-mongodb-storage/types'; import { MongoConnectionConfig } from '@powersync/service-module-mongodb/types'; import { MSSQLConnectionConfig } from '@powersync/service-module-mssql/types'; @@ -21,7 +22,8 @@ const mergedDataSourceConfig = configFile.genericDataSourceConfig .or(MongoConnectionConfig) .or(MSSQLConnectionConfig) .or(MySQLConnectionConfig) - .or(PostgresConnectionConfig); + .or(PostgresConnectionConfig) + .or(ConvexConnectionConfig); const mergedStorageConfig = configFile.GenericStorageConfig.or(PostgresStorageConfig).or(MongoStorageConfig); diff --git a/packages/schema/tsconfig.json b/packages/schema/tsconfig.json index a56435d55..68c986e19 100644 --- a/packages/schema/tsconfig.json +++ b/packages/schema/tsconfig.json @@ -26,6 +26,9 @@ { "path": "../../modules/module-mysql" }, + { + "path": "../../modules/module-convex" + }, { "path": "../../modules/module-mssql" } diff --git a/packages/service-core-tests/package.json b/packages/service-core-tests/package.json index f1491e9c8..37c057999 100644 --- a/packages/service-core-tests/package.json +++ b/packages/service-core-tests/package.json @@ -14,6 +14,7 @@ "clean": "rm -rf ./dist && tsc -b --clean" }, "dependencies": { + "@powersync/lib-services-framework": "workspace:^", "@powersync/service-core": "workspace:^", "@powersync/service-jsonbig": "workspace:^", "@powersync/service-sync-rules": "workspace:^" diff --git a/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts b/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts new file mode 100644 index 000000000..33265f124 --- /dev/null +++ b/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts @@ -0,0 +1,179 @@ +import { ReplicationAbortedError } from '@powersync/lib-services-framework'; +import { + BucketStorageFactory, + InternalOpId, + settledPromise, + storage, + SyncRulesBucketStorage, + unsettledPromise, + updateSyncRulesFromYaml +} from '@powersync/service-core'; +import { StorageDataHelpers } from './StorageDataHelpers.js'; +import { bucketRequest } from './general-utils.js'; +import { fromAsync } from './stream_utils.js'; + +export abstract class AbstractStreamTestContext implements AsyncDisposable { + protected abortController = new AbortController(); + protected syncRulesContent?: storage.PersistedSyncRulesContent; + public storage?: SyncRulesBucketStorage; + protected settledReplicationPromise?: Promise>; + + abstract get factory(): BucketStorageFactory; + protected abstract get storageVersion(): number; + + async [Symbol.asyncDispose]() { + await this.dispose(); + } + + protected abstract _dispose(): Promise; + + async dispose() { + this.abortController.abort(); + try { + await this.settledReplicationPromise; + await this._dispose(); + await this.factory?.[Symbol.asyncDispose](); + } catch (e) { + // Throwing here may result in SuppressedError. The underlying errors often don't show up + // in the test output, so we log it here. + // If we could get vitest to log SuppressedError.error and SuppressedError.suppressed, we + // could remove this. + console.error('Error during ConvexStreamTestContext dispose', e); + throw e; + } + } + + async updateSyncRules(content: string) { + const syncRules = await this.factory.updateSyncRules( + updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }) + ); + this.syncRulesContent = syncRules; + this.storage = this.factory.getInstance(syncRules); + return this.storage!; + } + + async loadNextSyncRules() { + const syncRules = await this.factory.getNextSyncRulesContent(); + if (syncRules == null) { + throw new Error(`Next sync rules not available`); + } + + this.syncRulesContent = syncRules; + this.storage = this.factory.getInstance(syncRules); + return this.storage!; + } + + async loadActiveSyncRules() { + const syncRules = await this.factory.getActiveSyncRulesContent(); + if (syncRules == null) { + throw new Error(`Active sync rules not available`); + } + + this.syncRulesContent = syncRules; + this.storage = this.factory.getInstance(syncRules); + return this.storage!; + } + + private getSyncRulesContent(): storage.PersistedSyncRulesContent { + if (this.syncRulesContent == null) { + throw new Error('Sync rules not configured - call updateSyncRules() first'); + } + return this.syncRulesContent; + } + + /** + * Replicate a snapshot, start streaming, and wait for a consistent checkpoint. + */ + async initializeReplication() { + await this.replicateSnapshot(); + // Make sure we're up to date + await this.getCheckpoint(); + } + + protected abstract triggerReplication(): Promise; + protected abstract waitForInitialSnapshot(): Promise; + + /** + * Replicate the initial snapshot, and start streaming. + */ + async replicateSnapshot() { + // Use a settledPromise to avoid unhandled rejections + this.settledReplicationPromise = settledPromise(this.triggerReplication()); + try { + await Promise.race([unsettledPromise(this.settledReplicationPromise), this.waitForInitialSnapshot()]); + } catch (e) { + if (e instanceof ReplicationAbortedError && e.cause != null) { + // Edge case for tests: replicate() can throw an error, but we'd receive the ReplicationAbortedError from + // waitForInitialSnapshot() first. In that case, prioritize the cause, e.g. MissingReplicationSlotError. + // This is not a concern for production use, since we only use waitForInitialSnapshot() in tests. + throw e.cause; + } + throw e; + } + } + + abstract getClientCheckpoint(options?: { timeout?: number }): Promise; + + async getCheckpoint(options?: { timeout?: number }) { + let checkpoint = await Promise.race([ + this.getClientCheckpoint(options), + unsettledPromise(this.settledReplicationPromise!) + ]); + if (checkpoint == null) { + // This indicates an issue with the test setup - replicationPromise completed instead + // of getClientCheckpoint() + throw new Error('Test failure - replicationPromise completed'); + } + return checkpoint; + } + + async getBucketsDataBatch(buckets: Record, options?: { timeout?: number }) { + const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent()); + const checkpoint = await this.getCheckpoint(options); + return helpers.getBucketsDataBatch(buckets, checkpoint); + } + + /** + * This waits for a client checkpoint. + */ + async getBucketData(bucket: string, start?: InternalOpId | string | undefined, options?: { timeout?: number }) { + const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent()); + const checkpoint = await this.getCheckpoint(options); + return helpers.getBucketData(bucket, checkpoint, start); + } + + async getChecksums(buckets: string[], options?: { timeout?: number }) { + const checkpoint = await this.getCheckpoint(options); + const syncRules = this.getSyncRulesContent(); + const versionedBuckets = buckets.map((bucket) => bucketRequest(syncRules, bucket, 0n)); + const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets); + + const unversioned = new Map(); + for (let i = 0; i < buckets.length; i++) { + unversioned.set(buckets[i], checksums.get(versionedBuckets[i].bucket)!); + } + + return unversioned; + } + + async getChecksum(bucket: string, options?: { timeout?: number }) { + const checksums = await this.getChecksums([bucket], options); + return checksums.get(bucket); + } + + /** + * This does not wait for a client checkpoint. + */ + async getCurrentBucketData(bucket: string, start?: InternalOpId | string | undefined) { + start ??= 0n; + if (typeof start == 'string') { + start = BigInt(start); + } + const syncRules = this.getSyncRulesContent(); + const { checkpoint } = await this.storage!.getCheckpoint(); + const map = [bucketRequest(syncRules, bucket, start)]; + const batch = this.storage!.getBucketDataBatch(checkpoint, map); + const batches = await fromAsync(batch); + return batches[0]?.chunkData.data ?? []; + } +} diff --git a/packages/service-core-tests/src/test-utils/test-utils-index.ts b/packages/service-core-tests/src/test-utils/test-utils-index.ts index a79c44098..d356a9392 100644 --- a/packages/service-core-tests/src/test-utils/test-utils-index.ts +++ b/packages/service-core-tests/src/test-utils/test-utils-index.ts @@ -1,3 +1,4 @@ +export * from './AbstractStreamTestContext.js'; export * from './bucket-validation.js'; export * from './general-utils.js'; export * from './MetricsHelper.js'; diff --git a/packages/service-core-tests/tsconfig.json b/packages/service-core-tests/tsconfig.json index 64548dbfc..31a397bf7 100644 --- a/packages/service-core-tests/tsconfig.json +++ b/packages/service-core-tests/tsconfig.json @@ -17,6 +17,9 @@ }, { "path": "../service-core" + }, + { + "path": "../../libs/lib-services" } ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cfbfaaa4e..4c2d7ad03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -94,10 +94,10 @@ importers: version: 6.0.3 vite: specifier: ^8.0.9 - version: 8.0.9(@types/node@22.16.2)(yaml@2.8.3) + version: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) ws: specifier: ^8.2.3 version: 8.18.0 @@ -115,7 +115,7 @@ importers: version: 6.10.4 mongodb: specifier: ^6.20.0 - version: 6.20.0(@mongodb-js/zstd@2.0.1)(snappy@7.3.3)(socks@2.8.3) + version: 6.20.0(@mongodb-js/zstd@2.0.1)(snappy@7.3.3) mongodb-connection-string-url: specifier: ^3.0.2 version: 3.0.2 @@ -194,7 +194,53 @@ importers: version: 4.17.6 vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) + + modules/module-convex: + dependencies: + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../../libs/lib-services + '@powersync/service-core': + specifier: workspace:* + version: link:../../packages/service-core + '@powersync/service-jsonbig': + specifier: workspace:* + version: link:../../packages/jsonbig + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + bson: + specifier: ^6.10.4 + version: 6.10.4 + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 + ts-codec: + specifier: ^1.3.0 + version: 1.3.0 + devDependencies: + '@convex-dev/auth': + specifier: ^0.0.92 + version: 0.0.92(@auth/core@0.37.4)(convex@1.38.0) + '@powersync/service-core-tests': + specifier: workspace:* + version: link:../../packages/service-core-tests + '@powersync/service-module-mongodb-storage': + specifier: workspace:* + version: link:../module-mongodb-storage + '@powersync/service-module-postgres-storage': + specifier: workspace:* + version: link:../module-postgres-storage + convex: + specifier: ^1.38.0 + version: 1.38.0 + dotenv: + specifier: ^16.4.5 + version: 16.4.5 modules/module-core: dependencies: @@ -516,7 +562,7 @@ importers: devDependencies: vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) packages/jsonbig: dependencies: @@ -554,6 +600,9 @@ importers: packages/schema: devDependencies: + '@powersync/service-module-convex': + specifier: workspace:* + version: link:../../modules/module-convex '@powersync/service-module-mongodb': specifier: workspace:* version: link:../../modules/module-mongodb @@ -693,6 +742,9 @@ importers: packages/service-core-tests: dependencies: + '@powersync/lib-services-framework': + specifier: workspace:^ + version: link:../../libs/lib-services '@powersync/service-core': specifier: workspace:^ version: link:../service-core @@ -704,7 +756,7 @@ importers: version: link:../sync-rules vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) devDependencies: '@opentelemetry/sdk-metrics': specifier: ^1.30.1 @@ -747,7 +799,7 @@ importers: version: 1.0.0 vitest: specifier: 'catalog:' - version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)) + version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) packages/types: dependencies: @@ -769,6 +821,9 @@ importers: '@powersync/service-core': specifier: workspace:* version: link:../packages/service-core + '@powersync/service-module-convex': + specifier: workspace:* + version: link:../modules/module-convex '@powersync/service-module-core': specifier: workspace:* version: link:../modules/module-core @@ -846,6 +901,20 @@ packages: peerDependencies: '@types/json-schema': ^7.0.15 + '@auth/core@0.37.4': + resolution: {integrity: sha512-HOXJwXWXQRhbBDHlMU0K/6FT1v+wjtzdKhsNg0ZN7/gne6XPsIrjZ4daMcFnbq0Z/vsAbYBinQhhua0d77v7qw==} + peerDependencies: + '@simplewebauthn/browser': ^9.0.1 + '@simplewebauthn/server': ^9.0.2 + nodemailer: ^6.8.0 + peerDependenciesMeta: + '@simplewebauthn/browser': + optional: true + '@simplewebauthn/server': + optional: true + nodemailer: + optional: true + '@azure-rest/core-client@2.5.1': resolution: {integrity: sha512-EHaOXW0RYDKS5CFffnixdyRPak5ytiCtU7uXDcP/uiY+A6jFRwNGzzJBiznkCzvi5EYpY+YWinieqHb0oY916A==} engines: {node: '>=20.0.0'} @@ -1002,6 +1071,17 @@ packages: resolution: {integrity: sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==} engines: {node: '>=0.1.90'} + '@convex-dev/auth@0.0.92': + resolution: {integrity: sha512-tNRIMTDxi2vrbT+3vz1FgNR1321IfIBDDBy59zul7E1DyzWQKoU0OzgFqWbiVm3o8gn0eQsYTU3UHNRX9kp3wQ==} + hasBin: true + peerDependencies: + '@auth/core': ^0.37.0 + convex: ^1.17.0 + react: ^18.2.0 || ^19.0.0-0 + peerDependenciesMeta: + react: + optional: true + '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -1027,6 +1107,162 @@ packages: '@emnapi/wasi-threads@1.2.1': resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} + '@esbuild/aix-ppc64@0.27.0': + resolution: {integrity: sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.27.0': + resolution: {integrity: sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.27.0': + resolution: {integrity: sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.27.0': + resolution: {integrity: sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.27.0': + resolution: {integrity: sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.0': + resolution: {integrity: sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.27.0': + resolution: {integrity: sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.0': + resolution: {integrity: sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.27.0': + resolution: {integrity: sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.27.0': + resolution: {integrity: sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.27.0': + resolution: {integrity: sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.27.0': + resolution: {integrity: sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.27.0': + resolution: {integrity: sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.27.0': + resolution: {integrity: sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.0': + resolution: {integrity: sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.27.0': + resolution: {integrity: sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.27.0': + resolution: {integrity: sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.27.0': + resolution: {integrity: sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.0': + resolution: {integrity: sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.27.0': + resolution: {integrity: sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.0': + resolution: {integrity: sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.27.0': + resolution: {integrity: sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.27.0': + resolution: {integrity: sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.27.0': + resolution: {integrity: sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.27.0': + resolution: {integrity: sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.27.0': + resolution: {integrity: sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@fastify/ajv-compiler@4.0.5': resolution: {integrity: sha512-KoWKW+MhvfTRWL4qrhUwAAZoaChluo0m0vbiJlGMt2GXvL4LVPQEjt8kSpHI3IBq5Rez8fg+XeH3cneztq+C7A==} @@ -1499,9 +1735,24 @@ packages: peerDependencies: '@opentelemetry/api': ^1.1.0 + '@oslojs/asn1@1.0.0': + resolution: {integrity: sha512-zw/wn0sj0j0QKbIXfIlnEcTviaCzYOY3V5rAyjR6YtOByFtJiT574+8p9Wlach0lZH9fddD4yb9laEAIl4vXQA==} + + '@oslojs/binary@1.0.0': + resolution: {integrity: sha512-9RCU6OwXU6p67H4NODbuxv2S3eenuQ4/WFLrsq+K/k682xrznH5EVWA7N4VFk9VYVcbFtKqur5YQQZc0ySGhsQ==} + + '@oslojs/crypto@1.0.1': + resolution: {integrity: sha512-7n08G8nWjAr/Yu3vu9zzrd0L9XnrJfpMioQcvCMxBIiF5orECHe5/3J0jmXRVvgfqMm/+4oxlQ+Sq39COYLcNQ==} + + '@oslojs/encoding@1.1.0': + resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==} + '@oxc-project/types@0.126.0': resolution: {integrity: sha512-oGfVtjAgwQVVpfBrbtk4e1XDyWHRFta6BS3GWVzrF8xYBT2VGQAk39yJS/wFSMrZqoiCU4oghT3Ch0HaHGIHcQ==} + '@panva/hkdf@1.2.1': + resolution: {integrity: sha512-6oclG6Y3PiDFcoyk8srjLfVKyMfVCKJ27JwNPViuXziFpmdz+MZnZN/aKY0JGXgYuO/VghU0jcOAZgWXZ1Dmrw==} + '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} @@ -2219,6 +2470,25 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + convex@1.38.0: + resolution: {integrity: sha512-122AC6y5lUS7mr39cluLw9+TOtRX5d/XxeivHhHObs/NTXoVvOnIgDzexVcxaz6Rk0oLFSoydSR1rDCltEz/0A==} + engines: {node: '>=18.0.0', npm: '>=7.0.0'} + hasBin: true + peerDependencies: + '@auth0/auth0-react': ^2.0.1 + '@clerk/clerk-react': ^4.12.8 || ^5.0.0 + '@clerk/react': ^6.4.3 + react: ^18.0.0 || ^19.0.0-0 || ^19.0.0 + peerDependenciesMeta: + '@auth0/auth0-react': + optional: true + '@clerk/clerk-react': + optional: true + '@clerk/react': + optional: true + react: + optional: true + cookie@1.0.2: resolution: {integrity: sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==} engines: {node: '>=18'} @@ -2350,6 +2620,11 @@ packages: es-module-lexer@2.0.0: resolution: {integrity: sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==} + esbuild@0.27.0: + resolution: {integrity: sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==} + engines: {node: '>=18'} + hasBin: true + escalade@3.1.2: resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} engines: {node: '>=6'} @@ -2604,10 +2879,6 @@ packages: resolution: {integrity: sha512-SVRCRovA7KaT6nqWB2mCNpTvU4cuZ0hOXo5KPyiyOcNNUIZwq/JKtvXuDJNaxfuJKabBYRu1ecHze0YEwDYoRQ==} engines: {node: '>=18'} - ip-address@9.0.5: - resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==} - engines: {node: '>= 12'} - ipaddr.js@2.2.0: resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} engines: {node: '>= 10'} @@ -2649,6 +2920,10 @@ packages: resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==} engines: {node: '>=8'} + is-network-error@1.3.1: + resolution: {integrity: sha512-6QCxa49rQbmUWLfk0nuGqzql9U8uaV2H6279bRErPBHe/109hCzsLUBUHfbEtvLIHBd6hyXbgedBSHevm43Edw==} + engines: {node: '>=16'} + is-number@7.0.0: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} @@ -2712,6 +2987,9 @@ packages: jose@4.15.9: resolution: {integrity: sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==} + jose@5.10.0: + resolution: {integrity: sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==} + js-md4@0.3.2: resolution: {integrity: sha512-/GDnfQYsltsjRswQhN9fhv3EMw2sCpUdrdxyWDOUK7eyD++r3gRhzgiQgc/x4MAv2i1iuQ4lxO5mvqM3vj4bwA==} @@ -2729,9 +3007,6 @@ packages: resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} hasBin: true - jsbn@1.1.0: - resolution: {integrity: sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==} - json-schema-ref-resolver@2.0.1: resolution: {integrity: sha512-HG0SIB9X4J8bwbxCbnd5FfPEbcXAJYTi1pBJeP/QPON+w8ovSME8iRG+ElHNxZNX2Qh6eYn1GdzJFS4cDFfx0Q==} @@ -2764,6 +3039,10 @@ packages: jws@4.0.1: resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + jwt-decode@4.0.0: + resolution: {integrity: sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==} + engines: {node: '>=18'} + kuler@2.0.0: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} @@ -2908,6 +3187,10 @@ packages: resolution: {integrity: sha512-FbAj6lXil6t8z4z3j0E5mfRlPzxkySotzUHwRXjlpRh10vc6AI6WN62ehZj82VG7M20rqogJ0GLwar2Xa05a8Q==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} + lucia@3.2.2: + resolution: {integrity: sha512-P1FlFBGCMPMXu+EGdVD9W4Mjm0DqsusmKgO7Xc33mI5X1bklmsQb0hfzPhXomQr9waWIBDsiOjvr1e6BTaUqpA==} + deprecated: This package has been deprecated. Please see https://lucia-auth.com/lucia-v3/migrate. + magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} @@ -3090,6 +3373,9 @@ packages: resolution: {integrity: sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==} engines: {node: '>=18'} + oauth4webapi@3.8.6: + resolution: {integrity: sha512-iwemM91xz8nryHti2yTmg5fhyEMVOkOXwHNqbvcATjyajb5oQxCQzrNOA6uElRHuMhQQTKUyFKV9y/CNyg25BQ==} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -3187,6 +3473,9 @@ packages: path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + path-to-regexp@6.3.0: + resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} + path-type@4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} @@ -3257,9 +3546,18 @@ packages: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} + preact-render-to-string@6.5.11: + resolution: {integrity: sha512-ubnauqoGczeGISiOh6RjX0/cdaF8v/oDXIjO85XALCQjwQP+SB4RDXXtvZ6yTYSjG+PC1QRP2AhPgCEsM2EvUw==} + peerDependencies: + preact: '>=10' + + preact@10.24.3: + resolution: {integrity: sha512-Z2dPnBnMUfyQfSQ+GBdsGa16hz35YmLmtTLhM169uW944hYL6xzTYkJjC07j+Wosz733pMWx0fgON3JNw1jJQA==} + prebuild-install@7.1.3: resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} engines: {node: '>=10'} + deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. hasBin: true prettier-plugin-embed@0.4.15: @@ -3477,6 +3775,9 @@ packages: seq-queue@0.0.5: resolution: {integrity: sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==} + server-only@0.0.1: + resolution: {integrity: sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==} + set-cookie-parser@2.6.0: resolution: {integrity: sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==} @@ -3526,18 +3827,10 @@ packages: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} - smart-buffer@4.2.0: - resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} - engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} - snappy@7.3.3: resolution: {integrity: sha512-UDJVCunvgblRpfTOjo/uT7pQzfrTsSICJ4yVS4aq7SsGBaUSpJwaVP15nF//jqinSLpN7boe/BqbUmtWMTQ5MQ==} engines: {node: '>= 10'} - socks@2.8.3: - resolution: {integrity: sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==} - engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} - sonic-boom@4.2.0: resolution: {integrity: sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==} @@ -3998,6 +4291,14 @@ snapshots: '@types/json-schema': 7.0.15 js-yaml: 4.1.1 + '@auth/core@0.37.4': + dependencies: + '@panva/hkdf': 1.2.1 + jose: 5.10.0 + oauth4webapi: 3.8.6 + preact: 10.24.3 + preact-render-to-string: 6.5.11(preact@10.24.3) + '@azure-rest/core-client@2.5.1': dependencies: '@azure/abort-controller': 2.1.2 @@ -4313,6 +4614,21 @@ snapshots: '@colors/colors@1.6.0': {} + '@convex-dev/auth@0.0.92(@auth/core@0.37.4)(convex@1.38.0)': + dependencies: + '@auth/core': 0.37.4 + '@oslojs/crypto': 1.0.1 + '@oslojs/encoding': 1.1.0 + convex: 1.38.0 + cookie: 1.0.2 + is-network-error: 1.3.1 + jose: 5.10.0 + jwt-decode: 4.0.0 + lucia: 3.2.2 + oauth4webapi: 3.8.6 + path-to-regexp: 6.3.0 + server-only: 0.0.1 + '@cspotcode/source-map-support@0.8.1': dependencies: '@jridgewell/trace-mapping': 0.3.9 @@ -4355,6 +4671,84 @@ snapshots: tslib: 2.8.1 optional: true + '@esbuild/aix-ppc64@0.27.0': + optional: true + + '@esbuild/android-arm64@0.27.0': + optional: true + + '@esbuild/android-arm@0.27.0': + optional: true + + '@esbuild/android-x64@0.27.0': + optional: true + + '@esbuild/darwin-arm64@0.27.0': + optional: true + + '@esbuild/darwin-x64@0.27.0': + optional: true + + '@esbuild/freebsd-arm64@0.27.0': + optional: true + + '@esbuild/freebsd-x64@0.27.0': + optional: true + + '@esbuild/linux-arm64@0.27.0': + optional: true + + '@esbuild/linux-arm@0.27.0': + optional: true + + '@esbuild/linux-ia32@0.27.0': + optional: true + + '@esbuild/linux-loong64@0.27.0': + optional: true + + '@esbuild/linux-mips64el@0.27.0': + optional: true + + '@esbuild/linux-ppc64@0.27.0': + optional: true + + '@esbuild/linux-riscv64@0.27.0': + optional: true + + '@esbuild/linux-s390x@0.27.0': + optional: true + + '@esbuild/linux-x64@0.27.0': + optional: true + + '@esbuild/netbsd-arm64@0.27.0': + optional: true + + '@esbuild/netbsd-x64@0.27.0': + optional: true + + '@esbuild/openbsd-arm64@0.27.0': + optional: true + + '@esbuild/openbsd-x64@0.27.0': + optional: true + + '@esbuild/openharmony-arm64@0.27.0': + optional: true + + '@esbuild/sunos-x64@0.27.0': + optional: true + + '@esbuild/win32-arm64@0.27.0': + optional: true + + '@esbuild/win32-ia32@0.27.0': + optional: true + + '@esbuild/win32-x64@0.27.0': + optional: true + '@fastify/ajv-compiler@4.0.5': dependencies: ajv: 8.18.0 @@ -4877,8 +5271,23 @@ snapshots: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 2.0.1(@opentelemetry/api@1.9.0) + '@oslojs/asn1@1.0.0': + dependencies: + '@oslojs/binary': 1.0.0 + + '@oslojs/binary@1.0.0': {} + + '@oslojs/crypto@1.0.1': + dependencies: + '@oslojs/asn1': 1.0.0 + '@oslojs/binary': 1.0.0 + + '@oslojs/encoding@1.1.0': {} + '@oxc-project/types@0.126.0': {} + '@panva/hkdf@1.2.1': {} + '@pinojs/redact@0.4.0': {} '@pnpm/cli.meta@1000.0.11': @@ -5323,7 +5732,7 @@ snapshots: obug: 2.1.1 std-env: 4.0.0 tinyrainbow: 3.1.0 - vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)) + vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) '@vitest/expect@4.1.5': dependencies: @@ -5334,21 +5743,21 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3))': + '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.9(@types/node@22.16.2)(yaml@2.8.3) + vite: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) - '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3))': + '@vitest/mocker@4.1.5(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.9(@types/node@25.5.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3) '@vitest/pretty-format@4.1.5': dependencies: @@ -5377,7 +5786,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)) + vitest: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) '@vitest/utils@4.1.5': dependencies: @@ -5643,6 +6052,15 @@ snapshots: convert-source-map@2.0.0: {} + convex@1.38.0: + dependencies: + esbuild: 0.27.0 + prettier: 3.4.2 + ws: 8.18.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + cookie@1.0.2: {} core-util-is@1.0.3: {} @@ -5734,6 +6152,35 @@ snapshots: es-module-lexer@2.0.0: {} + esbuild@0.27.0: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.0 + '@esbuild/android-arm': 0.27.0 + '@esbuild/android-arm64': 0.27.0 + '@esbuild/android-x64': 0.27.0 + '@esbuild/darwin-arm64': 0.27.0 + '@esbuild/darwin-x64': 0.27.0 + '@esbuild/freebsd-arm64': 0.27.0 + '@esbuild/freebsd-x64': 0.27.0 + '@esbuild/linux-arm': 0.27.0 + '@esbuild/linux-arm64': 0.27.0 + '@esbuild/linux-ia32': 0.27.0 + '@esbuild/linux-loong64': 0.27.0 + '@esbuild/linux-mips64el': 0.27.0 + '@esbuild/linux-ppc64': 0.27.0 + '@esbuild/linux-riscv64': 0.27.0 + '@esbuild/linux-s390x': 0.27.0 + '@esbuild/linux-x64': 0.27.0 + '@esbuild/netbsd-arm64': 0.27.0 + '@esbuild/netbsd-x64': 0.27.0 + '@esbuild/openbsd-arm64': 0.27.0 + '@esbuild/openbsd-x64': 0.27.0 + '@esbuild/openharmony-arm64': 0.27.0 + '@esbuild/sunos-x64': 0.27.0 + '@esbuild/win32-arm64': 0.27.0 + '@esbuild/win32-ia32': 0.27.0 + '@esbuild/win32-x64': 0.27.0 + escalade@3.1.2: {} esprima@4.0.1: {} @@ -6003,12 +6450,6 @@ snapshots: wrap-ansi: 6.2.0 yoctocolors-cjs: 2.1.2 - ip-address@9.0.5: - dependencies: - jsbn: 1.1.0 - sprintf-js: 1.1.3 - optional: true - ipaddr.js@2.2.0: {} is-arrayish@0.3.2: {} @@ -6037,6 +6478,8 @@ snapshots: is-interactive@1.0.0: {} + is-network-error@1.3.1: {} + is-number@7.0.0: {} is-plain-obj@4.1.0: {} @@ -6085,6 +6528,8 @@ snapshots: jose@4.15.9: {} + jose@5.10.0: {} + js-md4@0.3.2: {} js-tokens@10.0.0: {} @@ -6100,9 +6545,6 @@ snapshots: dependencies: argparse: 2.0.1 - jsbn@1.1.0: - optional: true - json-schema-ref-resolver@2.0.1: dependencies: dequal: 2.0.3 @@ -6128,7 +6570,7 @@ snapshots: lodash.isstring: 4.0.1 lodash.once: 4.1.1 ms: 2.1.3 - semver: 7.7.3 + semver: 7.7.4 jsox@1.2.121: {} @@ -6143,6 +6585,8 @@ snapshots: jwa: 2.0.1 safe-buffer: 5.2.1 + jwt-decode@4.0.0: {} + kuler@2.0.0: {} leven@3.1.0: {} @@ -6250,6 +6694,11 @@ snapshots: lru.min@1.1.1: {} + lucia@3.2.2: + dependencies: + '@oslojs/crypto': 1.0.1 + '@oslojs/encoding': 1.1.0 + magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -6306,7 +6755,7 @@ snapshots: '@types/whatwg-url': 11.0.5 whatwg-url: 13.0.0 - mongodb@6.20.0(@mongodb-js/zstd@2.0.1)(snappy@7.3.3)(socks@2.8.3): + mongodb@6.20.0(@mongodb-js/zstd@2.0.1)(snappy@7.3.3): dependencies: '@mongodb-js/saslprep': 1.3.1 bson: 6.10.4 @@ -6314,7 +6763,6 @@ snapshots: optionalDependencies: '@mongodb-js/zstd': 2.0.1 snappy: 7.3.3 - socks: 2.8.3 moo@0.5.2: {} @@ -6412,6 +6860,8 @@ snapshots: path-key: 4.0.0 unicorn-magic: 0.3.0 + oauth4webapi@3.8.6: {} + object-assign@4.1.1: {} obug@2.1.1: {} @@ -6499,6 +6949,8 @@ snapshots: path-parse@1.0.7: {} + path-to-regexp@6.3.0: {} + path-type@4.0.0: {} pathe@2.0.3: {} @@ -6566,6 +7018,12 @@ snapshots: dependencies: xtend: 4.0.2 + preact-render-to-string@6.5.11(preact@10.24.3): + dependencies: + preact: 10.24.3 + + preact@10.24.3: {} + prebuild-install@7.1.3: dependencies: detect-libc: 2.1.2 @@ -6811,6 +7269,8 @@ snapshots: seq-queue@0.0.5: {} + server-only@0.0.1: {} + set-cookie-parser@2.6.0: {} shebang-command@2.0.0: @@ -6853,9 +7313,6 @@ snapshots: slash@3.0.0: {} - smart-buffer@4.2.0: - optional: true - snappy@7.3.3: optionalDependencies: '@napi-rs/snappy-android-arm-eabi': 7.3.3 @@ -6877,12 +7334,6 @@ snapshots: '@napi-rs/snappy-win32-ia32-msvc': 7.3.3 '@napi-rs/snappy-win32-x64-msvc': 7.3.3 - socks@2.8.3: - dependencies: - ip-address: 9.0.5 - smart-buffer: 4.2.0 - optional: true - sonic-boom@4.2.0: dependencies: atomic-sleep: 1.0.0 @@ -7106,7 +7557,7 @@ snapshots: vary@1.1.2: {} - vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3): + vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -7115,10 +7566,11 @@ snapshots: tinyglobby: 0.2.16 optionalDependencies: '@types/node': 22.16.2 + esbuild: 0.27.0 fsevents: 2.3.3 yaml: 2.8.3 - vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3): + vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -7127,13 +7579,14 @@ snapshots: tinyglobby: 0.2.16 optionalDependencies: '@types/node': 25.5.0 + esbuild: 0.27.0 fsevents: 2.3.3 yaml: 2.8.3 - vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)): + vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.5 - '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)) + '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) '@vitest/pretty-format': 4.1.5 '@vitest/runner': 4.1.5 '@vitest/snapshot': 4.1.5 @@ -7150,7 +7603,7 @@ snapshots: tinyexec: 1.0.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.9(@types/node@22.16.2)(yaml@2.8.3) + vite: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.0 @@ -7160,10 +7613,10 @@ snapshots: transitivePeerDependencies: - msw - vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)): + vitest@4.1.5(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.5)(@vitest/ui@4.1.5)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.5 - '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)) + '@vitest/mocker': 4.1.5(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) '@vitest/pretty-format': 4.1.5 '@vitest/runner': 4.1.5 '@vitest/snapshot': 4.1.5 @@ -7180,7 +7633,7 @@ snapshots: tinyexec: 1.0.2 tinyglobby: 0.2.16 tinyrainbow: 3.1.0 - vite: 8.0.9(@types/node@25.5.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index de0963119..7cba9ccfc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -18,6 +18,7 @@ catalog: allowBuilds: # These need install scripts to download/build native modules '@mongodb-js/zstd': true + esbuild: false # We don't need to run these checks: https://github.com/protobufjs/protobuf.js/blob/master/scripts/postinstall.js protobufjs: false diff --git a/service/Dockerfile b/service/Dockerfile index babc032fa..512482297 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -23,6 +23,7 @@ COPY modules/module-mongodb/package.json modules/module-mongodb/tsconfig.json mo COPY modules/module-mongodb-storage/package.json modules/module-mongodb-storage/tsconfig.json modules/module-mongodb-storage/ COPY modules/module-mysql/package.json modules/module-mysql/tsconfig.json modules/module-mysql/ COPY modules/module-mssql/package.json modules/module-mssql/tsconfig.json modules/module-mssql/ +COPY modules/module-convex/package.json modules/module-convex/tsconfig.json modules/module-convex/ RUN corepack enable pnpm && corepack install RUN pnpm install --frozen-lockfile @@ -51,6 +52,7 @@ COPY modules/module-mongodb/src modules/module-mongodb/src/ COPY modules/module-mongodb-storage/src modules/module-mongodb-storage/src/ COPY modules/module-mysql/src modules/module-mysql/src/ COPY modules/module-mssql/src modules/module-mssql/src/ +COPY modules/module-convex/src modules/module-convex/src/ RUN pnpm build:production && \ rm -rf node_modules **/node_modules && \ diff --git a/service/package.json b/service/package.json index a91c82e00..26dae8ff5 100644 --- a/service/package.json +++ b/service/package.json @@ -20,6 +20,7 @@ "@powersync/service-module-mysql": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", "@powersync/service-module-core": "workspace:*", + "@powersync/service-module-convex": "workspace:*", "@sentry/node": "^10.2.0" }, "devDependencies": { diff --git a/service/src/util/modules.ts b/service/src/util/modules.ts index 89ec9820f..45631cd7e 100644 --- a/service/src/util/modules.ts +++ b/service/src/util/modules.ts @@ -2,6 +2,7 @@ import * as core from '@powersync/service-core'; export const DYNAMIC_MODULES: core.ModuleLoaders = { connection: { + convex: () => import('@powersync/service-module-convex').then((module) => new module.ConvexModule()), mongodb: () => import('@powersync/service-module-mongodb').then((module) => new module.MongoModule()), mysql: () => import('@powersync/service-module-mysql').then((module) => new module.MySQLModule()), mssql: () => import('@powersync/service-module-mssql').then((module) => new module.MSSQLModule()), diff --git a/service/tsconfig.json b/service/tsconfig.json index 432b87d3b..80a86a7b4 100644 --- a/service/tsconfig.json +++ b/service/tsconfig.json @@ -36,6 +36,9 @@ { "path": "../modules/module-mysql" }, + { + "path": "../modules/module-convex" + }, { "path": "../modules/module-mssql" } diff --git a/tsconfig.json b/tsconfig.json index 980946ddf..e01370b9f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -40,6 +40,9 @@ { "path": "./modules/module-mysql" }, + { + "path": "./modules/module-convex" + }, { "path": "./modules/module-mongodb" }, @@ -100,6 +103,9 @@ { "path": "./modules/module-mysql/test" }, + { + "path": "./modules/module-convex/test" + }, { "path": "./modules/module-mongodb/test" },