From 83955df47d7ab8c2fe0870fa7076f431d24aae5b Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Thu, 5 Feb 2026 17:58:34 -0700 Subject: [PATCH 01/86] PoC convex adapter built using gpt-5.3 --- modules/module-convex/README.md | 24 + modules/module-convex/package.json | 41 ++ .../src/api/ConvexRouteAPIAdapter.ts | 263 +++++++ .../src/client/ConvexApiClient.ts | 436 ++++++++++++ modules/module-convex/src/common/ConvexLSN.ts | 107 +++ modules/module-convex/src/index.ts | 4 + .../module-convex/src/module/ConvexModule.ts | 77 +++ .../replication/ConvexConnectionManager.ts | 33 + .../ConvexConnectionManagerFactory.ts | 28 + .../src/replication/ConvexErrorRateLimiter.ts | 47 ++ .../src/replication/ConvexReplicationJob.ts | 75 ++ .../src/replication/ConvexReplicator.ts | 62 ++ .../src/replication/ConvexStream.ts | 650 ++++++++++++++++++ .../src/replication/replication-index.ts | 6 + modules/module-convex/src/types/types.ts | 84 +++ .../test/src/ConvexApiClient.test.ts | 158 +++++ .../module-convex/test/src/ConvexLSN.test.ts | 33 + .../test/src/ConvexRouteAPIAdapter.test.ts | 72 ++ .../test/src/ConvexStream.test.ts | 200 ++++++ modules/module-convex/test/src/setup.ts | 11 + modules/module-convex/test/src/types.test.ts | 38 + modules/module-convex/test/tsconfig.json | 25 + modules/module-convex/tsconfig.json | 28 + modules/module-convex/vitest.config.ts | 3 + packages/schema/package.json | 3 +- .../schema/src/scripts/compile-json-schema.ts | 4 +- packages/schema/tsconfig.json | 3 + pnpm-lock.yaml | 31 + service/Dockerfile | 2 + service/package.json | 3 +- service/src/util/modules.ts | 1 + service/tsconfig.json | 3 + tsconfig.json | 6 + 33 files changed, 2558 insertions(+), 3 deletions(-) create mode 100644 modules/module-convex/README.md create mode 100644 modules/module-convex/package.json create mode 100644 modules/module-convex/src/api/ConvexRouteAPIAdapter.ts create mode 100644 modules/module-convex/src/client/ConvexApiClient.ts create mode 100644 modules/module-convex/src/common/ConvexLSN.ts create mode 100644 modules/module-convex/src/index.ts create mode 100644 modules/module-convex/src/module/ConvexModule.ts create mode 100644 modules/module-convex/src/replication/ConvexConnectionManager.ts create mode 100644 modules/module-convex/src/replication/ConvexConnectionManagerFactory.ts create mode 100644 modules/module-convex/src/replication/ConvexErrorRateLimiter.ts create mode 100644 modules/module-convex/src/replication/ConvexReplicationJob.ts create mode 100644 modules/module-convex/src/replication/ConvexReplicator.ts create mode 100644 modules/module-convex/src/replication/ConvexStream.ts create mode 100644 modules/module-convex/src/replication/replication-index.ts create mode 100644 modules/module-convex/src/types/types.ts create mode 100644 modules/module-convex/test/src/ConvexApiClient.test.ts create mode 100644 modules/module-convex/test/src/ConvexLSN.test.ts create mode 100644 modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts create mode 100644 modules/module-convex/test/src/ConvexStream.test.ts create mode 100644 modules/module-convex/test/src/setup.ts create mode 100644 modules/module-convex/test/src/types.test.ts create mode 100644 modules/module-convex/test/tsconfig.json create mode 100644 modules/module-convex/tsconfig.json create mode 100644 modules/module-convex/vitest.config.ts diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md new file mode 100644 index 000000000..05086b515 --- /dev/null +++ b/modules/module-convex/README.md @@ -0,0 +1,24 @@ +# PowerSync Convex Module + +Convex replication module for PowerSync. + +## Configuration + +```yaml +replication: + connections: + - type: convex + id: default + tag: default + deployment_url: https://.convex.cloud + deploy_key: + polling_interval_ms: 1000 + request_timeout_ms: 30000 +``` + +## Manual smoke test + +1. Start PowerSync with a Convex source config and valid sync rules that reference `convex.`. +2. Confirm diagnostics reports the Convex source as connected. +3. Verify initial snapshot data appears in the expected bucket(s). +4. Insert/update/delete rows in Convex and verify bucket changes are replicated. diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json new file mode 100644 index 000000000..d705bbb02 --- /dev/null +++ b/modules/module-convex/package.json @@ -0,0 +1,41 @@ +{ + "name": "@powersync/service-module-convex", + "repository": "https://github.com/powersync-ja/powersync-service", + "types": "dist/index.d.ts", + "version": "0.1.0", + "license": "FSL-1.1-ALv2", + "main": "dist/index.js", + "type": "module", + "publishConfig": { + "access": "public" + }, + "scripts": { + "build": "tsc -b", + "build:tests": "tsc -b test/tsconfig.json", + "clean": "rm -rf ./dist && tsc -b --clean", + "test": "vitest" + }, + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./types": { + "import": "./dist/types/types.js", + "require": "./dist/types/types.js", + "default": "./dist/types/types.js" + } + }, + "dependencies": { + "@powersync/lib-services-framework": "workspace:*", + "@powersync/service-core": "workspace:*", + "@powersync/service-jsonbig": "workspace:*", + "@powersync/service-sync-rules": "workspace:*", + "@powersync/service-types": "workspace:*", + "ts-codec": "^1.3.0" + }, + "devDependencies": { + "@powersync/service-core-tests": "workspace:*" + } +} diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts new file mode 100644 index 000000000..203ce0b6d --- /dev/null +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -0,0 +1,263 @@ +import { api, ParseSyncRulesOptions, ReplicationHeadCallback, ReplicationLagOptions, SourceTable } from '@powersync/service-core'; +import * as sync_rules from '@powersync/service-sync-rules'; +import * as service_types from '@powersync/service-types'; +import { ConvexLSN } from '../common/ConvexLSN.js'; +import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; +import * as types from '../types/types.js'; + +export class ConvexRouteAPIAdapter implements api.RouteAPI { + protected connectionManager: ConvexConnectionManager; + + constructor(protected config: types.ResolvedConvexConnectionConfig) { + this.connectionManager = new ConvexConnectionManager(config); + } + + async getSourceConfig(): Promise { + return this.config; + } + + async getConnectionStatus(): Promise { + const base = { + id: this.config.id, + uri: types.baseUri(this.config) + }; + + try { + await this.connectionManager.client.getJsonSchemas(); + return { + ...base, + connected: true, + errors: [] + }; + } catch (error) { + return { + ...base, + connected: false, + errors: [{ level: 'fatal', message: error instanceof Error ? error.message : `${error}` }] + }; + } + } + + async getDebugTablesInfo( + tablePatterns: sync_rules.TablePattern[], + sqlSyncRules: sync_rules.SqlSyncRules + ): Promise { + const schema = await this.connectionManager.client.getJsonSchemas(); + const tablesByName = new Map(schema.tables.map((table) => [table.tableName, table])); + + const result: api.PatternResult[] = []; + + for (const tablePattern of tablePatterns) { + const patternResult: api.PatternResult = { + schema: tablePattern.schema, + pattern: tablePattern.tablePattern, + wildcard: tablePattern.isWildcard + }; + + result.push(patternResult); + + if (tablePattern.connectionTag != this.connectionManager.connectionTag) { + if (tablePattern.isWildcard) { + patternResult.tables = []; + } else { + patternResult.table = createTableInfo({ + tablePattern, + connectionTag: this.connectionManager.connectionTag, + syncRules: sqlSyncRules, + errors: [{ level: 'warning', message: 'Skipped: connection tag does not match Convex connection tag' }] + }); + } + continue; + } + + const matchedTableNames = [...tablesByName.keys()] + .filter((name) => { + if (tablePattern.schema != this.connectionManager.schema) { + return false; + } + if (tablePattern.isWildcard) { + return name.startsWith(tablePattern.tablePrefix); + } + return name == tablePattern.name; + }) + .sort(); + + if (tablePattern.isWildcard) { + patternResult.tables = matchedTableNames.map((tableName) => + createTableInfo({ + tablePattern, + connectionTag: this.connectionManager.connectionTag, + syncRules: sqlSyncRules, + tableName + }) + ); + } else { + const tableName = matchedTableNames[0] ?? tablePattern.name; + patternResult.table = createTableInfo({ + tablePattern, + connectionTag: this.connectionManager.connectionTag, + syncRules: sqlSyncRules, + tableName, + errors: + matchedTableNames.length == 0 + ? [{ level: 'warning', message: `Table ${tablePattern.schema}.${tablePattern.name} not found` }] + : [] + }); + } + } + + return result; + } + + async getReplicationLagBytes(options: ReplicationLagOptions): Promise { + return undefined; + } + + async createReplicationHead(callback: ReplicationHeadCallback): Promise { + const tableName = (await this.connectionManager.client.getJsonSchemas()).tables[0]?.tableName; + const head = await this.connectionManager.client.getHeadCursor({ tableName }); + return await callback(ConvexLSN.fromCursor(head).comparable); + } + + async getConnectionSchema(): Promise { + const schema = await this.connectionManager.client.getJsonSchemas(); + + return [ + { + name: this.connectionManager.schema, + tables: schema.tables.map((table) => ({ + name: table.tableName, + columns: Object.entries({ + _id: { type: 'string' }, + ...extractProperties(table.schema) + }) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([columnName, property]) => { + const jsonType = readJsonSchemaType(property); + const sqliteType = toSqliteType(jsonType); + + return { + name: columnName, + type: jsonType, + sqlite_type: sqliteType.typeFlags, + internal_type: jsonType, + pg_type: jsonType + }; + }) + })) + } + ]; + } + + async executeQuery(query: string, params: any[]): Promise { + return service_types.internal_routes.ExecuteSqlResponse.encode({ + results: { + columns: [], + rows: [] + }, + success: false, + error: 'SQL querying is not supported for Convex' + }); + } + + async shutdown(): Promise { + await this.connectionManager.end(); + } + + async [Symbol.asyncDispose]() { + await this.shutdown(); + } + + getParseSyncRulesOptions(): ParseSyncRulesOptions { + return { + defaultSchema: this.connectionManager.schema + }; + } +} + +function createTableInfo(options: { + tablePattern: sync_rules.TablePattern; + connectionTag: string; + syncRules: sync_rules.SqlSyncRules; + tableName?: string; + errors?: service_types.ReplicationError[]; +}) { + const tableName = + options.tableName ?? (options.tablePattern.isWildcard ? options.tablePattern.tablePrefix : options.tablePattern.name); + const sourceTable = new SourceTable({ + id: 0, + connectionTag: options.connectionTag, + objectId: tableName, + schema: options.tablePattern.schema, + name: tableName, + replicaIdColumns: [{ name: '_id' }], + snapshotComplete: true + }); + + return { + schema: options.tablePattern.schema, + name: tableName, + pattern: options.tablePattern.isWildcard ? options.tablePattern.tablePattern : undefined, + replication_id: ['_id'], + data_queries: options.syncRules.tableSyncsData(sourceTable), + parameter_queries: options.syncRules.tableSyncsParameters(sourceTable), + errors: options.errors ?? [] + }; +} + +function extractProperties(schema: Record) { + const direct = schema.properties; + if (isRecord(direct)) { + return direct; + } + + const nested = schema.schema?.properties; + if (isRecord(nested)) { + return nested; + } + + return {}; +} + +function readJsonSchemaType(value: unknown): string { + if (!isRecord(value)) { + return 'unknown'; + } + + const type = value.type; + if (typeof type == 'string') { + return type; + } + + if (Array.isArray(type)) { + const firstString = type.find((entry) => typeof entry == 'string'); + if (firstString) { + return firstString; + } + } + + return 'unknown'; +} + +function toSqliteType(type: string): sync_rules.ExpressionType { + switch (type) { + case 'integer': + return sync_rules.ExpressionType.INTEGER; + case 'number': + return sync_rules.ExpressionType.REAL; + case 'boolean': + return sync_rules.ExpressionType.INTEGER; + case 'null': + return sync_rules.ExpressionType.NONE; + case 'array': + case 'object': + return sync_rules.ExpressionType.TEXT; + case 'string': + default: + return sync_rules.ExpressionType.TEXT; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value == 'object' && value != null && !Array.isArray(value); +} diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts new file mode 100644 index 000000000..4dba0a5c3 --- /dev/null +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -0,0 +1,436 @@ +import { setTimeout as delay } from 'timers/promises'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { NormalizedConvexConnectionConfig } from '../types/types.js'; + +export interface ConvexRawDocument { + _id?: string; + _table?: string; + _deleted?: boolean; + [key: string]: any; +} + +export interface ConvexTableSchema { + tableName: string; + schema: Record; +} + +export interface ConvexJsonSchemasResult { + tables: ConvexTableSchema[]; + raw: Record; +} + +export interface ConvexListSnapshotOptions { + snapshot?: string; + cursor?: string; + tableName?: string; + signal?: AbortSignal; +} + +export interface ConvexListSnapshotResult { + snapshot: string; + cursor: string | null; + hasMore: boolean; + values: ConvexRawDocument[]; +} + +export interface ConvexDocumentDeltasOptions { + cursor?: string; + tableName?: string; + signal?: AbortSignal; +} + +export interface ConvexDocumentDeltasResult { + cursor: string; + hasMore: boolean; + values: ConvexRawDocument[]; +} + +export class ConvexApiError extends Error { + readonly status?: number; + readonly retryable: boolean; + readonly body?: unknown; + + constructor(options: { + message: string; + status?: number; + retryable: boolean; + body?: unknown; + cause?: unknown; + }) { + super(options.message); + this.name = 'ConvexApiError'; + this.status = options.status; + this.retryable = options.retryable; + this.body = options.body; + if (options.cause !== undefined) { + (this as Error & { cause?: unknown }).cause = options.cause; + } + } +} + +export class ConvexApiClient { + constructor(private readonly config: NormalizedConvexConnectionConfig) {} + + async getJsonSchemas(options?: { signal?: AbortSignal }): Promise { + const payload = await this.requestJson({ + endpoint: 'json_schemas', + signal: options?.signal, + allowStreamingFallback: true + }); + + return { + tables: extractTableSchemas(payload), + raw: payload + }; + } + + async listSnapshot(options: ConvexListSnapshotOptions): Promise { + const payload = await this.requestJson({ + endpoint: 'list_snapshot', + params: { + snapshot: options.snapshot, + cursor: options.cursor, + tableName: options.tableName, + table_name: options.tableName + }, + signal: options.signal, + allowStreamingFallback: true + }); + + const values = parseValues(payload); + const snapshot = parseSnapshot(payload, options.snapshot); + const cursor = payload.cursor == null ? null : stringifyCursor(payload.cursor); + + return { + snapshot, + cursor, + hasMore: parseHasMore(payload), + values + }; + } + + async documentDeltas(options: ConvexDocumentDeltasOptions): Promise { + const payload = await this.requestJson({ + endpoint: 'document_deltas', + params: { + cursor: options.cursor, + tableName: options.tableName, + table_name: options.tableName + }, + signal: options.signal, + allowStreamingFallback: true + }); + + return { + cursor: stringifyCursor(payload.cursor), + hasMore: parseHasMore(payload), + values: parseValues(payload) + }; + } + + async getHeadCursor(options?: { tableName?: string; signal?: AbortSignal }): Promise { + const page = await this.listSnapshot({ + tableName: options?.tableName, + signal: options?.signal + }); + return page.snapshot; + } + + private async requestJson(options: { + endpoint: string; + params?: Record; + signal?: AbortSignal; + allowStreamingFallback?: boolean; + }): Promise> { + await this.assertHostAllowed(); + + const primaryPath = `/api/${options.endpoint}`; + const fallbackPath = `/api/streaming_export/${options.endpoint}`; + + try { + return await this.performRequest({ + path: primaryPath, + params: options.params, + signal: options.signal + }); + } catch (error) { + if ( + options.allowStreamingFallback && + error instanceof ConvexApiError && + error.status == 404 && + primaryPath != fallbackPath + ) { + return await this.performRequest({ + path: fallbackPath, + params: options.params, + signal: options.signal + }); + } + throw error; + } + } + + private async performRequest(options: { + path: string; + params?: Record; + signal?: AbortSignal; + }): Promise> { + const url = new URL(options.path, this.config.deploymentUrl); + url.searchParams.set('format', 'json'); + + for (const [key, value] of Object.entries(options.params ?? {})) { + if (value == null) { + continue; + } + url.searchParams.set(key, `${value}`); + } + + const timeout = new AbortController(); + const timeoutPromise = delay(this.config.requestTimeoutMs, undefined, { + signal: timeout.signal + }).then(() => { + timeout.abort(new Error(`Convex API request timed out after ${this.config.requestTimeoutMs}ms`)); + }); + + const signals = [timeout.signal]; + if (options.signal) { + signals.push(options.signal); + } + + const signal = AbortSignal.any(signals); + + try { + const response = await fetch(url, { + method: 'GET', + headers: { + Authorization: `Convex ${this.config.deployKey}`, + Accept: 'application/json' + }, + signal + }); + + const text = await response.text(); + const json = text == '' ? {} : safeParseJson(text); + + if (!response.ok) { + const retryable = response.status == 429 || response.status >= 500; + throw new ConvexApiError({ + message: `Convex API request failed (${response.status}) for ${url.pathname}`, + status: response.status, + retryable, + body: json, + cause: new Error(text) + }); + } + + if (!isRecord(json)) { + throw new ConvexApiError({ + message: `Convex API response was not an object for ${url.pathname}`, + retryable: false, + body: json + }); + } + + return json; + } catch (error) { + if (error instanceof ConvexApiError) { + throw error; + } + + const message = error instanceof Error ? error.message : `${error}`; + const retryable = message.includes('timed out') || message.includes('fetch') || message.includes('ECONN'); + throw new ConvexApiError({ + message: `Convex API request failed for ${url.pathname}: ${message}`, + retryable, + cause: error + }); + } finally { + timeout.abort(); + await timeoutPromise.catch(() => { + // no-op + }); + } + } + + private async assertHostAllowed(): Promise { + if (!this.config.lookup) { + return; + } + + const hostname = new URL(this.config.deploymentUrl).hostname; + + await new Promise((resolve, reject) => { + this.config.lookup!(hostname, {}, (error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + } +} + +export function isCursorExpiredError(error: unknown): boolean { + if (!(error instanceof ConvexApiError)) { + return false; + } + + const asString = `${error.message} ${JSON.stringify(error.body ?? {})}`.toLowerCase(); + + return ( + asString.includes('cursor') && (asString.includes('expired') || asString.includes('invalid')) + ) || (asString.includes('snapshot') && asString.includes('expired')); +} + +function parseHasMore(payload: Record): boolean { + return Boolean(payload.has_more ?? payload.hasMore ?? false); +} + +function parseValues(payload: Record): ConvexRawDocument[] { + const values = payload.values ?? payload.page ?? []; + if (!Array.isArray(values)) { + return []; + } + + return values.filter((value) => isRecord(value)) as ConvexRawDocument[]; +} + +function extractTableSchemas(payload: Record): ConvexTableSchema[] { + const resolved = new Map(); + + const tableList = payload.tables; + if (Array.isArray(tableList)) { + for (const table of tableList) { + if (!isRecord(table)) { + continue; + } + const tableName = table.tableName ?? table.table_name ?? table.name; + if (typeof tableName != 'string' || tableName.length == 0) { + continue; + } + resolved.set(tableName, { + tableName, + schema: isRecord(table.schema) ? table.schema : {} + }); + } + } else if (isRecord(tableList)) { + for (const [tableName, schema] of Object.entries(tableList)) { + resolved.set(tableName, { + tableName, + schema: isRecord(schema) ? schema : {} + }); + } + } + + const schemaMap = payload.schema; + if (isRecord(schemaMap)) { + for (const [tableName, schema] of Object.entries(schemaMap)) { + if (!resolved.has(tableName)) { + resolved.set(tableName, { + tableName, + schema: isRecord(schema) ? schema : {} + }); + } + } + } + + // Self-hosted Convex may return table schemas as the top-level object: + // { "table_a": { ...json schema... }, "table_b": { ...json schema... } }. + for (const [tableName, schema] of Object.entries(payload)) { + if (resolved.has(tableName) || RESERVED_SCHEMA_KEYS.has(tableName)) { + continue; + } + + if (!looksLikeJsonSchema(schema)) { + continue; + } + + resolved.set(tableName, { + tableName, + schema: isRecord(schema) ? schema : {} + }); + } + + return [...resolved.values()].sort((a, b) => a.tableName.localeCompare(b.tableName)); +} + +function stringifyCursor(value: unknown): string { + if (typeof value == 'string') { + if (value.length == 0) { + throw new ConvexApiError({ + message: 'Convex cursor cannot be empty', + retryable: false, + body: value + }); + } + return value; + } + + if (typeof value == 'number' || typeof value == 'bigint') { + return `${value}`; + } + + throw new ConvexApiError({ + message: `Convex cursor is missing or invalid: ${JSON.stringify(value)}`, + retryable: false, + body: value + }); +} + +function parseSnapshot(payload: Record, requestedSnapshot?: string): string { + const responseSnapshot = payload.snapshot ?? payload.snapshot_ts ?? payload.snapshotTs; + if (responseSnapshot != null) { + return stringifyCursor(responseSnapshot); + } + + if (requestedSnapshot != null && requestedSnapshot.length > 0) { + return requestedSnapshot; + } + + throw new ConvexApiError({ + message: 'Convex list_snapshot response is missing snapshot', + retryable: false, + body: payload + }); +} + +function safeParseJson(value: string): unknown { + try { + return JSONBig.parse(value); + } catch { + return { raw: value }; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value == 'object' && value != null && !Array.isArray(value); +} + +const RESERVED_SCHEMA_KEYS = new Set([ + 'tables', + 'schema', + 'snapshot', + 'cursor', + 'values', + 'value', + 'page', + 'hasMore', + 'has_more', + 'error', + 'errors' +]); + +function looksLikeJsonSchema(value: unknown): boolean { + if (!isRecord(value)) { + return false; + } + + return ( + typeof value.type == 'string' || + typeof value.$schema == 'string' || + Array.isArray(value.required) || + isRecord(value.properties) || + isRecord(value.schema) + ); +} diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts new file mode 100644 index 000000000..e06f4a55f --- /dev/null +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -0,0 +1,107 @@ +const CURSOR_WIDTH = 20; +const DELIMITER = '|'; + +type CursorMode = 'numeric' | 'opaque'; + +export interface ConvexLSNSpecification { + cursor: string; + mode: CursorMode; + sortKey: bigint; +} + +export class ConvexLSN { + static ZERO = ConvexLSN.fromCursor('0'); + + static fromSerialized(comparable: string): ConvexLSN { + if (!comparable.includes(DELIMITER)) { + // Legacy support: raw cursor persisted directly. + return ConvexLSN.fromCursor(comparable); + } + + const [first, second] = comparable.split(DELIMITER, 2); + const prefixed = first.match(/^([no])([0-9]+)$/); + if (prefixed && second != null) { + const decoded = decodeCursor(second); + return ConvexLSN.fromCursor(decoded); + } + + // Backwards compatibility for early PoC format: "|". + if (/^[0-9]+$/.test(first)) { + const decoded = decodeCursor(second ?? ''); + return ConvexLSN.fromCursor(decoded.length == 0 ? first : decoded); + } + + // Unknown future format: treat whole input as an opaque cursor. + return ConvexLSN.fromCursor(comparable); + } + + static fromCursor(cursor: string | number | bigint): ConvexLSN { + const asString = `${cursor}`; + if (asString.length == 0) { + throw new Error('Convex cursor cannot be empty'); + } + + const numericSortKey = getNumericSortKey(asString); + const mode: CursorMode = numericSortKey == null ? 'opaque' : 'numeric'; + + return new ConvexLSN({ + cursor: asString, + mode, + sortKey: numericSortKey ?? 0n + }); + } + + constructor(private readonly options: ConvexLSNSpecification) {} + + get timestamp() { + return this.options.sortKey; + } + + get cursor() { + return this.options.cursor; + } + + get comparable() { + const prefix = this.options.mode == 'numeric' ? 'n' : 'o'; + const sortPart = this.options.sortKey.toString().padStart(CURSOR_WIDTH, '0'); + return `${prefix}${sortPart}${DELIMITER}${encodeCursor(this.options.cursor)}`; + } + + toString() { + return this.comparable; + } + + toCursorString() { + return this.options.cursor; + } +} + +function getNumericSortKey(cursor: string): bigint | null { + if (/^[0-9]+$/.test(cursor)) { + return BigInt(cursor); + } + + // Common Convex-style cursors on self-hosted deployments, e.g. "00000000/0183E3A8". + const slashHex = cursor.match(/^([0-9A-Fa-f]{8})\/([0-9A-Fa-f]{8})$/); + if (slashHex) { + const high = BigInt(`0x${slashHex[1]}`); + const low = BigInt(`0x${slashHex[2]}`); + return (high << 32n) + low; + } + + // MSSQL-style hexadecimal LSN string, useful for migration compatibility. + const colonHex = cursor.match(/^([0-9A-Fa-f]{8}):([0-9A-Fa-f]{8}):([0-9A-Fa-f]{4})$/); + if (colonHex) { + return BigInt(`0x${colonHex[1]}${colonHex[2]}${colonHex[3]}`); + } + + return null; +} + +function encodeCursor(value: string): string { + return Buffer.from(value, 'utf8').toString('base64url'); +} + +function decodeCursor(value: string): string { + return Buffer.from(value, 'base64url').toString('utf8'); +} diff --git a/modules/module-convex/src/index.ts b/modules/module-convex/src/index.ts new file mode 100644 index 000000000..90ee6204f --- /dev/null +++ b/modules/module-convex/src/index.ts @@ -0,0 +1,4 @@ +export * from './api/ConvexRouteAPIAdapter.js'; +export * from './module/ConvexModule.js'; +export * from './replication/replication-index.js'; +export * from './types/types.js'; diff --git a/modules/module-convex/src/module/ConvexModule.ts b/modules/module-convex/src/module/ConvexModule.ts new file mode 100644 index 000000000..b873ed813 --- /dev/null +++ b/modules/module-convex/src/module/ConvexModule.ts @@ -0,0 +1,77 @@ +import { + api, + ConfigurationFileSyncRulesProvider, + ConnectionTestResult, + replication, + system, + TearDownOptions +} from '@powersync/service-core'; +import { ConvexRouteAPIAdapter } from '../api/ConvexRouteAPIAdapter.js'; +import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; +import { ConvexConnectionManagerFactory } from '../replication/ConvexConnectionManagerFactory.js'; +import { ConvexErrorRateLimiter } from '../replication/ConvexErrorRateLimiter.js'; +import { ConvexReplicator } from '../replication/ConvexReplicator.js'; +import * as types from '../types/types.js'; + +export class ConvexModule extends replication.ReplicationModule { + constructor() { + super({ + name: 'Convex', + type: types.CONVEX_CONNECTION_TYPE, + configSchema: types.ConvexConnectionConfig + }); + } + + async onInitialized(context: system.ServiceContextContainer): Promise {} + + protected createRouteAPIAdapter(): api.RouteAPI { + return new ConvexRouteAPIAdapter(this.resolveConfig(this.decodedConfig!)); + } + + protected createReplicator(context: system.ServiceContext): replication.AbstractReplicator { + const normalizedConfig = this.resolveConfig(this.decodedConfig!); + const syncRuleProvider = new ConfigurationFileSyncRulesProvider(context.configuration.sync_rules); + const connectionFactory = new ConvexConnectionManagerFactory(normalizedConfig); + + return new ConvexReplicator({ + id: this.getDefaultId(normalizedConfig.deploymentUrl), + syncRuleProvider, + storageEngine: context.storageEngine, + metricsEngine: context.metricsEngine, + connectionFactory, + rateLimiter: new ConvexErrorRateLimiter() + }); + } + + private resolveConfig(config: types.ConvexConnectionConfig): types.ResolvedConvexConnectionConfig { + return { + ...config, + ...types.normalizeConnectionConfig(config) + }; + } + + async teardown(options: TearDownOptions): Promise { + // No source-side teardown required. + } + + async testConnection(config: types.ConvexConnectionConfig) { + this.decodeConfig(config); + const normalizedConfig = this.resolveConfig(this.decodedConfig!); + return await ConvexModule.testConnection(normalizedConfig); + } + + static async testConnection( + normalizedConfig: types.ResolvedConvexConnectionConfig + ): Promise { + const connectionManager = new ConvexConnectionManager(normalizedConfig); + try { + await connectionManager.client.getJsonSchemas(); + } finally { + await connectionManager.end(); + } + + return { + connectionDescription: normalizedConfig.deploymentUrl + }; + } +} diff --git a/modules/module-convex/src/replication/ConvexConnectionManager.ts b/modules/module-convex/src/replication/ConvexConnectionManager.ts new file mode 100644 index 000000000..bbd7bef0e --- /dev/null +++ b/modules/module-convex/src/replication/ConvexConnectionManager.ts @@ -0,0 +1,33 @@ +import { DEFAULT_TAG } from '@powersync/service-sync-rules'; +import { ConvexApiClient } from '../client/ConvexApiClient.js'; +import { ResolvedConvexConnectionConfig } from '../types/types.js'; + +export interface ConvexConnectionManagerListener { + onEnded?: () => void; +} + +export class ConvexConnectionManager { + readonly client: ConvexApiClient; + readonly schema = 'convex'; + readonly connectionTag: string; + readonly connectionId: string; + + private listeners = new Set(); + + constructor(public readonly config: ResolvedConvexConnectionConfig) { + this.client = new ConvexApiClient(config); + this.connectionTag = config.tag ?? DEFAULT_TAG; + this.connectionId = config.id ?? 'default'; + } + + registerListener(listener: ConvexConnectionManagerListener) { + this.listeners.add(listener); + } + + async end(): Promise { + for (const listener of [...this.listeners]) { + listener.onEnded?.(); + } + this.listeners.clear(); + } +} diff --git a/modules/module-convex/src/replication/ConvexConnectionManagerFactory.ts b/modules/module-convex/src/replication/ConvexConnectionManagerFactory.ts new file mode 100644 index 000000000..aef23df3e --- /dev/null +++ b/modules/module-convex/src/replication/ConvexConnectionManagerFactory.ts @@ -0,0 +1,28 @@ +import { logger } from '@powersync/lib-services-framework'; +import { ResolvedConvexConnectionConfig } from '../types/types.js'; +import { ConvexConnectionManager } from './ConvexConnectionManager.js'; + +export class ConvexConnectionManagerFactory { + private readonly connectionManagers = new Set(); + + constructor(public readonly connectionConfig: ResolvedConvexConnectionConfig) {} + + create() { + const manager = new ConvexConnectionManager(this.connectionConfig); + manager.registerListener({ + onEnded: () => { + this.connectionManagers.delete(manager); + } + }); + this.connectionManagers.add(manager); + return manager; + } + + async shutdown() { + logger.info('Shutting down Convex connection managers...'); + for (const manager of [...this.connectionManagers]) { + await manager.end(); + } + logger.info('Convex connection managers shutdown completed.'); + } +} diff --git a/modules/module-convex/src/replication/ConvexErrorRateLimiter.ts b/modules/module-convex/src/replication/ConvexErrorRateLimiter.ts new file mode 100644 index 000000000..110510170 --- /dev/null +++ b/modules/module-convex/src/replication/ConvexErrorRateLimiter.ts @@ -0,0 +1,47 @@ +import { ErrorRateLimiter } from '@powersync/service-core'; +import { setTimeout } from 'timers/promises'; +import { ConvexApiError } from '../client/ConvexApiClient.js'; + +export class ConvexErrorRateLimiter implements ErrorRateLimiter { + private nextAllowed = Date.now(); + + async waitUntilAllowed(options?: { signal?: AbortSignal | undefined } | undefined): Promise { + const delay = Math.max(0, this.nextAllowed - Date.now()); + this.setDelay(500); + await setTimeout(delay, undefined, { signal: options?.signal }); + } + + mayPing(): boolean { + return Date.now() >= this.nextAllowed; + } + + reportError(error: any): void { + if (error instanceof ConvexApiError) { + if (error.status == 401 || error.status == 403) { + this.setDelay(120_000); + return; + } + if (error.status == 429) { + this.setDelay(15_000); + return; + } + if (error.retryable) { + this.setDelay(10_000); + return; + } + this.setDelay(45_000); + return; + } + + const message = (error?.message as string | undefined) ?? ''; + if (message.includes('ENOTFOUND') || message.includes('ECONNREFUSED')) { + this.setDelay(120_000); + } else { + this.setDelay(20_000); + } + } + + private setDelay(delay: number) { + this.nextAllowed = Math.max(this.nextAllowed, Date.now() + delay); + } +} diff --git a/modules/module-convex/src/replication/ConvexReplicationJob.ts b/modules/module-convex/src/replication/ConvexReplicationJob.ts new file mode 100644 index 000000000..5cb7058f7 --- /dev/null +++ b/modules/module-convex/src/replication/ConvexReplicationJob.ts @@ -0,0 +1,75 @@ +import { container, logger as defaultLogger } from '@powersync/lib-services-framework'; +import { replication } from '@powersync/service-core'; +import { ConvexConnectionManagerFactory } from './ConvexConnectionManagerFactory.js'; +import { ConvexCursorExpiredError, ConvexStream } from './ConvexStream.js'; + +export interface ConvexReplicationJobOptions extends replication.AbstractReplicationJobOptions { + connectionFactory: ConvexConnectionManagerFactory; +} + +export class ConvexReplicationJob extends replication.AbstractReplicationJob { + private readonly connectionFactory: ConvexConnectionManagerFactory; + private lastStream: ConvexStream | null = null; + + constructor(options: ConvexReplicationJobOptions) { + super(options); + this.connectionFactory = options.connectionFactory; + this.logger = defaultLogger.child({ prefix: `[powersync_${this.options.storage.group_id}] ` }); + } + + async keepAlive() { + // streaming API is polled continuously; no dedicated keepalive operation required here + } + + async replicate() { + try { + await this.replicateOnce(); + } catch (error) { + if (!this.isStopped) { + this.logger.error('Replication error', error); + if (error?.cause != null) { + this.logger.error('cause', error.cause); + } + + container.reporter.captureException(error, { + metadata: {} + }); + + this.rateLimiter.reportError(error); + } + + if (error instanceof ConvexCursorExpiredError) { + await this.options.storage.factory.restartReplication(this.storage.group_id); + } + } finally { + this.abortController.abort(); + } + } + + async replicateOnce() { + const manager = this.connectionFactory.create(); + try { + await this.rateLimiter.waitUntilAllowed({ signal: this.abortController.signal }); + if (this.isStopped) { + return; + } + + const stream = new ConvexStream({ + abortSignal: this.abortController.signal, + connections: manager, + logger: this.logger, + metrics: this.options.metrics, + storage: this.options.storage + }); + + this.lastStream = stream; + await stream.replicate(); + } finally { + await manager.end(); + } + } + + async getReplicationLagMillis(): Promise { + return this.lastStream?.getReplicationLagMillis(); + } +} diff --git a/modules/module-convex/src/replication/ConvexReplicator.ts b/modules/module-convex/src/replication/ConvexReplicator.ts new file mode 100644 index 000000000..26cb90cc3 --- /dev/null +++ b/modules/module-convex/src/replication/ConvexReplicator.ts @@ -0,0 +1,62 @@ +import { replication, storage } from '@powersync/service-core'; +import { ConvexModule } from '../module/ConvexModule.js'; +import { ConvexConnectionManagerFactory } from './ConvexConnectionManagerFactory.js'; +import { ConvexReplicationJob } from './ConvexReplicationJob.js'; + +export interface ConvexReplicatorOptions extends replication.AbstractReplicatorOptions { + connectionFactory: ConvexConnectionManagerFactory; +} + +export class ConvexReplicator extends replication.AbstractReplicator { + private readonly connectionFactory: ConvexConnectionManagerFactory; + + constructor(options: ConvexReplicatorOptions) { + super(options); + this.connectionFactory = options.connectionFactory; + } + + createJob(options: replication.CreateJobOptions): ConvexReplicationJob { + return new ConvexReplicationJob({ + id: this.createJobId(options.storage.group_id), + storage: options.storage, + metrics: this.metrics, + lock: options.lock, + connectionFactory: this.connectionFactory, + rateLimiter: this.rateLimiter + }); + } + + async cleanUp(syncRulesStorage: storage.SyncRulesBucketStorage): Promise { + // No source-side cleanup needed for Convex. + } + + async stop(): Promise { + await super.stop(); + await this.connectionFactory.shutdown(); + } + + async testConnection() { + return await ConvexModule.testConnection(this.connectionFactory.connectionConfig); + } + + async getReplicationLagMillis(): Promise { + const lag = await super.getReplicationLagMillis(); + if (lag != null) { + return lag; + } + + const content = await this.storage.getActiveSyncRulesContent(); + if (content == null) { + return undefined; + } + + const checkpointTs = content.last_checkpoint_ts?.getTime() ?? 0; + const keepaliveTs = content.last_keepalive_ts?.getTime() ?? 0; + const latestTs = Math.max(checkpointTs, keepaliveTs); + if (latestTs == 0) { + return undefined; + } + + return Date.now() - latestTs; + } +} diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts new file mode 100644 index 000000000..5b9580ed8 --- /dev/null +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -0,0 +1,650 @@ +import { + container, + DatabaseConnectionError, + ErrorCode, + Logger, + logger as defaultLogger, + ReplicationAbortedError, + ReplicationAssertionError +} from '@powersync/lib-services-framework'; +import { + MetricsEngine, + RelationCache, + SaveOperationTag, + SourceEntityDescriptor, + SourceTable, + storage +} from '@powersync/service-core'; +import { HydratedSyncRules, SqliteInputRow, TablePattern } from '@powersync/service-sync-rules'; +import { ReplicationMetric } from '@powersync/service-types'; +import { setTimeout as delay } from 'timers/promises'; +import { ConvexRawDocument, ConvexTableSchema, isCursorExpiredError } from '../client/ConvexApiClient.js'; +import { ConvexLSN } from '../common/ConvexLSN.js'; +import { ConvexConnectionManager } from './ConvexConnectionManager.js'; + +export interface ConvexStreamOptions { + connections: ConvexConnectionManager; + storage: storage.SyncRulesBucketStorage; + metrics: MetricsEngine; + abortSignal: AbortSignal; + logger?: Logger; +} + +export class ConvexCursorExpiredError extends DatabaseConnectionError { + constructor(message: string, cause: unknown) { + super(ErrorCode.PSYNC_S1500, message, cause); + } +} + +export class ConvexStream { + private readonly storage: storage.SyncRulesBucketStorage; + private readonly metrics: MetricsEngine; + private readonly syncRules: HydratedSyncRules; + private readonly logger: Logger; + + private readonly relationCache = new RelationCache(getCacheIdentifier); + + private tableSchemaCache: ConvexTableSchema[] | null = null; + + private oldestUncommittedChange: Date | null = null; + private lastKeepaliveAt = 0; + private isStartingReplication = true; + private lastTouchedAt = performance.now(); + + constructor(private readonly options: ConvexStreamOptions) { + this.storage = options.storage; + this.metrics = options.metrics; + this.syncRules = options.storage.getParsedSyncRules({ defaultSchema: options.connections.schema }); + this.logger = options.logger ?? defaultLogger; + } + + private get connections() { + return this.options.connections; + } + + private get abortSignal() { + return this.options.abortSignal; + } + + private get defaultSchema() { + return this.connections.schema; + } + + private get pollingIntervalMs() { + return this.connections.config.pollingIntervalMs; + } + + get stopped() { + return this.abortSignal.aborted; + } + + async replicate() { + try { + await this.initReplication(); + await this.streamChanges(); + } catch (error) { + await this.storage.reportError(error); + throw error; + } + } + + async initReplication() { + const status = await this.initSlot(); + if (!status.needsInitialSync) { + return; + } + + if (status.snapshotLsn == null) { + await this.storage.clear({ signal: this.abortSignal }); + } + + const { lastOpId } = await this.initialReplication(status.snapshotLsn); + if (lastOpId != null) { + await this.storage.populatePersistentChecksumCache({ + signal: this.abortSignal, + maxOpId: lastOpId + }); + } + } + + async streamChanges() { + await this.storage.startBatch( + { + logger: this.logger, + zeroLSN: ConvexLSN.ZERO.comparable, + defaultSchema: this.defaultSchema, + storeCurrentData: false, + skipExistingRows: false + }, + async (batch) => { + let resumeFromLsn = batch.resumeFromLsn; + if (resumeFromLsn == null) { + throw new ReplicationAssertionError(`No LSN found to resume replication from.`); + } + + // Resolve tables up-front so we can pass an optional single-table optimization to Convex. + const sourceTables = await this.resolveAllSourceTables(batch); + const singleTable = sourceTables.length == 1 ? sourceTables[0]!.name : undefined; + + let cursor = ConvexLSN.fromSerialized(resumeFromLsn).toCursorString(); + + while (!this.abortSignal.aborted) { + const page = await this.connections.client + .documentDeltas({ + cursor, + tableName: singleTable, + signal: this.abortSignal + }) + .catch((error) => { + if (isCursorExpiredError(error)) { + throw new ConvexCursorExpiredError('Convex cursor expired; initial replication restart required', error); + } + throw error; + }); + + const nextCursor = page.cursor; + const pageLsn = ConvexLSN.fromCursor(nextCursor).comparable; + + let changesInPage = 0; + for (const change of page.values) { + if (this.abortSignal.aborted) { + throw new ReplicationAbortedError('Replication interrupted'); + } + + const tableName = readTableName(change, singleTable); + if (tableName == null) { + continue; + } + + const table = await this.getOrResolveTable(batch, tableName); + if (table == null || !table.syncAny) { + continue; + } + + const changed = await this.writeChange(batch, table, change); + if (!changed) { + continue; + } + + changesInPage += 1; + if (this.oldestUncommittedChange == null) { + this.oldestUncommittedChange = new Date(); + } + } + + if (changesInPage > 0) { + const didCommit = await batch.commit(pageLsn, { + createEmptyCheckpoints: false, + oldestUncommittedChange: this.oldestUncommittedChange + }); + + if (didCommit) { + this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); + this.oldestUncommittedChange = null; + this.isStartingReplication = false; + } + } else if (nextCursor != cursor && Date.now() - this.lastKeepaliveAt > 60_000) { + await batch.keepalive(pageLsn); + this.lastKeepaliveAt = Date.now(); + this.isStartingReplication = false; + } + + cursor = nextCursor; + + if (!page.hasMore) { + await delay(this.pollingIntervalMs, undefined, { signal: this.abortSignal }).catch((error) => { + if (this.abortSignal.aborted) { + return; + } + throw error; + }); + } + + this.touch(); + } + } + ); + } + + async getReplicationLagMillis(): Promise { + if (this.oldestUncommittedChange == null) { + if (this.isStartingReplication) { + return undefined; + } + return 0; + } + + return Date.now() - this.oldestUncommittedChange.getTime(); + } + + private async initSlot(): Promise<{ needsInitialSync: boolean; snapshotLsn: string | null }> { + const status = await this.storage.getStatus(); + if (status.snapshot_done && status.checkpoint_lsn) { + this.logger.info('Initial replication already done'); + return { + needsInitialSync: false, + snapshotLsn: null + }; + } + + return { + needsInitialSync: true, + snapshotLsn: status.snapshot_lsn + }; + } + + private async initialReplication(snapshotLsn: string | null) { + const flushResult = await this.storage.startBatch( + { + logger: this.logger, + zeroLSN: ConvexLSN.ZERO.comparable, + defaultSchema: this.defaultSchema, + storeCurrentData: false, + skipExistingRows: true + }, + async (batch) => { + let snapshotCursor = snapshotLsn == null ? null : ConvexLSN.fromSerialized(snapshotLsn).toCursorString(); + + const sourceTables = await this.resolveAllSourceTables(batch); + if (sourceTables.length == 0 && snapshotCursor == null) { + const head = await this.connections.client.getHeadCursor({ signal: this.abortSignal }); + snapshotCursor = head; + await batch.setResumeLsn(ConvexLSN.fromCursor(head).comparable); + } + + for (const sourceTable of sourceTables) { + if (sourceTable.snapshotComplete) { + this.logger.info(`Skipping table [${sourceTable.qualifiedName}] - snapshot already done.`); + continue; + } + + const tableWithProgress = await batch.updateTableProgress(sourceTable, { + totalEstimatedCount: -1 + }); + this.relationCache.update(tableWithProgress); + + const result = await this.snapshotTable(batch, tableWithProgress, snapshotCursor); + snapshotCursor = result.snapshotCursor; + } + + if (snapshotCursor == null) { + throw new ReplicationAssertionError('Convex snapshot cursor is missing'); + } + + const snapshotComparable = ConvexLSN.fromCursor(snapshotCursor).comparable; + await batch.commit(snapshotComparable); + + this.logger.info(`Snapshot done. Need to replicate from ${snapshotComparable} for consistency.`); + } + ); + + return { + lastOpId: flushResult?.flushed_op + }; + } + + private async snapshotTable( + batch: storage.BucketStorageBatch, + table: SourceTable, + initialSnapshotCursor: string | null + ): Promise<{ table: SourceTable; snapshotCursor: string }> { + let snapshotCursor = initialSnapshotCursor; + let pageCursor = decodeSnapshotProgressCursor(table.snapshotStatus?.lastKey ?? null); + let replicatedCount = table.snapshotStatus?.replicatedCount ?? 0; + let latestTable = table; + + while (!this.abortSignal.aborted) { + const page = await this.connections.client + .listSnapshot({ + tableName: table.name, + snapshot: snapshotCursor ?? undefined, + cursor: pageCursor ?? undefined, + signal: this.abortSignal + }) + .catch((error) => { + if (isCursorExpiredError(error)) { + throw new ConvexCursorExpiredError('Convex snapshot cursor expired; restart required', error); + } + throw error; + }); + + if (snapshotCursor == null) { + snapshotCursor = page.snapshot; + await batch.setResumeLsn(ConvexLSN.fromCursor(snapshotCursor).comparable); + this.logger.info(`Marking snapshot at ${snapshotCursor}`); + } + + if (snapshotCursor != page.snapshot) { + throw new ReplicationAssertionError( + `Convex snapshot cursor changed while snapshotting ${table.qualifiedName}: ${snapshotCursor} -> ${page.snapshot}` + ); + } + + for (const rawDocument of page.values) { + if (rawDocument._deleted) { + continue; + } + + const replicaId = rawDocument._id; + if (replicaId == null) { + this.logger.warn(`Skipping Convex document without _id on table ${table.qualifiedName}`); + continue; + } + + const row = this.toSqliteRow(rawDocument); + await batch.save({ + tag: SaveOperationTag.INSERT, + sourceTable: latestTable, + before: undefined, + beforeReplicaId: undefined, + after: row, + afterReplicaId: replicaId + }); + replicatedCount += 1; + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); + } + + await batch.flush(); + + pageCursor = page.cursor; + latestTable = await batch.updateTableProgress(latestTable, { + replicatedCount, + totalEstimatedCount: -1, + lastKey: encodeSnapshotProgressCursor(pageCursor) + }); + this.relationCache.update(latestTable); + + if (!page.hasMore) { + break; + } + + this.touch(); + } + + if (this.abortSignal.aborted) { + throw new ReplicationAbortedError('Initial replication interrupted'); + } + + if (snapshotCursor == null) { + throw new ReplicationAssertionError(`Convex snapshot cursor missing for table ${table.qualifiedName}`); + } + + const snapshotComparable = ConvexLSN.fromCursor(snapshotCursor).comparable; + const [doneTable] = await batch.markSnapshotDone([latestTable], snapshotComparable); + this.relationCache.update(doneTable); + + return { + table: doneTable, + snapshotCursor + }; + } + + private async resolveAllSourceTables(batch: storage.BucketStorageBatch): Promise { + const sourceTables = this.syncRules.getSourceTables(); + const resolved: SourceTable[] = []; + + for (const tablePattern of sourceTables) { + const tables = await this.resolveQualifiedTableNames(batch, tablePattern); + resolved.push(...tables); + } + + return resolved; + } + + private async resolveQualifiedTableNames( + batch: storage.BucketStorageBatch, + tablePattern: TablePattern + ): Promise { + if (tablePattern.connectionTag != this.connections.connectionTag) { + return []; + } + + if (tablePattern.schema != this.defaultSchema) { + return []; + } + + const availableTableNames = (await this.getAllTableSchemas()).map((table) => table.tableName); + + const matchedTableNames = availableTableNames + .filter((tableName) => { + if (tablePattern.isWildcard) { + return tableName.startsWith(tablePattern.tablePrefix); + } + return tableName == tablePattern.name; + }) + .sort(); + + if (!tablePattern.isWildcard && matchedTableNames.length == 0) { + this.logger.warn(`Table ${tablePattern.schema}.${tablePattern.name} not found`); + } + + const resolved: SourceTable[] = []; + for (const tableName of matchedTableNames) { + const table = await this.processTable( + batch, + { + schema: this.defaultSchema, + name: tableName, + objectId: tableName, + replicaIdColumns: [{ name: '_id' }] + }, + false + ); + resolved.push(table); + } + + return resolved; + } + + private async getOrResolveTable(batch: storage.BucketStorageBatch, tableName: string): Promise { + const descriptor: SourceEntityDescriptor = { + schema: this.defaultSchema, + name: tableName, + objectId: tableName, + replicaIdColumns: [{ name: '_id' }] + }; + + const existing = this.relationCache.get(descriptor); + if (existing) { + return existing; + } + + if (!this.isTableSelectedBySyncRules(tableName)) { + return null; + } + + // Refresh schema cache when we discover a new table while streaming. + await this.getAllTableSchemas({ force: true }); + + return await this.processTable(batch, descriptor, false); + } + + private isTableSelectedBySyncRules(tableName: string): boolean { + for (const sourceTablePattern of this.syncRules.getSourceTables()) { + if (sourceTablePattern.connectionTag != this.connections.connectionTag) { + continue; + } + if (sourceTablePattern.schema != this.defaultSchema) { + continue; + } + + if (sourceTablePattern.isWildcard) { + if (tableName.startsWith(sourceTablePattern.tablePrefix)) { + return true; + } + } else if (sourceTablePattern.name == tableName) { + return true; + } + } + + return false; + } + + private async processTable( + batch: storage.BucketStorageBatch, + descriptor: SourceEntityDescriptor, + snapshot: boolean + ): Promise { + const resolved = await this.storage.resolveTable({ + group_id: this.storage.group_id, + connection_id: Number.parseInt(this.connections.connectionId) || 1, + connection_tag: this.connections.connectionTag, + entity_descriptor: descriptor, + sync_rules: this.syncRules + }); + + if (resolved.dropTables.length > 0) { + await batch.drop(resolved.dropTables); + } + + this.relationCache.update(resolved.table); + + if (snapshot && !resolved.table.snapshotComplete && resolved.table.syncAny) { + await batch.truncate([resolved.table]); + } + + return resolved.table; + } + + private async writeChange( + batch: storage.BucketStorageBatch, + table: SourceTable, + change: ConvexRawDocument + ): Promise { + const replicaId = change._id; + if (replicaId == null) { + this.logger.warn(`Skipping Convex change without _id for ${table.qualifiedName}`); + return false; + } + + if (change._deleted) { + await batch.save({ + tag: SaveOperationTag.DELETE, + sourceTable: table, + before: undefined, + beforeReplicaId: replicaId, + after: undefined, + afterReplicaId: undefined + }); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); + return true; + } + + const after = this.toSqliteRow(change); + await batch.save({ + tag: SaveOperationTag.UPDATE, + sourceTable: table, + before: undefined, + beforeReplicaId: undefined, + after, + afterReplicaId: replicaId + }); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); + return true; + } + + private toSqliteRow(change: ConvexRawDocument) { + const row: SqliteInputRow = {}; + + for (const [key, value] of Object.entries(change)) { + if (key == '_table' || key == '_deleted') { + continue; + } + row[key] = toConvexSyncValue(value); + } + + return this.syncRules.applyRowContext(row); + } + + private async getAllTableSchemas(options?: { force?: boolean }): Promise { + if (!options?.force && this.tableSchemaCache != null) { + return this.tableSchemaCache; + } + + const schema = await this.connections.client.getJsonSchemas({ signal: this.abortSignal }); + this.tableSchemaCache = schema.tables; + return schema.tables; + } + + private touch() { + if (performance.now() - this.lastTouchedAt < 1_000) { + return; + } + + this.lastTouchedAt = performance.now(); + container.probes.touch().catch((error) => { + this.logger.error(`Failed to touch the container probe: ${error instanceof Error ? error.message : `${error}`}`); + }); + } +} + +function getCacheIdentifier(source: SourceEntityDescriptor | SourceTable): string { + if (source instanceof SourceTable) { + return `${source.schema}.${source.name}`; + } + + return `${source.schema}.${source.name}`; +} + +function readTableName(change: ConvexRawDocument, fallback?: string): string | null { + const table = change._table ?? change.tableName ?? change.table_name ?? fallback; + if (typeof table != 'string' || table.length == 0) { + return null; + } + return table; +} + +function toConvexSyncValue(value: unknown): any { + if (value == null) { + return null; + } + + if (typeof value == 'string') { + return value; + } + + if (typeof value == 'number') { + if (Number.isInteger(value)) { + return BigInt(value); + } + return value; + } + + if (typeof value == 'bigint') { + return value; + } + + if (typeof value == 'boolean') { + return value ? 1n : 0n; + } + + if (value instanceof Uint8Array) { + return value; + } + + if (value instanceof Date) { + return value.toISOString(); + } + + if (Array.isArray(value) || typeof value == 'object') { + return JSON.stringify(value); + } + + return null; +} + +function encodeSnapshotProgressCursor(cursor: string | null): Uint8Array | null { + if (cursor == null) { + return null; + } + + return Buffer.from(cursor, 'utf8'); +} + +function decodeSnapshotProgressCursor(cursor: Uint8Array | null): string | null { + if (cursor == null) { + return null; + } + + return Buffer.from(cursor).toString('utf8'); +} diff --git a/modules/module-convex/src/replication/replication-index.ts b/modules/module-convex/src/replication/replication-index.ts new file mode 100644 index 000000000..1a3436644 --- /dev/null +++ b/modules/module-convex/src/replication/replication-index.ts @@ -0,0 +1,6 @@ +export * from './ConvexConnectionManager.js'; +export * from './ConvexConnectionManagerFactory.js'; +export * from './ConvexErrorRateLimiter.js'; +export * from './ConvexReplicator.js'; +export * from './ConvexReplicationJob.js'; +export * from './ConvexStream.js'; diff --git a/modules/module-convex/src/types/types.ts b/modules/module-convex/src/types/types.ts new file mode 100644 index 000000000..db8a23405 --- /dev/null +++ b/modules/module-convex/src/types/types.ts @@ -0,0 +1,84 @@ +import { ErrorCode, makeHostnameLookupFunction, ServiceError } from '@powersync/lib-services-framework'; +import * as service_types from '@powersync/service-types'; +import { LookupFunction } from 'node:net'; +import * as t from 'ts-codec'; + +export const CONVEX_CONNECTION_TYPE = 'convex' as const; + +export interface NormalizedConvexConnectionConfig { + id: string; + tag: string; + + deploymentUrl: string; + deployKey: string; + + debugApi: boolean; + pollingIntervalMs: number; + requestTimeoutMs: number; + + lookup?: LookupFunction; +} + +export const ConvexConnectionConfig = service_types.configFile.DataSourceConfig.and( + t.object({ + type: t.literal(CONVEX_CONNECTION_TYPE), + deployment_url: t.string, + deploy_key: t.string, + polling_interval_ms: t.number.optional(), + request_timeout_ms: t.number.optional(), + reject_ip_ranges: t.array(t.string).optional() + }) +); + +export type ConvexConnectionConfig = t.Decoded; + +export type ResolvedConvexConnectionConfig = ConvexConnectionConfig & NormalizedConvexConnectionConfig; + +export function normalizeConnectionConfig(options: ConvexConnectionConfig): NormalizedConvexConnectionConfig { + let deploymentURL: URL; + try { + deploymentURL = new URL(options.deployment_url); + } catch (error) { + throw new ServiceError( + ErrorCode.PSYNC_S1109, + `Convex connection: invalid deployment_url ${error instanceof Error ? `- ${error.message}` : ''}` + ); + } + + if (deploymentURL.protocol != 'https:' && deploymentURL.protocol != 'http:') { + throw new ServiceError( + ErrorCode.PSYNC_S1109, + `Convex connection: deployment_url must use http or https, got ${JSON.stringify(deploymentURL.protocol)}` + ); + } + + if (deploymentURL.hostname == '') { + throw new ServiceError(ErrorCode.PSYNC_S1106, `Convex connection: hostname required`); + } + + if (options.deploy_key == '') { + throw new ServiceError(ErrorCode.PSYNC_S1108, `Convex connection: deploy_key required`); + } + + const lookup = makeHostnameLookupFunction(deploymentURL.hostname, { + reject_ip_ranges: options.reject_ip_ranges ?? [] + }); + + return { + id: options.id ?? 'default', + tag: options.tag ?? 'default', + + deploymentUrl: deploymentURL.toString().replace(/\/$/, ''), + deployKey: options.deploy_key, + + debugApi: options.debug_api ?? false, + pollingIntervalMs: options.polling_interval_ms ?? 1_000, + requestTimeoutMs: options.request_timeout_ms ?? 30_000, + + lookup + }; +} + +export function baseUri(config: ResolvedConvexConnectionConfig) { + return config.deploymentUrl; +} diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts new file mode 100644 index 000000000..9fe627014 --- /dev/null +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -0,0 +1,158 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { ConvexApiClient } from '@module/client/ConvexApiClient.js'; +import { normalizeConnectionConfig } from '@module/types/types.js'; + +const baseConfig = normalizeConnectionConfig({ + type: 'convex', + deployment_url: 'https://example.convex.cloud', + deploy_key: 'test-key', + request_timeout_ms: 5000 +}); + +describe('ConvexApiClient', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('sends Convex authorization header and format=json', async () => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ + tables: { + users: { properties: { _id: { type: 'string' } } } + } + }), + { status: 200 } + ) + ); + + const client = new ConvexApiClient(baseConfig); + const result = await client.getJsonSchemas(); + + expect(result.tables.map((table) => table.tableName)).toEqual(['users']); + + const [url, init] = fetchSpy.mock.calls[0]!; + expect(String(url)).toContain('/api/json_schemas'); + expect(String(url)).toContain('format=json'); + expect((init?.headers as Record).Authorization).toBe('Convex test-key'); + }); + + it('falls back to /api/streaming_export path on 404', async () => { + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValueOnce(new Response('{}', { status: 404 })) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + snapshot: '100', + cursor: '100', + has_more: false, + values: [] + }), + { status: 200 } + ) + ); + + const client = new ConvexApiClient(baseConfig); + const page = await client.listSnapshot({ tableName: 'users' }); + + expect(page.snapshot).toBe('100'); + expect(fetchSpy.mock.calls.length).toBe(2); + expect(String(fetchSpy.mock.calls[1]![0])).toContain('/api/streaming_export/list_snapshot'); + }); + + it('reuses requested snapshot when response omits snapshot', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ + cursor: 'next-page', + has_more: true, + values: [] + }), + { status: 200 } + ) + ); + + const client = new ConvexApiClient(baseConfig); + const page = await client.listSnapshot({ + snapshot: '1770335566197683', + tableName: 'lists' + }); + + expect(page.snapshot).toBe('1770335566197683'); + expect(page.cursor).toBe('next-page'); + }); + + it('fails when first list_snapshot page omits snapshot', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ + cursor: 'next-page', + has_more: true, + values: [] + }), + { status: 200 } + ) + ); + + const client = new ConvexApiClient(baseConfig); + await expect(client.listSnapshot({ tableName: 'lists' })).rejects.toMatchObject({ + message: expect.stringContaining('missing snapshot'), + retryable: false + }); + }); + + it('preserves high-precision numeric snapshot values', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + '{"values":[],"snapshot":1770335566197682922,"cursor":"{\\"tablet\\":\\"X0yj4Cm7GfuikfsSBm9QCQ\\",\\"id\\":\\"j5700000000000000000000000001qv0\\"}","hasMore":true}', + { status: 200 } + ) + ); + + const client = new ConvexApiClient(baseConfig); + const page = await client.listSnapshot({ tableName: 'lists' }); + + expect(page.snapshot).toBe('1770335566197682922'); + expect(page.cursor).toContain('"tablet":"X0yj4Cm7GfuikfsSBm9QCQ"'); + expect(page.hasMore).toBe(true); + }); + + it('parses self-hosted top-level json_schemas table map', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ + todos: { + type: 'object', + properties: { + _id: { type: 'string' } + } + }, + lists: { + type: 'object', + properties: { + _id: { type: 'string' }, + name: { type: 'string' } + } + } + }), + { status: 200 } + ) + ); + + const client = new ConvexApiClient(baseConfig); + const result = await client.getJsonSchemas(); + + expect(result.tables.map((table) => table.tableName)).toEqual(['lists', 'todos']); + }); + + it('marks network failures as retryable', async () => { + vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('fetch failed: ECONNRESET')); + + const client = new ConvexApiClient(baseConfig); + + await expect(client.getJsonSchemas()).rejects.toMatchObject({ + retryable: true + }); + }); +}); diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts new file mode 100644 index 000000000..6430123f6 --- /dev/null +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; +import { ConvexLSN } from '@module/common/ConvexLSN.js'; + +describe('ConvexLSN', () => { + it('serializes and deserializes cursor and timestamp', () => { + const source = ConvexLSN.fromCursor('12345'); + const roundTrip = ConvexLSN.fromSerialized(source.comparable); + + expect(roundTrip.timestamp).toBe(12345n); + expect(roundTrip.toCursorString()).toBe('12345'); + }); + + it('sorts lexicographically by timestamp', () => { + const older = ConvexLSN.fromCursor('9').comparable; + const newer = ConvexLSN.fromCursor('10').comparable; + + expect(older < newer).toBe(true); + }); + + it('supports legacy plain timestamp cursor format', () => { + const parsed = ConvexLSN.fromSerialized('42'); + expect(parsed.timestamp).toBe(42n); + expect(parsed.toCursorString()).toBe('42'); + }); + + it('supports self-hosted slash-hex cursor format', () => { + const parsed = ConvexLSN.fromCursor('00000000/0183E3A8'); + const roundTrip = ConvexLSN.fromSerialized(parsed.comparable); + + expect(roundTrip.toCursorString()).toBe('00000000/0183E3A8'); + expect(roundTrip.timestamp).toBe(0x0183e3a8n); + }); +}); diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts new file mode 100644 index 000000000..51107adf8 --- /dev/null +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -0,0 +1,72 @@ +import { SqlSyncRules } from '@powersync/service-sync-rules'; +import { describe, expect, it } from 'vitest'; +import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; +import { normalizeConnectionConfig } from '@module/types/types.js'; + +function createAdapter() { + const config = normalizeConnectionConfig({ + type: 'convex', + deployment_url: 'https://example.convex.cloud', + deploy_key: 'test-key' + }); + + const adapter = new ConvexRouteAPIAdapter({ + ...config, + type: 'convex', + deployment_url: 'https://example.convex.cloud', + deploy_key: 'test-key' + }); + + (adapter as any).connectionManager.client = { + getJsonSchemas: async () => ({ + tables: [ + { + tableName: 'users', + schema: { + properties: { + _id: { type: 'string' }, + age: { type: 'integer' } + } + } + } + ], + raw: {} + }), + getHeadCursor: async () => '123' + }; + + return adapter; +} + +describe('ConvexRouteAPIAdapter', () => { + it('returns connection schema from Convex json schema', async () => { + const adapter = createAdapter(); + + const schema = await adapter.getConnectionSchema(); + expect(schema[0]?.name).toBe('convex'); + expect(schema[0]?.tables[0]?.name).toBe('users'); + + await adapter.shutdown(); + }); + + it('builds debug table info for matching patterns', async () => { + const adapter = createAdapter(); + + const syncRules = SqlSyncRules.fromYaml( + ` +bucket_definitions: + test: + data: + - SELECT _id AS id FROM users +`, + { + defaultSchema: 'convex' + } + ); + + const result = await adapter.getDebugTablesInfo(syncRules.getSourceTables(), syncRules); + expect(result[0]?.table?.name).toBe('users'); + + await adapter.shutdown(); + }); +}); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts new file mode 100644 index 000000000..6146e54c1 --- /dev/null +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -0,0 +1,200 @@ +import { SaveOperationTag, SourceTable } from '@powersync/service-core'; +import { TablePattern } from '@powersync/service-sync-rules'; +import { describe, expect, it, vi } from 'vitest'; +import { ConvexLSN } from '@module/common/ConvexLSN.js'; +import { ConvexStream } from '@module/replication/ConvexStream.js'; + +function createFakeStorage(options?: { + snapshotDone?: boolean; + snapshotLsn?: string | null; + resumeFromLsn?: string | null; +}) { + const saves: any[] = []; + const commits: string[] = []; + const keepalives: string[] = []; + const resumeLsnUpdates: string[] = []; + + const table = new SourceTable({ + id: 1, + connectionTag: 'default', + objectId: 'users', + schema: 'convex', + name: 'users', + replicaIdColumns: [{ name: '_id' }], + snapshotComplete: false + }); + + const batch: any = { + lastCheckpointLsn: null, + resumeFromLsn: options?.resumeFromLsn ?? null, + noCheckpointBeforeLsn: ConvexLSN.ZERO.comparable, + async save(record: any) { + saves.push(record); + return null; + }, + async truncate(_tables: SourceTable[]) { + return null; + }, + async drop(_tables: SourceTable[]) { + return null; + }, + async flush() { + return null; + }, + async commit(lsn: string) { + commits.push(lsn); + this.lastCheckpointLsn = lsn; + return true; + }, + async keepalive(lsn: string) { + keepalives.push(lsn); + this.lastCheckpointLsn = lsn; + return true; + }, + async setResumeLsn(lsn: string) { + resumeLsnUpdates.push(lsn); + this.resumeFromLsn = lsn; + }, + async markSnapshotDone(tables: SourceTable[], _lsn: string) { + for (const sourceTable of tables) { + sourceTable.snapshotComplete = true; + } + return tables; + }, + async updateTableProgress(sourceTable: SourceTable, progress: any) { + sourceTable.snapshotStatus = { + totalEstimatedCount: progress.totalEstimatedCount ?? sourceTable.snapshotStatus?.totalEstimatedCount ?? -1, + replicatedCount: progress.replicatedCount ?? sourceTable.snapshotStatus?.replicatedCount ?? 0, + lastKey: progress.lastKey ?? sourceTable.snapshotStatus?.lastKey ?? null + }; + return sourceTable; + } + }; + + const storage = { + group_id: 1, + getParsedSyncRules: () => ({ + getSourceTables: () => [new TablePattern('convex', 'users')], + applyRowContext: (row: Record) => row + }), + async getStatus() { + return { + active: true, + snapshot_done: options?.snapshotDone ?? false, + checkpoint_lsn: options?.snapshotDone ? ConvexLSN.fromCursor('100').comparable : null, + snapshot_lsn: options?.snapshotLsn ?? null + }; + }, + clear: vi.fn(async () => undefined), + populatePersistentChecksumCache: vi.fn(async () => ({ buckets: 0 })), + resolveTable: vi.fn(async () => ({ + table, + dropTables: [] + })), + startBatch: vi.fn(async (_options: any, callback: (batch: any) => Promise) => { + await callback(batch); + return { flushed_op: 1n }; + }), + reportError: vi.fn(async () => undefined) + }; + + return { + storage, + batch, + table, + saves, + commits, + keepalives, + resumeLsnUpdates + }; +} + +describe('ConvexStream', () => { + it('runs initial snapshot and stores resume LSN', async () => { + const context = createFakeStorage(); + const abortController = new AbortController(); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: {} }], + raw: {} + }), + listSnapshot: async () => ({ + snapshot: '100', + cursor: null, + hasMore: false, + values: [{ _table: 'users', _id: 'u1', name: 'Alice' }] + }) + } + } as any + }); + + await stream.initReplication(); + + expect(context.saves.length).toBe(1); + expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); + expect(context.resumeLsnUpdates.length).toBe(1); + expect(context.commits.at(-1)).toBe(ConvexLSN.fromCursor('100').comparable); + }); + + it('streams deltas and commits checkpoint', async () => { + const context = createFakeStorage({ + snapshotDone: true, + resumeFromLsn: ConvexLSN.fromCursor('100').comparable + }); + const abortController = new AbortController(); + + let calls = 0; + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: {} }], + raw: {} + }), + documentDeltas: async () => { + calls += 1; + setTimeout(() => abortController.abort(), 0); + return { + cursor: '101', + hasMore: false, + values: [ + { _table: 'users', _id: 'u1', name: 'Updated' }, + { _table: 'users', _id: 'u2', _deleted: true } + ] + }; + } + } + } as any + }); + + await stream.streamChanges(); + + expect(calls).toBeGreaterThan(0); + expect(context.saves.length).toBe(2); + expect(context.saves[0]?.tag).toBe(SaveOperationTag.UPDATE); + expect(context.saves[1]?.tag).toBe(SaveOperationTag.DELETE); + expect(context.commits.at(-1)).toBe(ConvexLSN.fromCursor('101').comparable); + }); +}); diff --git a/modules/module-convex/test/src/setup.ts b/modules/module-convex/test/src/setup.ts new file mode 100644 index 000000000..b14ebcec9 --- /dev/null +++ b/modules/module-convex/test/src/setup.ts @@ -0,0 +1,11 @@ +import { container } from '@powersync/lib-services-framework'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; +import { beforeAll, beforeEach } from 'vitest'; + +beforeAll(async () => { + container.registerDefaults(); +}); + +beforeEach(async () => { + METRICS_HELPER.resetMetrics(); +}); diff --git a/modules/module-convex/test/src/types.test.ts b/modules/module-convex/test/src/types.test.ts new file mode 100644 index 000000000..ab7fd4e51 --- /dev/null +++ b/modules/module-convex/test/src/types.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; +import { CONVEX_CONNECTION_TYPE, normalizeConnectionConfig } from '@module/types/types.js'; + +describe('Convex connection config', () => { + it('normalizes defaults', () => { + const config = normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'https://example.convex.cloud', + deploy_key: 'secret-key' + }); + + expect(config.id).toBe('default'); + expect(config.tag).toBe('default'); + expect(config.pollingIntervalMs).toBe(1000); + expect(config.requestTimeoutMs).toBe(30000); + expect(config.deploymentUrl).toBe('https://example.convex.cloud'); + }); + + it('throws for invalid URL', () => { + expect(() => + normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'not-a-url', + deploy_key: 'secret-key' + }) + ).toThrow(); + }); + + it('throws for empty deploy key', () => { + expect(() => + normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'https://example.convex.cloud', + deploy_key: '' + }) + ).toThrow(); + }); +}); diff --git a/modules/module-convex/test/tsconfig.json b/modules/module-convex/test/tsconfig.json new file mode 100644 index 000000000..7d296f30d --- /dev/null +++ b/modules/module-convex/test/tsconfig.json @@ -0,0 +1,25 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "baseUrl": "./", + "noEmit": true, + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true, + "paths": { + "@/*": ["../../../packages/service-core/src/*"], + "@module/*": ["../src/*"], + "@core-tests/*": ["../../../packages/service-core/test/src/*"] + } + }, + "include": ["src"], + "references": [ + { + "path": "../" + }, + { + "path": "../../../packages/service-core-tests" + } + ] +} diff --git a/modules/module-convex/tsconfig.json b/modules/module-convex/tsconfig.json new file mode 100644 index 000000000..91db1044f --- /dev/null +++ b/modules/module-convex/tsconfig.json @@ -0,0 +1,28 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "references": [ + { + "path": "../../packages/types" + }, + { + "path": "../../packages/sync-rules" + }, + { + "path": "../../packages/service-core" + }, + { + "path": "../../packages/jsonbig" + }, + { + "path": "../../libs/lib-services" + } + ] +} diff --git a/modules/module-convex/vitest.config.ts b/modules/module-convex/vitest.config.ts new file mode 100644 index 000000000..7cc2993f9 --- /dev/null +++ b/modules/module-convex/vitest.config.ts @@ -0,0 +1,3 @@ +import { serviceIntegrationTestConfig } from '../test_config'; + +export default serviceIntegrationTestConfig(__dirname); diff --git a/packages/schema/package.json b/packages/schema/package.json index 41ef31621..5c94fcfe7 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -25,8 +25,9 @@ "@powersync/service-module-postgres-storage": "workspace:*", "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-convex": "workspace:*", "@powersync/service-module-mysql": "workspace:*", "@powersync/service-types": "workspace:*", "ts-codec": "^1.3.0" } -} \ No newline at end of file +} diff --git a/packages/schema/src/scripts/compile-json-schema.ts b/packages/schema/src/scripts/compile-json-schema.ts index e5a309dac..f03edccbb 100644 --- a/packages/schema/src/scripts/compile-json-schema.ts +++ b/packages/schema/src/scripts/compile-json-schema.ts @@ -1,5 +1,6 @@ import { MongoStorageConfig } from '@powersync/service-module-mongodb-storage/types'; import { MongoConnectionConfig } from '@powersync/service-module-mongodb/types'; +import { ConvexConnectionConfig } from '@powersync/service-module-convex/types'; import { MySQLConnectionConfig } from '@powersync/service-module-mysql/types'; import { PostgresStorageConfig } from '@powersync/service-module-postgres-storage/types'; import { PostgresConnectionConfig } from '@powersync/service-module-postgres/types'; @@ -20,7 +21,8 @@ const baseShape = configFile.powerSyncConfig.props.shape; const mergedDataSourceConfig = configFile.genericDataSourceConfig .or(PostgresConnectionConfig) .or(MongoConnectionConfig) - .or(MySQLConnectionConfig); + .or(MySQLConnectionConfig) + .or(ConvexConnectionConfig); const mergedStorageConfig = configFile.GenericStorageConfig.or(PostgresStorageConfig).or(MongoStorageConfig); diff --git a/packages/schema/tsconfig.json b/packages/schema/tsconfig.json index b107a308e..034520201 100644 --- a/packages/schema/tsconfig.json +++ b/packages/schema/tsconfig.json @@ -25,6 +25,9 @@ }, { "path": "../../modules/module-mysql" + }, + { + "path": "../../modules/module-convex" } ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d990311eb..22bf665dd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -181,6 +181,31 @@ importers: specifier: 'catalog:' version: 4.0.16(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.0.16)(yaml@2.5.0) + modules/module-convex: + dependencies: + '@powersync/lib-services-framework': + specifier: workspace:* + version: link:../../libs/lib-services + '@powersync/service-core': + specifier: workspace:* + version: link:../../packages/service-core + '@powersync/service-jsonbig': + specifier: workspace:* + version: link:../../packages/jsonbig + '@powersync/service-sync-rules': + specifier: workspace:* + version: link:../../packages/sync-rules + '@powersync/service-types': + specifier: workspace:* + version: link:../../packages/types + ts-codec: + specifier: ^1.3.0 + version: 1.3.0 + devDependencies: + '@powersync/service-core-tests': + specifier: workspace:* + version: link:../../packages/service-core-tests + modules/module-core: dependencies: '@fastify/cors': @@ -542,6 +567,9 @@ importers: packages/schema: devDependencies: + '@powersync/service-module-convex': + specifier: workspace:* + version: link:../../modules/module-convex '@powersync/service-module-mongodb': specifier: workspace:* version: link:../../modules/module-mongodb @@ -739,6 +767,9 @@ importers: '@powersync/service-core': specifier: workspace:* version: link:../packages/service-core + '@powersync/service-module-convex': + specifier: workspace:* + version: link:../modules/module-convex '@powersync/service-module-core': specifier: workspace:* version: link:../modules/module-core diff --git a/service/Dockerfile b/service/Dockerfile index 47b3e1b9d..376463da4 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -23,6 +23,7 @@ COPY modules/module-mongodb/package.json modules/module-mongodb/tsconfig.json mo COPY modules/module-mongodb-storage/package.json modules/module-mongodb-storage/tsconfig.json modules/module-mongodb-storage/ COPY modules/module-mysql/package.json modules/module-mysql/tsconfig.json modules/module-mysql/ COPY modules/module-mssql/package.json modules/module-mssql/tsconfig.json modules/module-mssql/ +COPY modules/module-convex/package.json modules/module-convex/tsconfig.json modules/module-convex/ RUN corepack enable pnpm && corepack install RUN pnpm install --frozen-lockfile @@ -50,6 +51,7 @@ COPY modules/module-mongodb/src modules/module-mongodb/src/ COPY modules/module-mongodb-storage/src modules/module-mongodb-storage/src/ COPY modules/module-mysql/src modules/module-mysql/src/ COPY modules/module-mssql/src modules/module-mssql/src/ +COPY modules/module-convex/src modules/module-convex/src/ RUN pnpm build:production && \ rm -rf node_modules **/node_modules && \ diff --git a/service/package.json b/service/package.json index c812b1328..78af53730 100644 --- a/service/package.json +++ b/service/package.json @@ -20,6 +20,7 @@ "@powersync/service-module-mysql": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", "@powersync/service-module-core": "workspace:*", + "@powersync/service-module-convex": "workspace:*", "@sentry/node": "^10.2.0" }, "devDependencies": { @@ -29,4 +30,4 @@ "npm-check-updates": "^16.14.4", "ts-node": "^10.9.1" } -} \ No newline at end of file +} diff --git a/service/src/util/modules.ts b/service/src/util/modules.ts index 89ec9820f..45631cd7e 100644 --- a/service/src/util/modules.ts +++ b/service/src/util/modules.ts @@ -2,6 +2,7 @@ import * as core from '@powersync/service-core'; export const DYNAMIC_MODULES: core.ModuleLoaders = { connection: { + convex: () => import('@powersync/service-module-convex').then((module) => new module.ConvexModule()), mongodb: () => import('@powersync/service-module-mongodb').then((module) => new module.MongoModule()), mysql: () => import('@powersync/service-module-mysql').then((module) => new module.MySQLModule()), mssql: () => import('@powersync/service-module-mssql').then((module) => new module.MSSQLModule()), diff --git a/service/tsconfig.json b/service/tsconfig.json index 432b87d3b..80a86a7b4 100644 --- a/service/tsconfig.json +++ b/service/tsconfig.json @@ -36,6 +36,9 @@ { "path": "../modules/module-mysql" }, + { + "path": "../modules/module-convex" + }, { "path": "../modules/module-mssql" } diff --git a/tsconfig.json b/tsconfig.json index 4cf06cda3..fbc832e0e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -37,6 +37,9 @@ { "path": "./modules/module-mysql" }, + { + "path": "./modules/module-convex" + }, { "path": "./modules/module-mongodb" }, @@ -94,6 +97,9 @@ { "path": "./modules/module-mysql/test" }, + { + "path": "./modules/module-convex/test" + }, { "path": "./modules/module-mongodb/test" }, From 2006bda464e7e8737bac1f8e48b202fa3c48ea9f Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Feb 2026 15:08:42 -0700 Subject: [PATCH 02/86] cleaner debug log for bucket storage entries --- .../implementation/MongoBucketBatch.ts | 33 ++++++++++++++++++- .../src/storage/batch/PostgresBucketBatch.ts | 33 ++++++++++++++++++- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 2961b2abb..833be765e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -915,7 +915,7 @@ export class MongoBucketBatch return null; } - this.logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); + this.logger.debug(`Saving ${record.tag}:${formatSaveReplicaIds(record)}`); this.batch ??= new OperationBatch(); this.batch.push(new RecordOperation(record)); @@ -1122,3 +1122,34 @@ export class MongoBucketBatch export function currentBucketKey(b: CurrentBucket) { return `${b.bucket}/${b.table}/${b.id}`; } + +function formatReplicaId(value: unknown): string { + if (value == null) { + return '-'; + } + + if (typeof value == 'string' || typeof value == 'number' || typeof value == 'bigint' || typeof value == 'boolean') { + return `${value}`; + } + + try { + return JSON.stringify(value); + } catch { + return '[unserializable]'; + } +} + +function formatSaveReplicaIds(record: storage.SaveOptions): string { + switch (record.tag) { + case SaveOperationTag.INSERT: + return formatReplicaId(record.afterReplicaId); + case SaveOperationTag.DELETE: + return formatReplicaId(record.beforeReplicaId); + case SaveOperationTag.UPDATE: { + if (record.beforeReplicaId == null) { + return formatReplicaId(record.afterReplicaId); + } + return `${formatReplicaId(record.beforeReplicaId)}->${formatReplicaId(record.afterReplicaId)}`; + } + } +} diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index aa4443e1f..1a7c50e43 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -139,7 +139,7 @@ export class PostgresBucketBatch return null; } - this.logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); + this.logger.debug(`Saving ${record.tag}:${formatSaveReplicaIds(record)}`); this.batch ??= new OperationBatch(this.options.batch_limits); this.batch.push(new RecordOperation(record)); @@ -1076,3 +1076,34 @@ export const notifySyncRulesUpdate = async (db: lib_postgres.DatabaseClient, upd statement: `NOTIFY ${NOTIFICATION_CHANNEL}, '${models.ActiveCheckpointNotification.encode({ active_checkpoint: update })}'` }); }; + +function formatReplicaId(value: unknown): string { + if (value == null) { + return '-'; + } + + if (typeof value == 'string' || typeof value == 'number' || typeof value == 'bigint' || typeof value == 'boolean') { + return `${value}`; + } + + try { + return JSON.stringify(value); + } catch { + return '[unserializable]'; + } +} + +function formatSaveReplicaIds(record: storage.SaveOptions): string { + switch (record.tag) { + case storage.SaveOperationTag.INSERT: + return formatReplicaId(record.afterReplicaId); + case storage.SaveOperationTag.DELETE: + return formatReplicaId(record.beforeReplicaId); + case storage.SaveOperationTag.UPDATE: { + if (record.beforeReplicaId == null) { + return formatReplicaId(record.afterReplicaId); + } + return `${formatReplicaId(record.beforeReplicaId)}->${formatReplicaId(record.afterReplicaId)}`; + } + } +} From e113032b37c03e5c09c500d991c8837343a2250c Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Mon, 9 Feb 2026 15:01:38 -0700 Subject: [PATCH 03/86] wip: global lsn --- modules/module-convex/README.md | 7 + .../src/api/ConvexRouteAPIAdapter.ts | 3 +- .../src/client/ConvexApiClient.ts | 11 ++ .../src/replication/ConvexStream.ts | 80 +++++---- .../test/src/ConvexRouteAPIAdapter.test.ts | 16 +- .../test/src/ConvexStream.test.ts | 156 +++++++++++++++++- 6 files changed, 219 insertions(+), 54 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 05086b515..21bbd09fd 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -22,3 +22,10 @@ replication: 2. Confirm diagnostics reports the Convex source as connected. 3. Verify initial snapshot data appears in the expected bucket(s). 4. Insert/update/delete rows in Convex and verify bucket changes are replicated. + +## Snapshot behavior + +- Initial replication pins a global Convex snapshot boundary using `list_snapshot` without `tableName`. +- Table hydration then runs per selected sync-rule table using that fixed snapshot boundary. +- Each table snapshot starts from the first page (cursor omitted on the first call), and only uses cursor pagination within the current run. +- If initial replication is interrupted, restart resumes from the stored snapshot boundary but restarts table paging from page 1. diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index 203ce0b6d..0d9db6079 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -114,8 +114,7 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { } async createReplicationHead(callback: ReplicationHeadCallback): Promise { - const tableName = (await this.connectionManager.client.getJsonSchemas()).tables[0]?.tableName; - const head = await this.connectionManager.client.getHeadCursor({ tableName }); + const head = await this.connectionManager.client.getHeadCursor(); return await callback(ConvexLSN.fromCursor(head).comparable); } diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index 4dba0a5c3..82b8d02cb 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -128,7 +128,18 @@ export class ConvexApiClient { }; } + async getGlobalSnapshotCursor(options?: { signal?: AbortSignal }): Promise { + const page = await this.listSnapshot({ + signal: options?.signal + }); + return page.snapshot; + } + async getHeadCursor(options?: { tableName?: string; signal?: AbortSignal }): Promise { + if (options?.tableName == null) { + return this.getGlobalSnapshotCursor({ signal: options?.signal }); + } + const page = await this.listSnapshot({ tableName: options?.tableName, signal: options?.signal diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 5b9580ed8..1eb770980 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -18,7 +18,12 @@ import { import { HydratedSyncRules, SqliteInputRow, TablePattern } from '@powersync/service-sync-rules'; import { ReplicationMetric } from '@powersync/service-types'; import { setTimeout as delay } from 'timers/promises'; -import { ConvexRawDocument, ConvexTableSchema, isCursorExpiredError } from '../client/ConvexApiClient.js'; +import { + ConvexListSnapshotResult, + ConvexRawDocument, + ConvexTableSchema, + isCursorExpiredError +} from '../client/ConvexApiClient.js'; import { ConvexLSN } from '../common/ConvexLSN.js'; import { ConvexConnectionManager } from './ConvexConnectionManager.js'; @@ -243,14 +248,11 @@ export class ConvexStream { skipExistingRows: true }, async (batch) => { - let snapshotCursor = snapshotLsn == null ? null : ConvexLSN.fromSerialized(snapshotLsn).toCursorString(); + const snapshotCursor = await this.resolveSnapshotBoundary(snapshotLsn); + const snapshotComparable = ConvexLSN.fromCursor(snapshotCursor).comparable; + await batch.setResumeLsn(snapshotComparable); const sourceTables = await this.resolveAllSourceTables(batch); - if (sourceTables.length == 0 && snapshotCursor == null) { - const head = await this.connections.client.getHeadCursor({ signal: this.abortSignal }); - snapshotCursor = head; - await batch.setResumeLsn(ConvexLSN.fromCursor(head).comparable); - } for (const sourceTable of sourceTables) { if (sourceTable.snapshotComplete) { @@ -259,19 +261,16 @@ export class ConvexStream { } const tableWithProgress = await batch.updateTableProgress(sourceTable, { - totalEstimatedCount: -1 + totalEstimatedCount: -1, + replicatedCount: 0, + lastKey: null }); this.relationCache.update(tableWithProgress); + this.logger.info(`Starting table snapshot from first page for [${tableWithProgress.qualifiedName}]`); - const result = await this.snapshotTable(batch, tableWithProgress, snapshotCursor); - snapshotCursor = result.snapshotCursor; - } - - if (snapshotCursor == null) { - throw new ReplicationAssertionError('Convex snapshot cursor is missing'); + await this.snapshotTable(batch, tableWithProgress, snapshotCursor); } - const snapshotComparable = ConvexLSN.fromCursor(snapshotCursor).comparable; await batch.commit(snapshotComparable); this.logger.info(`Snapshot done. Need to replicate from ${snapshotComparable} for consistency.`); @@ -286,19 +285,20 @@ export class ConvexStream { private async snapshotTable( batch: storage.BucketStorageBatch, table: SourceTable, - initialSnapshotCursor: string | null - ): Promise<{ table: SourceTable; snapshotCursor: string }> { - let snapshotCursor = initialSnapshotCursor; - let pageCursor = decodeSnapshotProgressCursor(table.snapshotStatus?.lastKey ?? null); - let replicatedCount = table.snapshotStatus?.replicatedCount ?? 0; + snapshotCursor: string + ): Promise<{ table: SourceTable }> { + let pageCursor: string | null = null; + let replicatedCount = 0; let latestTable = table; + let firstPage = true; while (!this.abortSignal.aborted) { - const page = await this.connections.client + const requestCursor: string | undefined = firstPage ? undefined : (pageCursor ?? undefined); + const page: ConvexListSnapshotResult = await this.connections.client .listSnapshot({ tableName: table.name, - snapshot: snapshotCursor ?? undefined, - cursor: pageCursor ?? undefined, + snapshot: snapshotCursor, + cursor: requestCursor, signal: this.abortSignal }) .catch((error) => { @@ -307,12 +307,7 @@ export class ConvexStream { } throw error; }); - - if (snapshotCursor == null) { - snapshotCursor = page.snapshot; - await batch.setResumeLsn(ConvexLSN.fromCursor(snapshotCursor).comparable); - this.logger.info(`Marking snapshot at ${snapshotCursor}`); - } + firstPage = false; if (snapshotCursor != page.snapshot) { throw new ReplicationAssertionError( @@ -365,20 +360,27 @@ export class ConvexStream { throw new ReplicationAbortedError('Initial replication interrupted'); } - if (snapshotCursor == null) { - throw new ReplicationAssertionError(`Convex snapshot cursor missing for table ${table.qualifiedName}`); - } - const snapshotComparable = ConvexLSN.fromCursor(snapshotCursor).comparable; const [doneTable] = await batch.markSnapshotDone([latestTable], snapshotComparable); this.relationCache.update(doneTable); return { - table: doneTable, - snapshotCursor + table: doneTable }; } + private async resolveSnapshotBoundary(snapshotLsn: string | null): Promise { + if (snapshotLsn != null) { + const snapshotCursor = ConvexLSN.fromSerialized(snapshotLsn).toCursorString(); + this.logger.info(`Using existing global snapshot ${snapshotCursor}`); + return snapshotCursor; + } + + const snapshotCursor = await this.connections.client.getGlobalSnapshotCursor({ signal: this.abortSignal }); + this.logger.info(`Pinned global snapshot ${snapshotCursor}`); + return snapshotCursor; + } + private async resolveAllSourceTables(batch: storage.BucketStorageBatch): Promise { const sourceTables = this.syncRules.getSourceTables(); const resolved: SourceTable[] = []; @@ -640,11 +642,3 @@ function encodeSnapshotProgressCursor(cursor: string | null): Uint8Array | null return Buffer.from(cursor, 'utf8'); } - -function decodeSnapshotProgressCursor(cursor: Uint8Array | null): string | null { - if (cursor == null) { - return null; - } - - return Buffer.from(cursor).toString('utf8'); -} diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index 51107adf8..8a70d288a 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -1,6 +1,7 @@ import { SqlSyncRules } from '@powersync/service-sync-rules'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; +import { ConvexLSN } from '@module/common/ConvexLSN.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; function createAdapter() { @@ -69,4 +70,17 @@ bucket_definitions: await adapter.shutdown(); }); + + it('creates replication head from the global snapshot cursor', async () => { + const adapter = createAdapter(); + const getHeadCursor = vi.fn(async (_options?: any) => '123'); + (adapter as any).connectionManager.client.getHeadCursor = getHeadCursor; + + const result = await adapter.createReplicationHead(async (head) => head); + expect(result).toBe(ConvexLSN.fromCursor('123').comparable); + expect(getHeadCursor).toHaveBeenCalledTimes(1); + expect(getHeadCursor).toHaveBeenCalledWith(); + + await adapter.shutdown(); + }); }); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 6146e54c1..e70e78ea6 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -8,11 +8,17 @@ function createFakeStorage(options?: { snapshotDone?: boolean; snapshotLsn?: string | null; resumeFromLsn?: string | null; + tableSnapshotStatus?: { + totalEstimatedCount?: number; + replicatedCount?: number; + lastKey?: Uint8Array | null; + }; }) { const saves: any[] = []; const commits: string[] = []; const keepalives: string[] = []; const resumeLsnUpdates: string[] = []; + const tableProgressUpdates: any[] = []; const table = new SourceTable({ id: 1, @@ -23,6 +29,13 @@ function createFakeStorage(options?: { replicaIdColumns: [{ name: '_id' }], snapshotComplete: false }); + if (options?.tableSnapshotStatus) { + table.snapshotStatus = { + totalEstimatedCount: options.tableSnapshotStatus.totalEstimatedCount ?? -1, + replicatedCount: options.tableSnapshotStatus.replicatedCount ?? 0, + lastKey: options.tableSnapshotStatus.lastKey ?? null + }; + } const batch: any = { lastCheckpointLsn: null, @@ -62,6 +75,7 @@ function createFakeStorage(options?: { return tables; }, async updateTableProgress(sourceTable: SourceTable, progress: any) { + tableProgressUpdates.push(progress); sourceTable.snapshotStatus = { totalEstimatedCount: progress.totalEstimatedCount ?? sourceTable.snapshotStatus?.totalEstimatedCount ?? -1, replicatedCount: progress.replicatedCount ?? sourceTable.snapshotStatus?.replicatedCount ?? 0, @@ -105,14 +119,34 @@ function createFakeStorage(options?: { saves, commits, keepalives, - resumeLsnUpdates + resumeLsnUpdates, + tableProgressUpdates }; } describe('ConvexStream', () => { - it('runs initial snapshot and stores resume LSN', async () => { + it('pins a global snapshot boundary before table hydration', async () => { const context = createFakeStorage(); const abortController = new AbortController(); + const snapshotCalls: any[] = []; + const listSnapshot = vi.fn(async (options: any) => { + snapshotCalls.push(options ?? {}); + if (options?.tableName == null) { + return { + snapshot: '100', + cursor: null, + hasMore: false, + values: [] + }; + } + + return { + snapshot: '100', + cursor: null, + hasMore: false, + values: [{ _table: 'users', _id: 'u1', name: 'Alice' }] + }; + }); const stream = new ConvexStream({ abortSignal: abortController.signal, @@ -130,24 +164,130 @@ describe('ConvexStream', () => { tables: [{ tableName: 'users', schema: {} }], raw: {} }), - listSnapshot: async () => ({ - snapshot: '100', - cursor: null, - hasMore: false, - values: [{ _table: 'users', _id: 'u1', name: 'Alice' }] - }) + listSnapshot, + getGlobalSnapshotCursor: async (options?: any) => (await listSnapshot(options)).snapshot } } as any }); await stream.initReplication(); + expect(snapshotCalls.length).toBe(2); + expect(snapshotCalls[0]?.tableName).toBeUndefined(); + expect(snapshotCalls[0]?.cursor).toBeUndefined(); + expect(snapshotCalls[1]?.tableName).toBe('users'); + expect(snapshotCalls[1]?.cursor).toBeUndefined(); + expect(snapshotCalls[1]?.snapshot).toBe('100'); expect(context.saves.length).toBe(1); expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); expect(context.resumeLsnUpdates.length).toBe(1); expect(context.commits.at(-1)).toBe(ConvexLSN.fromCursor('100').comparable); }); + it('starts each table snapshot from first page, then paginates within the run', async () => { + const context = createFakeStorage({ + snapshotLsn: ConvexLSN.fromCursor('200').comparable, + tableSnapshotStatus: { + replicatedCount: 99, + totalEstimatedCount: -1, + lastKey: Buffer.from('stale-cursor', 'utf8') + } + }); + const abortController = new AbortController(); + + const snapshotCalls: any[] = []; + const listSnapshot = vi.fn(async (options: any) => { + snapshotCalls.push(options ?? {}); + if (snapshotCalls.length == 1) { + return { + snapshot: '200', + cursor: 'page-2', + hasMore: true, + values: [{ _table: 'users', _id: 'u1', name: 'Alice' }] + }; + } + return { + snapshot: '200', + cursor: null, + hasMore: false, + values: [{ _table: 'users', _id: 'u2', name: 'Bob' }] + }; + }); + + const getGlobalSnapshotCursor = vi.fn(async () => 'should-not-be-called'); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: {} }], + raw: {} + }), + listSnapshot, + getGlobalSnapshotCursor + } + } as any + }); + + await stream.initReplication(); + + expect(getGlobalSnapshotCursor).not.toHaveBeenCalled(); + expect(snapshotCalls.length).toBe(2); + expect(snapshotCalls[0]?.snapshot).toBe('200'); + expect(snapshotCalls[0]?.cursor).toBeUndefined(); + expect(snapshotCalls[1]?.cursor).toBe('page-2'); + expect(context.tableProgressUpdates[0]).toMatchObject({ + replicatedCount: 0, + lastKey: null, + totalEstimatedCount: -1 + }); + }); + + it('fails when table snapshots return a different snapshot boundary', async () => { + const context = createFakeStorage({ + snapshotLsn: ConvexLSN.fromCursor('300').comparable + }); + const abortController = new AbortController(); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: {} }], + raw: {} + }), + getGlobalSnapshotCursor: async () => 'should-not-be-called', + listSnapshot: async () => ({ + snapshot: '301', + cursor: null, + hasMore: false, + values: [{ _table: 'users', _id: 'u1', name: 'Alice' }] + }) + } + } as any + }); + + await expect(stream.initReplication()).rejects.toThrow(/snapshot cursor changed while snapshotting/); + }); + it('streams deltas and commits checkpoint', async () => { const context = createFakeStorage({ snapshotDone: true, From 33ecf73e5bc4d35ea1c618dfd5a92c792a0e415f Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Wed, 11 Feb 2026 13:58:54 -0700 Subject: [PATCH 04/86] remove single table optimization sync rules will mostly have multiple table --- modules/module-convex/src/replication/ConvexStream.ts | 8 +++----- modules/module-convex/test/src/ConvexStream.test.ts | 5 ++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 1eb770980..933c23152 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -127,9 +127,8 @@ export class ConvexStream { throw new ReplicationAssertionError(`No LSN found to resume replication from.`); } - // Resolve tables up-front so we can pass an optional single-table optimization to Convex. - const sourceTables = await this.resolveAllSourceTables(batch); - const singleTable = sourceTables.length == 1 ? sourceTables[0]!.name : undefined; + // Resolve source tables up-front to warm table metadata and sync-rule matching. + await this.resolveAllSourceTables(batch); let cursor = ConvexLSN.fromSerialized(resumeFromLsn).toCursorString(); @@ -137,7 +136,6 @@ export class ConvexStream { const page = await this.connections.client .documentDeltas({ cursor, - tableName: singleTable, signal: this.abortSignal }) .catch((error) => { @@ -156,7 +154,7 @@ export class ConvexStream { throw new ReplicationAbortedError('Replication interrupted'); } - const tableName = readTableName(change, singleTable); + const tableName = readTableName(change); if (tableName == null) { continue; } diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index e70e78ea6..39a674cd4 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -296,6 +296,7 @@ describe('ConvexStream', () => { const abortController = new AbortController(); let calls = 0; + const deltaCalls: any[] = []; const stream = new ConvexStream({ abortSignal: abortController.signal, @@ -313,8 +314,9 @@ describe('ConvexStream', () => { tables: [{ tableName: 'users', schema: {} }], raw: {} }), - documentDeltas: async () => { + documentDeltas: async (options: any) => { calls += 1; + deltaCalls.push(options ?? {}); setTimeout(() => abortController.abort(), 0); return { cursor: '101', @@ -336,5 +338,6 @@ describe('ConvexStream', () => { expect(context.saves[0]?.tag).toBe(SaveOperationTag.UPDATE); expect(context.saves[1]?.tag).toBe(SaveOperationTag.DELETE); expect(context.commits.at(-1)).toBe(ConvexLSN.fromCursor('101').comparable); + expect(deltaCalls[0]?.tableName).toBeUndefined(); }); }); From de63c814fab1f97bb5fee114b8608558292f2539 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Wed, 11 Feb 2026 14:01:44 -0700 Subject: [PATCH 05/86] expirementing with agents.md --- modules/module-convex/AGENTS.md | 112 ++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 modules/module-convex/AGENTS.md diff --git a/modules/module-convex/AGENTS.md b/modules/module-convex/AGENTS.md new file mode 100644 index 000000000..7b55d3f96 --- /dev/null +++ b/modules/module-convex/AGENTS.md @@ -0,0 +1,112 @@ +# Convex Module Agent Guide + +This file is the working contract for agents modifying `module-convex`. + +## 1) Scope +- This module replicates Convex data into PowerSync bucket storage. +- Source API is Convex Streaming Export (`json_schemas`, `list_snapshot`, `document_deltas`). +- Initial scope is default Convex component only. + +## 2) Canonical Behavior +- Initial replication: + 1. Pin one **global snapshot** via `list_snapshot` **without** `tableName` and `cursor`. + 2. Snapshot each selected sync-rules table with that fixed `snapshot`. + 3. First per-table snapshot call omits `cursor`; pagination cursor is only for later pages in the same run. + 4. Commit snapshot LSN, then switch to deltas. +- Streaming replication: + - Start from persisted resume LSN. + - Poll `document_deltas`. + - Always stream globally (no `tableName` filter), then filter locally by selected sync-rules tables. + +## 3) Hard Invariants (Do Not Break) +- `snapshot` is the consistency boundary; page `cursor` is pagination state. +- All table snapshots in a run must use the same pinned `snapshot`; if response snapshot differs, fail fast. +- On restart during initial replication: + - Reuse persisted snapshot LSN boundary. + - Restart table page walk from first page (do not resume per-table `lastKey`). +- Delta streaming starts from resume LSN (snapshot boundary), not from table page cursor. +- `tablePattern.connectionTag` and schema must match before table selection. +- Source table replica identity is `_id`. +- The overall system must ensure causal consistency of replicated data in bucket storage. + +## 4) LSN and Cursor Rules +- Never assume Convex cursors are numeric-only. +- Supported cursor shapes include: + - decimal numeric timestamp strings, + - slash-hex cursor strings (`00000000/0183E3A8`), + - opaque strings. +- Persist LSNs using `ConvexLSN` comparable format (`mode + sortable key + encoded cursor`). +- Keep raw cursor round-trip safe. + +## 5) API Client Contract +- Auth header: `Authorization: Convex `. +- Always request `format=json`. +- Fallback path support: `/api/streaming_export/...` when `/api/...` returns `404`. +- Parse large numeric JSON using `JSONBig`. +- `json_schemas` must support: + - array/object under `tables`, + - self-host top-level table map shape. +- Retry classification: + - retryable: network, timeout, 429, 5xx. + - non-retryable: malformed responses, auth/config issues. + +## 6) Sync Rules and Connection Semantics +- Default schema is `convex`. +- SQL literal reminder for sync rules: string literals must use single quotes (`'sally'`), not double quotes. + +## 7) Datatype Mapping +- Current runtime mapping in stream writer: + - `number` integer -> `BigInt` (SQLite INTEGER), + - `number` float -> REAL, + - `boolean` -> `1n`/`0n`, + - `Uint8Array` -> BLOB, + - `object/array` -> JSON TEXT. + NOTE: + - Preserve bytes as BLOB in sync rules; use explicit base64 conversion in rules only when needed by consumers. + +## 8) Checkpointing and Consistency Caveat +- `createReplicationHead` currently uses best-effort source head capture. +- There is no source-side Convex marker write equivalent to Mongo/Postgres checkpoint triggers yet. +- Treat robust write-checkpoint acknowledgement as a separate production item. + +## 9) Logging Policy +- Keep logs high-signal and bounded. +- Required snapshot logs: + - pinned or reused global snapshot, + - table snapshot start, + - snapshot completion and resume LSN. +- Avoid noisy per-row debug logs unless behind explicit debug gating. + +## 10) Known Non-Goals (For Now) +- Convex component targeting beyond default component. +- A true push stream transport (module is polling deltas). +- Read-only Convex deploy key model (Convex deploy key is full access; use network isolation and secret controls). +- Cross-source transactional guarantees beyond Convex stream ordering. + +## 11) Conformance Suite (Must Stay Green) +- Unit tests: + - `test/src/ConvexApiClient.test.ts` + - `test/src/ConvexLSN.test.ts` + - `test/src/ConvexRouteAPIAdapter.test.ts` + - `test/src/ConvexStream.test.ts` + - `test/src/types.test.ts` +- Required assertions include: + - global snapshot pin call is unfiltered, + - first per-table snapshot call omits cursor, + - pagination cursor used only for subsequent pages, + - snapshot mismatch fails fast, + - route adapter head resolves global snapshot cursor. +- Manual smoke test (self-host acceptable): + - initial snapshot visible in buckets, + - inserts/updates/deletes propagate via deltas, + - no table-not-found due to schema/tag mismatch. + +## 12) Change Checklist for Agents +- If changing snapshot/delta flow: + - update `ConvexStream` tests first. +- If changing cursor/LSN encoding: + - update `ConvexLSN` tests and backward-compat coverage. +- If changing API response parsing: + - include self-host payload fixtures in `ConvexApiClient` tests. +- If changing public behavior: + - update `README.md` and this `AGENTS.md` in the same PR. From ee1e5df6e7667f2361cdd6116f0549f58ef04c09 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Wed, 11 Feb 2026 17:00:43 -0700 Subject: [PATCH 06/86] write checkpoints --- modules/module-convex/AGENTS.md | 35 ++++-- modules/module-convex/README.md | 8 ++ .../src/api/ConvexRouteAPIAdapter.ts | 45 +++++--- .../src/client/ConvexApiClient.ts | 109 +++++++++++++++++- .../src/common/ConvexCheckpoints.ts | 30 +++++ .../src/replication/ConvexStream.ts | 12 ++ .../test/src/ConvexApiClient.test.ts | 42 +++++++ .../test/src/ConvexCheckpoints.test.ts | 16 +++ .../test/src/ConvexRouteAPIAdapter.test.ts | 7 +- .../test/src/ConvexStream.test.ts | 55 +++++++++ 10 files changed, 326 insertions(+), 33 deletions(-) create mode 100644 modules/module-convex/src/common/ConvexCheckpoints.ts create mode 100644 modules/module-convex/test/src/ConvexCheckpoints.test.ts diff --git a/modules/module-convex/AGENTS.md b/modules/module-convex/AGENTS.md index 7b55d3f96..04fea591f 100644 --- a/modules/module-convex/AGENTS.md +++ b/modules/module-convex/AGENTS.md @@ -54,7 +54,13 @@ This file is the working contract for agents modifying `module-convex`. - Default schema is `convex`. - SQL literal reminder for sync rules: string literals must use single quotes (`'sally'`), not double quotes. -## 7) Datatype Mapping +## 7) Schema Change Caveat +- Convex `json_schemas` does not provide a schema change token or revision cursor that can be checkpointed. +- Current behavior uses `json_schemas` for discovery/debug, but does not continuously diff source schema versions. +- Operational caveat: if Convex schema changes (tables or columns), developers must review and redeploy sync rules manually. +- Future improvement: cache a canonicalized `json_schemas` hash, poll periodically, and raise diagnostics when schema drift is detected. + +## 8) Datatype Mapping - Current runtime mapping in stream writer: - `number` integer -> `BigInt` (SQLite INTEGER), - `number` float -> REAL, @@ -64,12 +70,20 @@ This file is the working contract for agents modifying `module-convex`. NOTE: - Preserve bytes as BLOB in sync rules; use explicit base64 conversion in rules only when needed by consumers. -## 8) Checkpointing and Consistency Caveat -- `createReplicationHead` currently uses best-effort source head capture. -- There is no source-side Convex marker write equivalent to Mongo/Postgres checkpoint triggers yet. -- Treat robust write-checkpoint acknowledgement as a separate production item. +## 9) Checkpointing and Consistency +- `createReplicationHead` must: + 1. resolve global head cursor, + 2. write a Convex source marker via streaming import, + 3. then pass the head to callback. +- Source marker tables: + - primary: `_powersync_checkpoints` + - fallback: `powersync_checkpoints` (for deployments that reject leading underscore identifiers) + - Convex may materialize source-ingest/system-invalid names with `source_` prefix, so markers can appear as `source_powersync_checkpoints`. +- Stream handling requirement: + - checkpoint marker tables must always be excluded from replicated source tables and ignored in delta row application. + - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). -## 9) Logging Policy +## 10) Logging Policy - Keep logs high-signal and bounded. - Required snapshot logs: - pinned or reused global snapshot, @@ -77,13 +91,13 @@ This file is the working contract for agents modifying `module-convex`. - snapshot completion and resume LSN. - Avoid noisy per-row debug logs unless behind explicit debug gating. -## 10) Known Non-Goals (For Now) +## 11) Known Non-Goals (For Now) - Convex component targeting beyond default component. - A true push stream transport (module is polling deltas). - Read-only Convex deploy key model (Convex deploy key is full access; use network isolation and secret controls). - Cross-source transactional guarantees beyond Convex stream ordering. -## 11) Conformance Suite (Must Stay Green) +## 12) Conformance Suite (Must Stay Green) - Unit tests: - `test/src/ConvexApiClient.test.ts` - `test/src/ConvexLSN.test.ts` @@ -95,13 +109,14 @@ This file is the working contract for agents modifying `module-convex`. - first per-table snapshot call omits cursor, - pagination cursor used only for subsequent pages, - snapshot mismatch fails fast, - - route adapter head resolves global snapshot cursor. + - route adapter head resolves global snapshot cursor and writes a source marker, + - marker-only pages advance LSN via immediate keepalive. - Manual smoke test (self-host acceptable): - initial snapshot visible in buckets, - inserts/updates/deletes propagate via deltas, - no table-not-found due to schema/tag mismatch. -## 12) Change Checklist for Agents +## 13) Change Checklist for Agents - If changing snapshot/delta flow: - update `ConvexStream` tests first. - If changing cursor/LSN encoding: diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 21bbd09fd..05b7416c2 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -29,3 +29,11 @@ replication: - Table hydration then runs per selected sync-rule table using that fixed snapshot boundary. - Each table snapshot starts from the first page (cursor omitted on the first call), and only uses cursor pagination within the current run. - If initial replication is interrupted, restart resumes from the stored snapshot boundary but restarts table paging from page 1. + +## Write checkpoint behavior + +- Route write checkpoints create a source marker row using Convex streaming import in `_powersync_checkpoints` (fallback: `powersync_checkpoints`). +- Convex may materialize system/invalid source table names with a `source_` prefix (for example `source_powersync_checkpoints`). +- Delta polling is global, so marker rows are observed even when not referenced in sync rules. +- Checkpoint marker tables are always ignored during snapshot and delta replication. +- Marker-only pages trigger immediate keepalive checkpoint advancement (not minute-throttled), reducing write-checkpoint acknowledgement latency. diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index 0d9db6079..52f5b2c5c 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -2,6 +2,7 @@ import { api, ParseSyncRulesOptions, ReplicationHeadCallback, ReplicationLagOpti import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import { ConvexLSN } from '../common/ConvexLSN.js'; +import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; import * as types from '../types/types.js'; @@ -75,6 +76,9 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { if (tablePattern.schema != this.connectionManager.schema) { return false; } + if (isConvexCheckpointTable(name)) { + return false; + } if (tablePattern.isWildcard) { return name.startsWith(tablePattern.tablePrefix); } @@ -115,6 +119,7 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { async createReplicationHead(callback: ReplicationHeadCallback): Promise { const head = await this.connectionManager.client.getHeadCursor(); + await this.connectionManager.client.createWriteCheckpointMarker({ headCursor: head }); return await callback(ConvexLSN.fromCursor(head).comparable); } @@ -124,26 +129,28 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { return [ { name: this.connectionManager.schema, - tables: schema.tables.map((table) => ({ - name: table.tableName, - columns: Object.entries({ - _id: { type: 'string' }, - ...extractProperties(table.schema) - }) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([columnName, property]) => { - const jsonType = readJsonSchemaType(property); - const sqliteType = toSqliteType(jsonType); - - return { - name: columnName, - type: jsonType, - sqlite_type: sqliteType.typeFlags, - internal_type: jsonType, - pg_type: jsonType - }; + tables: schema.tables + .filter((table) => !isConvexCheckpointTable(table.tableName)) + .map((table) => ({ + name: table.tableName, + columns: Object.entries({ + _id: { type: 'string' }, + ...extractProperties(table.schema) }) - })) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([columnName, property]) => { + const jsonType = readJsonSchemaType(property); + const sqliteType = toSqliteType(jsonType); + + return { + name: columnName, + type: jsonType, + sqlite_type: sqliteType.typeFlags, + internal_type: jsonType, + pg_type: jsonType + }; + }) + })) } ]; } diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index 82b8d02cb..6d15faad9 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -1,6 +1,7 @@ import { setTimeout as delay } from 'timers/promises'; import { JSONBig } from '@powersync/service-jsonbig'; import { NormalizedConvexConnectionConfig } from '../types/types.js'; +import { CONVEX_CHECKPOINT_TABLE_CANDIDATES, CONVEX_CHECKPOINT_TABLE_PRIMARY } from '../common/ConvexCheckpoints.js'; export interface ConvexRawDocument { _id?: string; @@ -45,6 +46,15 @@ export interface ConvexDocumentDeltasResult { values: ConvexRawDocument[]; } +export interface ConvexImportTable { + jsonSchema: Record; +} + +export interface ConvexImportMessage { + tableName: string; + data: Record; +} + export class ConvexApiError extends Error { readonly status?: number; readonly retryable: boolean; @@ -147,6 +157,63 @@ export class ConvexApiClient { return page.snapshot; } + async importAirbyteRecords(options: { + tables: Record; + messages: ConvexImportMessage[]; + signal?: AbortSignal; + }): Promise { + await this.performRequest({ + method: 'POST', + path: '/api/streaming_import/import_airbyte_records', + body: { + tables: options.tables, + messages: options.messages + }, + signal: options.signal, + extraHeaders: { + 'Content-Type': 'application/json', + 'Convex-Client': 'streaming-import-1.0.0' + }, + includeJsonFormat: false + }); + } + + async createWriteCheckpointMarker(options?: { headCursor?: string; signal?: AbortSignal }): Promise { + const marker = { + powersync_lsn: options?.headCursor ?? (await this.getHeadCursor({ signal: options?.signal })), + powersync_created_at: Date.now(), + powersync_source: 'powersync' + }; + + let lastError: unknown; + for (const tableName of CONVEX_CHECKPOINT_TABLE_CANDIDATES) { + try { + await this.importAirbyteRecords({ + tables: { + [tableName]: { + jsonSchema: POWERSYNC_CHECKPOINT_SCHEMA + } + }, + messages: [ + { + tableName, + data: marker + } + ], + signal: options?.signal + }); + return; + } catch (error) { + lastError = error; + if (!shouldRetryCheckpointWithFallbackTable(error, tableName)) { + throw error; + } + } + } + + throw lastError ?? new Error('Unable to create Convex write checkpoint marker'); + } + private async requestJson(options: { endpoint: string; params?: Record; @@ -182,12 +249,18 @@ export class ConvexApiClient { } private async performRequest(options: { + method?: 'GET' | 'POST'; path: string; params?: Record; + body?: unknown; signal?: AbortSignal; + extraHeaders?: Record; + includeJsonFormat?: boolean; }): Promise> { const url = new URL(options.path, this.config.deploymentUrl); - url.searchParams.set('format', 'json'); + if (options.includeJsonFormat ?? true) { + url.searchParams.set('format', 'json'); + } for (const [key, value] of Object.entries(options.params ?? {})) { if (value == null) { @@ -212,11 +285,13 @@ export class ConvexApiClient { try { const response = await fetch(url, { - method: 'GET', + method: options.method ?? 'GET', headers: { Authorization: `Convex ${this.config.deployKey}`, - Accept: 'application/json' + Accept: 'application/json', + ...(options.extraHeaders ?? {}) }, + body: options.body == null ? undefined : JSON.stringify(options.body), signal }); @@ -418,6 +493,22 @@ function isRecord(value: unknown): value is Record { return typeof value == 'object' && value != null && !Array.isArray(value); } +function shouldRetryCheckpointWithFallbackTable(error: unknown, tableName: string): boolean { + if (tableName != CONVEX_CHECKPOINT_TABLE_PRIMARY) { + return false; + } + + if (!(error instanceof ConvexApiError)) { + return false; + } + + if (error.retryable || error.status != 400) { + return false; + } + + return true; +} + const RESERVED_SCHEMA_KEYS = new Set([ 'tables', 'schema', @@ -432,6 +523,18 @@ const RESERVED_SCHEMA_KEYS = new Set([ 'errors' ]); +const POWERSYNC_CHECKPOINT_SCHEMA = { + type: 'object', + properties: { + powersync_lsn: { type: 'string' }, + powersync_created_at: { type: 'number' }, + powersync_source: { type: 'string' } + }, + additionalProperties: false, + required: ['powersync_lsn', 'powersync_created_at', 'powersync_source'], + $schema: 'http://json-schema.org/draft-07/schema#' +} as const; + function looksLikeJsonSchema(value: unknown): boolean { if (!isRecord(value)) { return false; diff --git a/modules/module-convex/src/common/ConvexCheckpoints.ts b/modules/module-convex/src/common/ConvexCheckpoints.ts new file mode 100644 index 000000000..a9b1fecb9 --- /dev/null +++ b/modules/module-convex/src/common/ConvexCheckpoints.ts @@ -0,0 +1,30 @@ +export const CONVEX_CHECKPOINT_TABLE_PRIMARY = '_powersync_checkpoints'; +export const CONVEX_CHECKPOINT_TABLE_FALLBACK = 'powersync_checkpoints'; +export const CONVEX_CHECKPOINT_TABLE_PREFIX = 'source_'; + +export const CONVEX_CHECKPOINT_TABLE_CANDIDATES = [ + CONVEX_CHECKPOINT_TABLE_PRIMARY, + CONVEX_CHECKPOINT_TABLE_FALLBACK +] as const; + +export function isConvexCheckpointTable(tableName: string): boolean { + let candidate = tableName; + + for (let depth = 0; depth < 4; depth += 1) { + if ( + CONVEX_CHECKPOINT_TABLE_CANDIDATES.includes( + candidate as (typeof CONVEX_CHECKPOINT_TABLE_CANDIDATES)[number] + ) + ) { + return true; + } + + if (!candidate.startsWith(CONVEX_CHECKPOINT_TABLE_PREFIX)) { + return false; + } + + candidate = candidate.slice(CONVEX_CHECKPOINT_TABLE_PREFIX.length); + } + + return false; +} diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 933c23152..6751554b4 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -24,6 +24,7 @@ import { ConvexTableSchema, isCursorExpiredError } from '../client/ConvexApiClient.js'; +import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { ConvexLSN } from '../common/ConvexLSN.js'; import { ConvexConnectionManager } from './ConvexConnectionManager.js'; @@ -149,6 +150,7 @@ export class ConvexStream { const pageLsn = ConvexLSN.fromCursor(nextCursor).comparable; let changesInPage = 0; + let sawCheckpointMarker = false; for (const change of page.values) { if (this.abortSignal.aborted) { throw new ReplicationAbortedError('Replication interrupted'); @@ -159,6 +161,11 @@ export class ConvexStream { continue; } + if (isConvexCheckpointTable(tableName)) { + sawCheckpointMarker = true; + continue; + } + const table = await this.getOrResolveTable(batch, tableName); if (table == null || !table.syncAny) { continue; @@ -186,6 +193,10 @@ export class ConvexStream { this.oldestUncommittedChange = null; this.isStartingReplication = false; } + } else if (sawCheckpointMarker) { + await batch.keepalive(pageLsn); + this.lastKeepaliveAt = Date.now(); + this.isStartingReplication = false; } else if (nextCursor != cursor && Date.now() - this.lastKeepaliveAt > 60_000) { await batch.keepalive(pageLsn); this.lastKeepaliveAt = Date.now(); @@ -412,6 +423,7 @@ export class ConvexStream { } return tableName == tablePattern.name; }) + .filter((tableName) => !isConvexCheckpointTable(tableName)) .sort(); if (!tablePattern.isWildcard && matchedTableNames.length == 0) { diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index 9fe627014..897b2114f 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -155,4 +155,46 @@ describe('ConvexApiClient', () => { retryable: true }); }); + + it('creates write checkpoint markers via streaming import', async () => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('', { status: 200 })); + + const client = new ConvexApiClient(baseConfig); + await client.createWriteCheckpointMarker({ headCursor: '123' }); + + expect(fetchSpy).toHaveBeenCalledTimes(1); + const [url, init] = fetchSpy.mock.calls[0]!; + expect(String(url)).toContain('/api/streaming_import/import_airbyte_records'); + expect(init?.method).toBe('POST'); + expect((init?.headers as Record).Authorization).toBe('Convex test-key'); + expect((init?.headers as Record)['Convex-Client']).toBe('streaming-import-1.0.0'); + + const body = JSON.parse(String(init?.body)); + expect(body.messages).toHaveLength(1); + expect(body.messages[0].tableName).toBe('_powersync_checkpoints'); + expect(body.messages[0].data.powersync_lsn).toBe('123'); + }); + + it('falls back to an alternate checkpoint table name on table validation errors', async () => { + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + code: 'InvalidTableName' + }), + { status: 400 } + ) + ) + .mockResolvedValueOnce(new Response('', { status: 200 })); + + const client = new ConvexApiClient(baseConfig); + await client.createWriteCheckpointMarker({ headCursor: '123' }); + + expect(fetchSpy).toHaveBeenCalledTimes(2); + const firstBody = JSON.parse(String(fetchSpy.mock.calls[0]![1]?.body)); + const secondBody = JSON.parse(String(fetchSpy.mock.calls[1]![1]?.body)); + expect(firstBody.messages[0].tableName).toBe('_powersync_checkpoints'); + expect(secondBody.messages[0].tableName).toBe('powersync_checkpoints'); + }); }); diff --git a/modules/module-convex/test/src/ConvexCheckpoints.test.ts b/modules/module-convex/test/src/ConvexCheckpoints.test.ts new file mode 100644 index 000000000..43fd94fa7 --- /dev/null +++ b/modules/module-convex/test/src/ConvexCheckpoints.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest'; +import { isConvexCheckpointTable } from '@module/common/ConvexCheckpoints.js'; + +describe('ConvexCheckpoints', () => { + it('recognizes checkpoint table names including source-prefixed variants', () => { + expect(isConvexCheckpointTable('_powersync_checkpoints')).toBe(true); + expect(isConvexCheckpointTable('powersync_checkpoints')).toBe(true); + expect(isConvexCheckpointTable('source_powersync_checkpoints')).toBe(true); + expect(isConvexCheckpointTable('source__powersync_checkpoints')).toBe(true); + }); + + it('does not match non-checkpoint tables', () => { + expect(isConvexCheckpointTable('lists')).toBe(false); + expect(isConvexCheckpointTable('source_lists')).toBe(false); + }); +}); diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index 8a70d288a..45c4af584 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -33,7 +33,8 @@ function createAdapter() { ], raw: {} }), - getHeadCursor: async () => '123' + getHeadCursor: async () => '123', + createWriteCheckpointMarker: async () => undefined }; return adapter; @@ -74,12 +75,16 @@ bucket_definitions: it('creates replication head from the global snapshot cursor', async () => { const adapter = createAdapter(); const getHeadCursor = vi.fn(async (_options?: any) => '123'); + const createWriteCheckpointMarker = vi.fn(async (_options?: any) => undefined); (adapter as any).connectionManager.client.getHeadCursor = getHeadCursor; + (adapter as any).connectionManager.client.createWriteCheckpointMarker = createWriteCheckpointMarker; const result = await adapter.createReplicationHead(async (head) => head); expect(result).toBe(ConvexLSN.fromCursor('123').comparable); expect(getHeadCursor).toHaveBeenCalledTimes(1); expect(getHeadCursor).toHaveBeenCalledWith(); + expect(createWriteCheckpointMarker).toHaveBeenCalledTimes(1); + expect(createWriteCheckpointMarker).toHaveBeenCalledWith({ headCursor: '123' }); await adapter.shutdown(); }); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 39a674cd4..e6a7009e8 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -340,4 +340,59 @@ describe('ConvexStream', () => { expect(context.commits.at(-1)).toBe(ConvexLSN.fromCursor('101').comparable); expect(deltaCalls[0]?.tableName).toBeUndefined(); }); + + it('keeps alive immediately when only checkpoint marker rows are streamed', async () => { + const context = createFakeStorage({ + snapshotDone: true, + resumeFromLsn: ConvexLSN.fromCursor('100').comparable + }); + const abortController = new AbortController(); + let calls = 0; + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: {} }], + raw: {} + }), + documentDeltas: async () => { + calls += 1; + if (calls == 1) { + return { + cursor: '101', + hasMore: true, + values: [] + }; + } + + setTimeout(() => abortController.abort(), 0); + return { + cursor: '102', + hasMore: false, + values: [{ _table: 'source_powersync_checkpoints', _id: 'cp1' }] + }; + } + } + } as any + }); + + await stream.streamChanges(); + + expect(context.saves.length).toBe(0); + expect(context.commits.length).toBe(0); + expect(context.keepalives).toEqual([ + ConvexLSN.fromCursor('101').comparable, + ConvexLSN.fromCursor('102').comparable + ]); + }); }); From facb78c0005571a48a043f20eaccc489a76dd3a1 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Mon, 16 Feb 2026 21:12:45 -0700 Subject: [PATCH 07/86] tighten convex LSN format --- modules/module-convex/AGENTS.md | 1 - modules/module-convex/src/common/ConvexLSN.ts | 14 -------------- modules/module-convex/test/src/ConvexLSN.test.ts | 8 -------- 3 files changed, 23 deletions(-) diff --git a/modules/module-convex/AGENTS.md b/modules/module-convex/AGENTS.md index 04fea591f..593efb2ad 100644 --- a/modules/module-convex/AGENTS.md +++ b/modules/module-convex/AGENTS.md @@ -33,7 +33,6 @@ This file is the working contract for agents modifying `module-convex`. - Never assume Convex cursors are numeric-only. - Supported cursor shapes include: - decimal numeric timestamp strings, - - slash-hex cursor strings (`00000000/0183E3A8`), - opaque strings. - Persist LSNs using `ConvexLSN` comparable format (`mode + sortable key + encoded cursor`). - Keep raw cursor round-trip safe. diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index e06f4a55f..b5592c695 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -81,20 +81,6 @@ function getNumericSortKey(cursor: string): bigint | null { return BigInt(cursor); } - // Common Convex-style cursors on self-hosted deployments, e.g. "00000000/0183E3A8". - const slashHex = cursor.match(/^([0-9A-Fa-f]{8})\/([0-9A-Fa-f]{8})$/); - if (slashHex) { - const high = BigInt(`0x${slashHex[1]}`); - const low = BigInt(`0x${slashHex[2]}`); - return (high << 32n) + low; - } - - // MSSQL-style hexadecimal LSN string, useful for migration compatibility. - const colonHex = cursor.match(/^([0-9A-Fa-f]{8}):([0-9A-Fa-f]{8}):([0-9A-Fa-f]{4})$/); - if (colonHex) { - return BigInt(`0x${colonHex[1]}${colonHex[2]}${colonHex[3]}`); - } - return null; } diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts index 6430123f6..d68884ddd 100644 --- a/modules/module-convex/test/src/ConvexLSN.test.ts +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -22,12 +22,4 @@ describe('ConvexLSN', () => { expect(parsed.timestamp).toBe(42n); expect(parsed.toCursorString()).toBe('42'); }); - - it('supports self-hosted slash-hex cursor format', () => { - const parsed = ConvexLSN.fromCursor('00000000/0183E3A8'); - const roundTrip = ConvexLSN.fromSerialized(parsed.comparable); - - expect(roundTrip.toCursorString()).toBe('00000000/0183E3A8'); - expect(roundTrip.timestamp).toBe(0x0183e3a8n); - }); }); From a25cd1b9c1f7a1dd4178b12aff4d05478b8f2b1c Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Mon, 16 Feb 2026 21:43:47 -0700 Subject: [PATCH 08/86] cleanup --- modules/module-convex/AGENTS.md | 8 +- modules/module-convex/README.md | 2 +- .../src/client/ConvexApiClient.ts | 92 ++++++------------- .../src/common/ConvexCheckpoints.ts | 35 ++++--- modules/module-convex/src/common/ConvexLSN.ts | 10 +- .../src/replication/ConvexStream.ts | 34 ++----- .../test/src/ConvexApiClient.test.ts | 31 +++---- .../test/src/ConvexCheckpoints.test.ts | 8 +- .../module-convex/test/src/ConvexLSN.test.ts | 11 ++- 9 files changed, 94 insertions(+), 137 deletions(-) diff --git a/modules/module-convex/AGENTS.md b/modules/module-convex/AGENTS.md index 593efb2ad..f0d2ce1fe 100644 --- a/modules/module-convex/AGENTS.md +++ b/modules/module-convex/AGENTS.md @@ -74,10 +74,9 @@ This file is the working contract for agents modifying `module-convex`. 1. resolve global head cursor, 2. write a Convex source marker via streaming import, 3. then pass the head to callback. -- Source marker tables: - - primary: `_powersync_checkpoints` - - fallback: `powersync_checkpoints` (for deployments that reject leading underscore identifiers) - - Convex may materialize source-ingest/system-invalid names with `source_` prefix, so markers can appear as `source_powersync_checkpoints`. +- Source marker table: `powersync_checkpoints` + - Convex rejects table names starting with `_`, so no leading-underscore variant is used. + - Convex streaming import materializes ingested tables with a `source_` prefix, so the marker may appear as `source_powersync_checkpoints` in streaming export. - Stream handling requirement: - checkpoint marker tables must always be excluded from replicated source tables and ignored in delta row application. - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). @@ -99,6 +98,7 @@ This file is the working contract for agents modifying `module-convex`. ## 12) Conformance Suite (Must Stay Green) - Unit tests: - `test/src/ConvexApiClient.test.ts` + - `test/src/ConvexCheckpoints.test.ts` - `test/src/ConvexLSN.test.ts` - `test/src/ConvexRouteAPIAdapter.test.ts` - `test/src/ConvexStream.test.ts` diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 05b7416c2..235c7ea90 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -32,7 +32,7 @@ replication: ## Write checkpoint behavior -- Route write checkpoints create a source marker row using Convex streaming import in `_powersync_checkpoints` (fallback: `powersync_checkpoints`). +- Route write checkpoints create a source marker row using Convex streaming import in `powersync_checkpoints`. - Convex may materialize system/invalid source table names with a `source_` prefix (for example `source_powersync_checkpoints`). - Delta polling is global, so marker rows are observed even when not referenced in sync rules. - Checkpoint marker tables are always ignored during snapshot and delta replication. diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index 6d15faad9..71d9784c5 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -1,7 +1,7 @@ import { setTimeout as delay } from 'timers/promises'; import { JSONBig } from '@powersync/service-jsonbig'; import { NormalizedConvexConnectionConfig } from '../types/types.js'; -import { CONVEX_CHECKPOINT_TABLE_CANDIDATES, CONVEX_CHECKPOINT_TABLE_PRIMARY } from '../common/ConvexCheckpoints.js'; +import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; export interface ConvexRawDocument { _id?: string; @@ -36,7 +36,6 @@ export interface ConvexListSnapshotResult { export interface ConvexDocumentDeltasOptions { cursor?: string; - tableName?: string; signal?: AbortSignal; } @@ -67,14 +66,11 @@ export class ConvexApiError extends Error { body?: unknown; cause?: unknown; }) { - super(options.message); + super(options.message, options.cause !== undefined ? { cause: options.cause } : undefined); this.name = 'ConvexApiError'; this.status = options.status; this.retryable = options.retryable; this.body = options.body; - if (options.cause !== undefined) { - (this as Error & { cause?: unknown }).cause = options.cause; - } } } @@ -100,8 +96,7 @@ export class ConvexApiClient { params: { snapshot: options.snapshot, cursor: options.cursor, - tableName: options.tableName, - table_name: options.tableName + tableName: options.tableName }, signal: options.signal, allowStreamingFallback: true @@ -123,9 +118,7 @@ export class ConvexApiClient { const payload = await this.requestJson({ endpoint: 'document_deltas', params: { - cursor: options.cursor, - tableName: options.tableName, - table_name: options.tableName + cursor: options.cursor }, signal: options.signal, allowStreamingFallback: true @@ -145,16 +138,8 @@ export class ConvexApiClient { return page.snapshot; } - async getHeadCursor(options?: { tableName?: string; signal?: AbortSignal }): Promise { - if (options?.tableName == null) { - return this.getGlobalSnapshotCursor({ signal: options?.signal }); - } - - const page = await this.listSnapshot({ - tableName: options?.tableName, - signal: options?.signal - }); - return page.snapshot; + async getHeadCursor(options?: { signal?: AbortSignal }): Promise { + return this.getGlobalSnapshotCursor({ signal: options?.signal }); } async importAirbyteRecords(options: { @@ -185,33 +170,20 @@ export class ConvexApiClient { powersync_source: 'powersync' }; - let lastError: unknown; - for (const tableName of CONVEX_CHECKPOINT_TABLE_CANDIDATES) { - try { - await this.importAirbyteRecords({ - tables: { - [tableName]: { - jsonSchema: POWERSYNC_CHECKPOINT_SCHEMA - } - }, - messages: [ - { - tableName, - data: marker - } - ], - signal: options?.signal - }); - return; - } catch (error) { - lastError = error; - if (!shouldRetryCheckpointWithFallbackTable(error, tableName)) { - throw error; + await this.importAirbyteRecords({ + tables: { + [CONVEX_CHECKPOINT_TABLE]: { + jsonSchema: POWERSYNC_CHECKPOINT_SCHEMA } - } - } - - throw lastError ?? new Error('Unable to create Convex write checkpoint marker'); + }, + messages: [ + { + tableName: CONVEX_CHECKPOINT_TABLE, + data: marker + } + ], + signal: options?.signal + }); } private async requestJson(options: { @@ -324,7 +296,15 @@ export class ConvexApiClient { } const message = error instanceof Error ? error.message : `${error}`; - const retryable = message.includes('timed out') || message.includes('fetch') || message.includes('ECONN'); + const retryable = + message.includes('timed out') || + message.includes('fetch failed') || + message.includes('ECONNREFUSED') || + message.includes('ECONNRESET') || + message.includes('ENOTFOUND') || + message.includes('EAI_AGAIN') || + message.includes('ETIMEDOUT') || + error instanceof DOMException; throw new ConvexApiError({ message: `Convex API request failed for ${url.pathname}: ${message}`, retryable, @@ -493,22 +473,6 @@ function isRecord(value: unknown): value is Record { return typeof value == 'object' && value != null && !Array.isArray(value); } -function shouldRetryCheckpointWithFallbackTable(error: unknown, tableName: string): boolean { - if (tableName != CONVEX_CHECKPOINT_TABLE_PRIMARY) { - return false; - } - - if (!(error instanceof ConvexApiError)) { - return false; - } - - if (error.retryable || error.status != 400) { - return false; - } - - return true; -} - const RESERVED_SCHEMA_KEYS = new Set([ 'tables', 'schema', diff --git a/modules/module-convex/src/common/ConvexCheckpoints.ts b/modules/module-convex/src/common/ConvexCheckpoints.ts index a9b1fecb9..5e86aa7bb 100644 --- a/modules/module-convex/src/common/ConvexCheckpoints.ts +++ b/modules/module-convex/src/common/ConvexCheckpoints.ts @@ -1,29 +1,36 @@ -export const CONVEX_CHECKPOINT_TABLE_PRIMARY = '_powersync_checkpoints'; -export const CONVEX_CHECKPOINT_TABLE_FALLBACK = 'powersync_checkpoints'; -export const CONVEX_CHECKPOINT_TABLE_PREFIX = 'source_'; +/** + * Table name used by PowerSync to write checkpoint markers into Convex via streaming import. + * + * Convex rejects table names starting with `_`, so we use a plain prefix. + * Convex streaming import may materialize this table with a `source_` prefix + * (e.g. `source_powersync_checkpoints`), so {@link isConvexCheckpointTable} + * strips that prefix when matching. + */ +export const CONVEX_CHECKPOINT_TABLE = 'powersync_checkpoints'; -export const CONVEX_CHECKPOINT_TABLE_CANDIDATES = [ - CONVEX_CHECKPOINT_TABLE_PRIMARY, - CONVEX_CHECKPOINT_TABLE_FALLBACK -] as const; +/** + * Prefix that Convex streaming import prepends to ingested table names. + */ +export const CONVEX_SOURCE_TABLE_PREFIX = 'source_'; +/** + * Returns true if `tableName` is the PowerSync checkpoint marker table, + * accounting for zero or more `source_` prefixes added by Convex. + */ export function isConvexCheckpointTable(tableName: string): boolean { let candidate = tableName; + // Strip up to 4 layers of source_ prefix (defensive; typically 0-1). for (let depth = 0; depth < 4; depth += 1) { - if ( - CONVEX_CHECKPOINT_TABLE_CANDIDATES.includes( - candidate as (typeof CONVEX_CHECKPOINT_TABLE_CANDIDATES)[number] - ) - ) { + if (candidate === CONVEX_CHECKPOINT_TABLE) { return true; } - if (!candidate.startsWith(CONVEX_CHECKPOINT_TABLE_PREFIX)) { + if (!candidate.startsWith(CONVEX_SOURCE_TABLE_PREFIX)) { return false; } - candidate = candidate.slice(CONVEX_CHECKPOINT_TABLE_PREFIX.length); + candidate = candidate.slice(CONVEX_SOURCE_TABLE_PREFIX.length); } return false; diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index b5592c695..8cc4db362 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -14,7 +14,7 @@ export class ConvexLSN { static fromSerialized(comparable: string): ConvexLSN { if (!comparable.includes(DELIMITER)) { - // Legacy support: raw cursor persisted directly. + // Bare cursor string (no encoded format). Treat as a raw cursor. return ConvexLSN.fromCursor(comparable); } @@ -25,13 +25,7 @@ export class ConvexLSN { return ConvexLSN.fromCursor(decoded); } - // Backwards compatibility for early PoC format: "|". - if (/^[0-9]+$/.test(first)) { - const decoded = decodeCursor(second ?? ''); - return ConvexLSN.fromCursor(decoded.length == 0 ? first : decoded); - } - - // Unknown future format: treat whole input as an opaque cursor. + // Unrecognized format: treat entire input as an opaque cursor. return ConvexLSN.fromCursor(comparable); } diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 6751554b4..e74eedcc0 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -432,16 +432,12 @@ export class ConvexStream { const resolved: SourceTable[] = []; for (const tableName of matchedTableNames) { - const table = await this.processTable( - batch, - { - schema: this.defaultSchema, - name: tableName, - objectId: tableName, - replicaIdColumns: [{ name: '_id' }] - }, - false - ); + const table = await this.processTable(batch, { + schema: this.defaultSchema, + name: tableName, + objectId: tableName, + replicaIdColumns: [{ name: '_id' }] + }); resolved.push(table); } @@ -468,7 +464,7 @@ export class ConvexStream { // Refresh schema cache when we discover a new table while streaming. await this.getAllTableSchemas({ force: true }); - return await this.processTable(batch, descriptor, false); + return await this.processTable(batch, descriptor); } private isTableSelectedBySyncRules(tableName: string): boolean { @@ -494,8 +490,7 @@ export class ConvexStream { private async processTable( batch: storage.BucketStorageBatch, - descriptor: SourceEntityDescriptor, - snapshot: boolean + descriptor: SourceEntityDescriptor ): Promise { const resolved = await this.storage.resolveTable({ group_id: this.storage.group_id, @@ -510,11 +505,6 @@ export class ConvexStream { } this.relationCache.update(resolved.table); - - if (snapshot && !resolved.table.snapshotComplete && resolved.table.syncAny) { - await batch.truncate([resolved.table]); - } - return resolved.table; } @@ -591,15 +581,11 @@ export class ConvexStream { } function getCacheIdentifier(source: SourceEntityDescriptor | SourceTable): string { - if (source instanceof SourceTable) { - return `${source.schema}.${source.name}`; - } - return `${source.schema}.${source.name}`; } -function readTableName(change: ConvexRawDocument, fallback?: string): string | null { - const table = change._table ?? change.tableName ?? change.table_name ?? fallback; +function readTableName(change: ConvexRawDocument): string | null { + const table = change._table; if (typeof table != 'string' || table.length == 0) { return null; } diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index 897b2114f..d6d797c12 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -171,30 +171,23 @@ describe('ConvexApiClient', () => { const body = JSON.parse(String(init?.body)); expect(body.messages).toHaveLength(1); - expect(body.messages[0].tableName).toBe('_powersync_checkpoints'); + expect(body.messages[0].tableName).toBe('powersync_checkpoints'); expect(body.messages[0].data.powersync_lsn).toBe('123'); + expect(body.tables).toHaveProperty('powersync_checkpoints'); }); - it('falls back to an alternate checkpoint table name on table validation errors', async () => { - const fetchSpy = vi - .spyOn(globalThis, 'fetch') - .mockResolvedValueOnce( - new Response( - JSON.stringify({ - code: 'InvalidTableName' - }), - { status: 400 } - ) + it('propagates checkpoint write errors directly (no fallback)', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( + new Response( + JSON.stringify({ code: 'SomeError' }), + { status: 400 } ) - .mockResolvedValueOnce(new Response('', { status: 200 })); + ); const client = new ConvexApiClient(baseConfig); - await client.createWriteCheckpointMarker({ headCursor: '123' }); - - expect(fetchSpy).toHaveBeenCalledTimes(2); - const firstBody = JSON.parse(String(fetchSpy.mock.calls[0]![1]?.body)); - const secondBody = JSON.parse(String(fetchSpy.mock.calls[1]![1]?.body)); - expect(firstBody.messages[0].tableName).toBe('_powersync_checkpoints'); - expect(secondBody.messages[0].tableName).toBe('powersync_checkpoints'); + await expect(client.createWriteCheckpointMarker({ headCursor: '123' })).rejects.toMatchObject({ + status: 400, + retryable: false + }); }); }); diff --git a/modules/module-convex/test/src/ConvexCheckpoints.test.ts b/modules/module-convex/test/src/ConvexCheckpoints.test.ts index 43fd94fa7..4249533e7 100644 --- a/modules/module-convex/test/src/ConvexCheckpoints.test.ts +++ b/modules/module-convex/test/src/ConvexCheckpoints.test.ts @@ -3,14 +3,18 @@ import { isConvexCheckpointTable } from '@module/common/ConvexCheckpoints.js'; describe('ConvexCheckpoints', () => { it('recognizes checkpoint table names including source-prefixed variants', () => { - expect(isConvexCheckpointTable('_powersync_checkpoints')).toBe(true); expect(isConvexCheckpointTable('powersync_checkpoints')).toBe(true); expect(isConvexCheckpointTable('source_powersync_checkpoints')).toBe(true); - expect(isConvexCheckpointTable('source__powersync_checkpoints')).toBe(true); + expect(isConvexCheckpointTable('source_source_powersync_checkpoints')).toBe(true); + }); + + it('does not match underscore-prefixed variant', () => { + expect(isConvexCheckpointTable('_powersync_checkpoints')).toBe(false); }); it('does not match non-checkpoint tables', () => { expect(isConvexCheckpointTable('lists')).toBe(false); expect(isConvexCheckpointTable('source_lists')).toBe(false); + expect(isConvexCheckpointTable('powersync_other')).toBe(false); }); }); diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts index d68884ddd..94922a8a2 100644 --- a/modules/module-convex/test/src/ConvexLSN.test.ts +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -17,9 +17,18 @@ describe('ConvexLSN', () => { expect(older < newer).toBe(true); }); - it('supports legacy plain timestamp cursor format', () => { + it('handles bare numeric cursor string (no delimiter)', () => { const parsed = ConvexLSN.fromSerialized('42'); expect(parsed.timestamp).toBe(42n); expect(parsed.toCursorString()).toBe('42'); }); + + it('round-trips opaque (non-numeric) cursors', () => { + const opaque = '{"tablet":"abc","id":"xyz"}'; + const source = ConvexLSN.fromCursor(opaque); + const roundTrip = ConvexLSN.fromSerialized(source.comparable); + + expect(roundTrip.toCursorString()).toBe(opaque); + expect(roundTrip.timestamp).toBe(0n); + }); }); From a34c0b577074b07e7bb2dbde50685570aefec7c8 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Mon, 16 Feb 2026 22:38:45 -0700 Subject: [PATCH 09/86] snake_case fix --- .../src/client/ConvexApiClient.ts | 3 ++- .../test/src/ConvexApiClient.test.ts | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index 71d9784c5..e22cbc8eb 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -96,7 +96,8 @@ export class ConvexApiClient { params: { snapshot: options.snapshot, cursor: options.cursor, - tableName: options.tableName + //TODO: NB: test this against cloud. Seems to be `tableName` in the docs, but `table_name` when self-hosting. + table_name: options.tableName }, signal: options.signal, allowStreamingFallback: true diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index d6d797c12..4574f31a6 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -146,6 +146,27 @@ describe('ConvexApiClient', () => { expect(result.tables.map((table) => table.tableName)).toEqual(['lists', 'todos']); }); + it('sends table_name as snake_case query parameter in list_snapshot', async () => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ + snapshot: '100', + cursor: null, + has_more: false, + values: [] + }), + { status: 200 } + ) + ); + + const client = new ConvexApiClient(baseConfig); + await client.listSnapshot({ tableName: 'lists', snapshot: '100' }); + + const url = String(fetchSpy.mock.calls[0]![0]); + expect(url).toContain('table_name=lists'); + expect(url).not.toContain('tableName=lists'); + }); + it('marks network failures as retryable', async () => { vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('fetch failed: ECONNRESET')); From ec9e0467288bf745aa8fc3828a240fa6346e4875 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Mon, 16 Feb 2026 22:57:48 -0700 Subject: [PATCH 10/86] slow tests --- modules/module-convex/package.json | 3 +- modules/module-convex/test/src/env.ts | 9 + .../module-convex/test/src/slow_tests.test.ts | 96 ++++++++++ modules/module-convex/test/src/util.ts | 179 ++++++++++++++++++ 4 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 modules/module-convex/test/src/env.ts create mode 100644 modules/module-convex/test/src/slow_tests.test.ts create mode 100644 modules/module-convex/test/src/util.ts diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json index d705bbb02..ba31d3708 100644 --- a/modules/module-convex/package.json +++ b/modules/module-convex/package.json @@ -36,6 +36,7 @@ "ts-codec": "^1.3.0" }, "devDependencies": { - "@powersync/service-core-tests": "workspace:*" + "@powersync/service-core-tests": "workspace:*", + "@powersync/service-module-mongodb-storage": "workspace:*" } } diff --git a/modules/module-convex/test/src/env.ts b/modules/module-convex/test/src/env.ts new file mode 100644 index 000000000..7451deb2d --- /dev/null +++ b/modules/module-convex/test/src/env.ts @@ -0,0 +1,9 @@ +import { utils } from '@powersync/lib-services-framework'; + +export const env = utils.collectEnvironmentVariables({ + CONVEX_URL: utils.type.string.default('http://127.0.0.1:3210'), + CONVEX_DEPLOY_KEY: utils.type.string.default(''), + MONGO_TEST_URL: utils.type.string.default('mongodb://127.0.0.1:27017/powersync_test?directConnection=true'), + CI: utils.type.boolean.default('false'), + SLOW_TESTS: utils.type.boolean.default('false') +}); diff --git a/modules/module-convex/test/src/slow_tests.test.ts b/modules/module-convex/test/src/slow_tests.test.ts new file mode 100644 index 000000000..5f563dbb8 --- /dev/null +++ b/modules/module-convex/test/src/slow_tests.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, test } from 'vitest'; + +import { ConvexStreamTestContext, INITIALIZED_MONGO_STORAGE_FACTORY, makeConvexConnectionManager } from './util.js'; +import { env } from './env.js'; + +describe.runIf(env.SLOW_TESTS && !!env.CONVEX_DEPLOY_KEY)( + 'convex slow tests', + { timeout: 120_000 }, + function () { + test('connects to Convex and lists table schemas', async () => { + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); + const schemas = await context.client.getJsonSchemas(); + + expect(schemas.tables.length).toBeGreaterThan(0); + + const tableNames = schemas.tables.map((t) => t.tableName); + expect(tableNames).toContain('lists'); + }); + + test('list_snapshot returns data from Convex', async () => { + const mgr = makeConvexConnectionManager(); + const page = await mgr.client.listSnapshot({ tableName: 'lists' }); + + expect(page.snapshot).toBeDefined(); + // If this fails, seed data first: + // curl -X POST http://127.0.0.1:3210/api/mutation \ + // -H 'Content-Type: application/json' \ + // -H 'Authorization: Convex ' \ + // -d '{"path":"seed:seedLists","args":{"count":10},"format":"json"}' + expect(page.values.length).toBeGreaterThan(0); + + await mgr.end(); + }); + + test('snapshot replicates existing data into buckets', async () => { + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); + + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT _id as id, name FROM "lists" + `); + + await context.replicateSnapshot(); + await context.markSnapshotQueryable(); + + const checksum = await context.getChecksum('global[]'); + expect(checksum).toBeDefined(); + expect(checksum!.count).toBeGreaterThan(0); + }); + + test('snapshot replicates with wildcard table pattern', async () => { + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT _id as id, * FROM "lists" + `); + + await context.replicateSnapshot(); + await context.markSnapshotQueryable(); + + const data = await context.getBucketData('global[]'); + expect(data.length).toBeGreaterThan(0); + }); + + test('delta streaming starts from snapshot cursor', async () => { + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT _id as id, name FROM "lists" + `); + + await context.replicateSnapshot(); + + // Start streaming in background; it should poll deltas without error. + const streamPromise = context.startStreaming(); + + // Give delta polling a couple of cycles to confirm it doesn't crash. + await new Promise((resolve) => setTimeout(resolve, 2_000)); + + // Abort streaming gracefully. + context.abort(); + await streamPromise.catch(() => {}); + + // Checkpoint should still be valid after streaming ran. + const checksum = await context.getChecksum('global[]'); + expect(checksum).toBeDefined(); + expect(checksum!.count).toBeGreaterThan(0); + }); + } +); diff --git a/modules/module-convex/test/src/util.ts b/modules/module-convex/test/src/util.ts new file mode 100644 index 000000000..9979144b5 --- /dev/null +++ b/modules/module-convex/test/src/util.ts @@ -0,0 +1,179 @@ +import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import { + BucketStorageFactory, + createCoreReplicationMetrics, + initializeCoreReplicationMetrics, + InternalOpId, + OplogEntry, + storage, + SyncRulesBucketStorage +} from '@powersync/service-core'; +import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; + +import { ConvexConnectionManager } from '@module/replication/ConvexConnectionManager.js'; +import { ConvexStream, ConvexStreamOptions } from '@module/replication/ConvexStream.js'; +import { normalizeConnectionConfig } from '@module/types/types.js'; +import { ConvexLSN } from '@module/common/ConvexLSN.js'; +import { env } from './env.js'; + +export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ + url: env.MONGO_TEST_URL, + isCI: env.CI +}); + +export function makeConvexConnectionManager() { + if (!env.CONVEX_DEPLOY_KEY) { + throw new Error('CONVEX_DEPLOY_KEY is required for slow tests'); + } + + const config = normalizeConnectionConfig({ + type: 'convex', + deployment_url: env.CONVEX_URL, + deploy_key: env.CONVEX_DEPLOY_KEY, + polling_interval_ms: 200, + request_timeout_ms: 15_000 + }); + + return new ConvexConnectionManager(config); +} + +export class ConvexStreamTestContext { + private _stream?: ConvexStream; + private abortController = new AbortController(); + private streamPromise?: Promise; + public storage?: SyncRulesBucketStorage; + + static async open(factory: storage.TestStorageFactory) { + const f = await factory({}); + const connectionManager = makeConvexConnectionManager(); + return new ConvexStreamTestContext(f, connectionManager); + } + + constructor( + public factory: BucketStorageFactory, + public connectionManager: ConvexConnectionManager + ) { + createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + } + + abort() { + this.abortController.abort(); + } + + async dispose() { + this.abort(); + await this.streamPromise?.catch(() => {}); + await this.factory[Symbol.asyncDispose](); + await this.connectionManager.end(); + } + + async [Symbol.asyncDispose]() { + await this.dispose(); + } + + get client() { + return this.connectionManager.client; + } + + async updateSyncRules(content: string) { + const syncRules = await this.factory.updateSyncRules({ content, validate: true }); + this.storage = this.factory.getInstance(syncRules); + return this.storage!; + } + + get stream(): ConvexStream { + if (this.storage == null) { + throw new Error('Call updateSyncRules() first'); + } + if (this._stream) { + return this._stream; + } + + const options: ConvexStreamOptions = { + storage: this.storage, + metrics: METRICS_HELPER.metricsEngine, + connections: this.connectionManager, + abortSignal: this.abortController.signal + }; + this._stream = new ConvexStream(options); + return this._stream; + } + + async replicateSnapshot() { + await this.stream.initReplication(); + } + + startStreaming() { + this.streamPromise = this.stream.streamChanges(); + return this.streamPromise; + } + + async waitForCheckpoint(options?: { timeout?: number }): Promise { + const timeout = options?.timeout ?? 30_000; + const start = Date.now(); + + while (Date.now() - start < timeout) { + const activeStorage = await this.factory.getActiveStorage(); + const cp = await activeStorage?.getCheckpoint(); + if (cp != null && cp.lsn) { + return cp.checkpoint; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + throw new Error(`Timeout waiting for checkpoint after ${timeout}ms`); + } + + async getBucketData(bucket: string, options?: { timeout?: number }): Promise { + const checkpoint = await this.waitForCheckpoint(options); + const map = new Map([[bucket, 0n]]); + let data: OplogEntry[] = []; + + while (true) { + const batch = this.storage!.getBucketDataBatch(checkpoint, map); + const batches = await test_utils.fromAsync(batch); + data = data.concat(batches[0]?.chunkData.data ?? []); + if (batches.length == 0 || !batches[0]!.chunkData.has_more) { + break; + } + map.set(bucket, BigInt(batches[0]!.chunkData.next_after)); + } + + return data; + } + + async getChecksums(buckets: string[], options?: { timeout?: number }) { + const checkpoint = await this.waitForCheckpoint(options); + return this.storage!.getChecksums(checkpoint, buckets); + } + + async getChecksum(bucket: string, options?: { timeout?: number }) { + const checkpoint = await this.waitForCheckpoint(options); + const map = await this.storage!.getChecksums(checkpoint, [bucket]); + return map.get(bucket); + } + + /** + * After snapshot, manually advance the checkpoint so bucket data is queryable + * without requiring full delta streaming to catch up. + */ + async markSnapshotQueryable() { + const status = await this.storage!.getStatus(); + if (!status.checkpoint_lsn) { + throw new Error('No checkpoint LSN available - run snapshot first'); + } + + await this.storage!.startBatch( + { + defaultSchema: 'convex', + zeroLSN: ConvexLSN.ZERO.comparable, + storeCurrentData: false, + skipExistingRows: false + }, + async (batch) => { + await batch.keepalive(status.checkpoint_lsn!); + } + ); + } +} From 950e3d760cf2d14b6f8d02bec6807f2f5224ec41 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Tue, 17 Feb 2026 21:01:15 -0700 Subject: [PATCH 11/86] fix test --- modules/module-convex/test/src/util.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/module-convex/test/src/util.ts b/modules/module-convex/test/src/util.ts index 9979144b5..04c690616 100644 --- a/modules/module-convex/test/src/util.ts +++ b/modules/module-convex/test/src/util.ts @@ -26,13 +26,15 @@ export function makeConvexConnectionManager() { throw new Error('CONVEX_DEPLOY_KEY is required for slow tests'); } - const config = normalizeConnectionConfig({ - type: 'convex', + const rawConfig = { + type: 'convex' as const, deployment_url: env.CONVEX_URL, deploy_key: env.CONVEX_DEPLOY_KEY, polling_interval_ms: 200, request_timeout_ms: 15_000 - }); + }; + + const config = { ...rawConfig, ...normalizeConnectionConfig(rawConfig) }; return new ConvexConnectionManager(config); } From 1c8dd1f8609631258d3a23342487e3d712e9cd96 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Thu, 19 Feb 2026 14:38:26 -0700 Subject: [PATCH 12/86] remove streaming import cruft --- modules/module-convex/AGENTS.md | 1 - modules/module-convex/README.md | 3 +- .../src/common/ConvexCheckpoints.ts | 30 ++----------------- .../test/src/ConvexCheckpoints.test.ts | 5 +--- .../test/src/ConvexStream.test.ts | 2 +- 5 files changed, 6 insertions(+), 35 deletions(-) diff --git a/modules/module-convex/AGENTS.md b/modules/module-convex/AGENTS.md index f0d2ce1fe..8a655b855 100644 --- a/modules/module-convex/AGENTS.md +++ b/modules/module-convex/AGENTS.md @@ -76,7 +76,6 @@ This file is the working contract for agents modifying `module-convex`. 3. then pass the head to callback. - Source marker table: `powersync_checkpoints` - Convex rejects table names starting with `_`, so no leading-underscore variant is used. - - Convex streaming import materializes ingested tables with a `source_` prefix, so the marker may appear as `source_powersync_checkpoints` in streaming export. - Stream handling requirement: - checkpoint marker tables must always be excluded from replicated source tables and ignored in delta row application. - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 235c7ea90..b39f4e198 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -32,8 +32,7 @@ replication: ## Write checkpoint behavior -- Route write checkpoints create a source marker row using Convex streaming import in `powersync_checkpoints`. -- Convex may materialize system/invalid source table names with a `source_` prefix (for example `source_powersync_checkpoints`). +- Route write checkpoints create a source marker row in `powersync_checkpoints`. - Delta polling is global, so marker rows are observed even when not referenced in sync rules. - Checkpoint marker tables are always ignored during snapshot and delta replication. - Marker-only pages trigger immediate keepalive checkpoint advancement (not minute-throttled), reducing write-checkpoint acknowledgement latency. diff --git a/modules/module-convex/src/common/ConvexCheckpoints.ts b/modules/module-convex/src/common/ConvexCheckpoints.ts index 5e86aa7bb..d797bceca 100644 --- a/modules/module-convex/src/common/ConvexCheckpoints.ts +++ b/modules/module-convex/src/common/ConvexCheckpoints.ts @@ -1,37 +1,13 @@ /** - * Table name used by PowerSync to write checkpoint markers into Convex via streaming import. + * Table name used by PowerSync to write checkpoint markers into Convex. * * Convex rejects table names starting with `_`, so we use a plain prefix. - * Convex streaming import may materialize this table with a `source_` prefix - * (e.g. `source_powersync_checkpoints`), so {@link isConvexCheckpointTable} - * strips that prefix when matching. */ export const CONVEX_CHECKPOINT_TABLE = 'powersync_checkpoints'; /** - * Prefix that Convex streaming import prepends to ingested table names. - */ -export const CONVEX_SOURCE_TABLE_PREFIX = 'source_'; - -/** - * Returns true if `tableName` is the PowerSync checkpoint marker table, - * accounting for zero or more `source_` prefixes added by Convex. + * Returns true if `tableName` is the PowerSync checkpoint marker table. */ export function isConvexCheckpointTable(tableName: string): boolean { - let candidate = tableName; - - // Strip up to 4 layers of source_ prefix (defensive; typically 0-1). - for (let depth = 0; depth < 4; depth += 1) { - if (candidate === CONVEX_CHECKPOINT_TABLE) { - return true; - } - - if (!candidate.startsWith(CONVEX_SOURCE_TABLE_PREFIX)) { - return false; - } - - candidate = candidate.slice(CONVEX_SOURCE_TABLE_PREFIX.length); - } - - return false; + return tableName === CONVEX_CHECKPOINT_TABLE; } diff --git a/modules/module-convex/test/src/ConvexCheckpoints.test.ts b/modules/module-convex/test/src/ConvexCheckpoints.test.ts index 4249533e7..fa07f7a5a 100644 --- a/modules/module-convex/test/src/ConvexCheckpoints.test.ts +++ b/modules/module-convex/test/src/ConvexCheckpoints.test.ts @@ -2,10 +2,8 @@ import { describe, expect, it } from 'vitest'; import { isConvexCheckpointTable } from '@module/common/ConvexCheckpoints.js'; describe('ConvexCheckpoints', () => { - it('recognizes checkpoint table names including source-prefixed variants', () => { + it('recognizes the checkpoint table name', () => { expect(isConvexCheckpointTable('powersync_checkpoints')).toBe(true); - expect(isConvexCheckpointTable('source_powersync_checkpoints')).toBe(true); - expect(isConvexCheckpointTable('source_source_powersync_checkpoints')).toBe(true); }); it('does not match underscore-prefixed variant', () => { @@ -14,7 +12,6 @@ describe('ConvexCheckpoints', () => { it('does not match non-checkpoint tables', () => { expect(isConvexCheckpointTable('lists')).toBe(false); - expect(isConvexCheckpointTable('source_lists')).toBe(false); expect(isConvexCheckpointTable('powersync_other')).toBe(false); }); }); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index e6a7009e8..136e66b70 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -379,7 +379,7 @@ describe('ConvexStream', () => { return { cursor: '102', hasMore: false, - values: [{ _table: 'source_powersync_checkpoints', _id: 'cp1' }] + values: [{ _table: 'powersync_checkpoints', _id: 'cp1' }] }; } } From 817e6b26bf50d960c1113aa36548abc358cc7b7e Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Thu, 19 Feb 2026 15:59:26 -0700 Subject: [PATCH 13/86] simplify LSN representation it's always a timestamp --- modules/module-convex/AGENTS.md | 9 +++--- modules/module-convex/src/common/ConvexLSN.ts | 28 ++++++------------- .../module-convex/test/src/ConvexLSN.test.ts | 11 +++----- 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/modules/module-convex/AGENTS.md b/modules/module-convex/AGENTS.md index 8a655b855..1793fc700 100644 --- a/modules/module-convex/AGENTS.md +++ b/modules/module-convex/AGENTS.md @@ -30,11 +30,10 @@ This file is the working contract for agents modifying `module-convex`. - The overall system must ensure causal consistency of replicated data in bucket storage. ## 4) LSN and Cursor Rules -- Never assume Convex cursors are numeric-only. -- Supported cursor shapes include: - - decimal numeric timestamp strings, - - opaque strings. -- Persist LSNs using `ConvexLSN` comparable format (`mode + sortable key + encoded cursor`). +- Convex snapshot and delta cursors are always `i64` timestamps (serialized as decimal numeric strings in JSON). +- `ConvexLSN.fromCursor` rejects non-numeric cursors. +- The `list_snapshot` pagination cursor is a separate JSON-serialized `{tablet, id}` string — it is pagination state, not a replication cursor. +- Persist LSNs using `ConvexLSN` comparable format (`n` + zero-padded timestamp + `|` + base64url-encoded cursor). - Keep raw cursor round-trip safe. ## 5) API Client Contract diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index 8cc4db362..912bbd351 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -1,12 +1,9 @@ const CURSOR_WIDTH = 20; const DELIMITER = '|'; -type CursorMode = 'numeric' | 'opaque'; - export interface ConvexLSNSpecification { cursor: string; - mode: CursorMode; - sortKey: bigint; + timestamp: bigint; } export class ConvexLSN { @@ -14,18 +11,16 @@ export class ConvexLSN { static fromSerialized(comparable: string): ConvexLSN { if (!comparable.includes(DELIMITER)) { - // Bare cursor string (no encoded format). Treat as a raw cursor. return ConvexLSN.fromCursor(comparable); } const [first, second] = comparable.split(DELIMITER, 2); - const prefixed = first.match(/^([no])([0-9]+)$/); + const prefixed = first.match(/^[no]([0-9]+)$/); if (prefixed && second != null) { const decoded = decodeCursor(second); return ConvexLSN.fromCursor(decoded); } - // Unrecognized format: treat entire input as an opaque cursor. return ConvexLSN.fromCursor(comparable); } @@ -35,20 +30,17 @@ export class ConvexLSN { throw new Error('Convex cursor cannot be empty'); } - const numericSortKey = getNumericSortKey(asString); - const mode: CursorMode = numericSortKey == null ? 'opaque' : 'numeric'; - + const ts = parseTimestamp(asString); return new ConvexLSN({ cursor: asString, - mode, - sortKey: numericSortKey ?? 0n + timestamp: ts }); } constructor(private readonly options: ConvexLSNSpecification) {} get timestamp() { - return this.options.sortKey; + return this.options.timestamp; } get cursor() { @@ -56,9 +48,8 @@ export class ConvexLSN { } get comparable() { - const prefix = this.options.mode == 'numeric' ? 'n' : 'o'; - const sortPart = this.options.sortKey.toString().padStart(CURSOR_WIDTH, '0'); - return `${prefix}${sortPart}${DELIMITER}${encodeCursor(this.options.cursor)}`; + const sortPart = this.options.timestamp.toString().padStart(CURSOR_WIDTH, '0'); + return `n${sortPart}${DELIMITER}${encodeCursor(this.options.cursor)}`; } toString() { @@ -70,12 +61,11 @@ export class ConvexLSN { } } -function getNumericSortKey(cursor: string): bigint | null { +function parseTimestamp(cursor: string): bigint { if (/^[0-9]+$/.test(cursor)) { return BigInt(cursor); } - - return null; + throw new Error(`Convex cursor is not a valid numeric timestamp: ${cursor}`); } function encodeCursor(value: string): string { diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts index 94922a8a2..25acea5a0 100644 --- a/modules/module-convex/test/src/ConvexLSN.test.ts +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -23,12 +23,9 @@ describe('ConvexLSN', () => { expect(parsed.toCursorString()).toBe('42'); }); - it('round-trips opaque (non-numeric) cursors', () => { - const opaque = '{"tablet":"abc","id":"xyz"}'; - const source = ConvexLSN.fromCursor(opaque); - const roundTrip = ConvexLSN.fromSerialized(source.comparable); - - expect(roundTrip.toCursorString()).toBe(opaque); - expect(roundTrip.timestamp).toBe(0n); + it('rejects non-numeric cursors', () => { + expect(() => ConvexLSN.fromCursor('{"tablet":"abc","id":"xyz"}')).toThrow( + 'Convex cursor is not a valid numeric timestamp' + ); }); }); From c688653babdf5a3c28fa79ecd1422ef22c1741d0 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Thu, 19 Feb 2026 15:59:29 -0700 Subject: [PATCH 14/86] comments --- modules/module-convex/src/api/ConvexRouteAPIAdapter.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index 52f5b2c5c..d4f65ca6c 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -73,6 +73,7 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { const matchedTableNames = [...tablesByName.keys()] .filter((name) => { + //Convex doesn't support user-defined schemas, so this is more a forwards compatibility check for when multiple connections are supported if (tablePattern.schema != this.connectionManager.schema) { return false; } @@ -113,6 +114,7 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { return result; } + //for convex we can calculate time-based lag, but not byte-based lag async getReplicationLagBytes(options: ReplicationLagOptions): Promise { return undefined; } From 6d4e11aac2d5e185b1bc3ca2de907a09dd63b5ce Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 20 Feb 2026 11:28:36 -0700 Subject: [PATCH 15/86] use BaseObserver --- .../src/replication/ConvexConnectionManager.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/modules/module-convex/src/replication/ConvexConnectionManager.ts b/modules/module-convex/src/replication/ConvexConnectionManager.ts index bbd7bef0e..2749e63ad 100644 --- a/modules/module-convex/src/replication/ConvexConnectionManager.ts +++ b/modules/module-convex/src/replication/ConvexConnectionManager.ts @@ -1,3 +1,4 @@ +import { BaseObserver } from '@powersync/lib-services-framework'; import { DEFAULT_TAG } from '@powersync/service-sync-rules'; import { ConvexApiClient } from '../client/ConvexApiClient.js'; import { ResolvedConvexConnectionConfig } from '../types/types.js'; @@ -6,28 +7,22 @@ export interface ConvexConnectionManagerListener { onEnded?: () => void; } -export class ConvexConnectionManager { +export class ConvexConnectionManager extends BaseObserver { readonly client: ConvexApiClient; readonly schema = 'convex'; readonly connectionTag: string; readonly connectionId: string; - private listeners = new Set(); - constructor(public readonly config: ResolvedConvexConnectionConfig) { + super(); this.client = new ConvexApiClient(config); this.connectionTag = config.tag ?? DEFAULT_TAG; this.connectionId = config.id ?? 'default'; } - registerListener(listener: ConvexConnectionManagerListener) { - this.listeners.add(listener); - } - async end(): Promise { - for (const listener of [...this.listeners]) { + this.iterateListeners((listener) => { listener.onEnded?.(); - } - this.listeners.clear(); + }); } } From ceb394195cb69d4597c362b20baf3f943999c02b Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 20 Feb 2026 12:18:47 -0700 Subject: [PATCH 16/86] use regular convex mutations for powersync_checkpoints --- modules/module-convex/AGENTS.md | 4 +- modules/module-convex/README.md | 70 ++++++++++++++++++- .../src/api/ConvexRouteAPIAdapter.ts | 2 +- .../src/client/ConvexApiClient.ts | 62 ++-------------- .../test/src/ConvexApiClient.test.ts | 21 +++--- .../test/src/ConvexRouteAPIAdapter.test.ts | 2 +- 6 files changed, 92 insertions(+), 69 deletions(-) diff --git a/modules/module-convex/AGENTS.md b/modules/module-convex/AGENTS.md index 1793fc700..e91de5c1c 100644 --- a/modules/module-convex/AGENTS.md +++ b/modules/module-convex/AGENTS.md @@ -71,10 +71,12 @@ This file is the working contract for agents modifying `module-convex`. ## 9) Checkpointing and Consistency - `createReplicationHead` must: 1. resolve global head cursor, - 2. write a Convex source marker via streaming import, + 2. write a Convex checkpoint marker via `POST /api/mutation` (calls `powersync_checkpoints:createCheckpoint`), 3. then pass the head to callback. - Source marker table: `powersync_checkpoints` - Convex rejects table names starting with `_`, so no leading-underscore variant is used. + - The table has a single `last_updated` field; the mutation upserts one row (bounded to one row total). + - The developer must deploy the `powersync_checkpoints` schema and mutation to their Convex project (see README). - Stream handling requirement: - checkpoint marker tables must always be excluded from replicated source tables and ignored in delta row application. - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index b39f4e198..a2a6b75a4 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -30,9 +30,77 @@ replication: - Each table snapshot starts from the first page (cursor omitted on the first call), and only uses cursor pagination within the current run. - If initial replication is interrupted, restart resumes from the stored snapshot boundary but restarts table paging from page 1. +## Convex Source Setup + +PowerSync requires a `powersync_checkpoints` table and a mutation function deployed to your Convex project. This is analogous to setting up a replication publication on Postgres — it enables PowerSync to write checkpoint markers that ensure causal consistency. + +### 1. Add the table to your schema + +In your Convex project's `convex/schema.ts`, add the `powersync_checkpoints` table: + +```typescript +import { defineSchema, defineTable } from "convex/server"; +import { v } from "convex/values"; + +export default defineSchema({ + // ... your existing tables ... + + powersync_checkpoints: defineTable({ + last_updated: v.float64(), + }), +}); +``` + +### 2. Add the checkpoint mutation + +Create `convex/powersync_checkpoints.ts`: + +```typescript +import { mutation } from "./_generated/server"; + +export const createCheckpoint = mutation({ + args: {}, + handler: async (ctx) => { + const existing = await ctx.db.query("powersync_checkpoints").first(); + + if (existing) { + await ctx.db.patch(existing._id, { last_updated: Date.now() }); + } else { + await ctx.db.insert("powersync_checkpoints", { last_updated: Date.now() }); + } + }, +}); +``` + +### 3. Deploy + +```bash +npx convex dev --once +``` + +### How it works + +PowerSync calls this mutation via the [Convex HTTP API](https://docs.convex.dev/http-api): + +``` +POST /api/mutation +Authorization: Convex +Content-Type: application/json + +{ + "path": "powersync_checkpoints:createCheckpoint", + "args": {}, + "format": "json" +} +``` + +The mutation upserts a single row (insert on first call, patch thereafter), so the table stays bounded to one row. This write generates a delta event that PowerSync observes to confirm write-checkpoint consistency. + +> **Note:** The `powersync_checkpoints` table is automatically excluded from sync rules — it is never replicated to client devices. + ## Write checkpoint behavior -- Route write checkpoints create a source marker row in `powersync_checkpoints`. +- Route write checkpoints create a source marker row in `powersync_checkpoints` via the deployed mutation. - Delta polling is global, so marker rows are observed even when not referenced in sync rules. - Checkpoint marker tables are always ignored during snapshot and delta replication. - Marker-only pages trigger immediate keepalive checkpoint advancement (not minute-throttled), reducing write-checkpoint acknowledgement latency. diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index d4f65ca6c..87608b574 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -121,7 +121,7 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { async createReplicationHead(callback: ReplicationHeadCallback): Promise { const head = await this.connectionManager.client.getHeadCursor(); - await this.connectionManager.client.createWriteCheckpointMarker({ headCursor: head }); + await this.connectionManager.client.createWriteCheckpointMarker(); return await callback(ConvexLSN.fromCursor(head).comparable); } diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index e22cbc8eb..9555208d5 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -45,15 +45,6 @@ export interface ConvexDocumentDeltasResult { values: ConvexRawDocument[]; } -export interface ConvexImportTable { - jsonSchema: Record; -} - -export interface ConvexImportMessage { - tableName: string; - data: Record; -} - export class ConvexApiError extends Error { readonly status?: number; readonly retryable: boolean; @@ -143,50 +134,23 @@ export class ConvexApiClient { return this.getGlobalSnapshotCursor({ signal: options?.signal }); } - async importAirbyteRecords(options: { - tables: Record; - messages: ConvexImportMessage[]; - signal?: AbortSignal; - }): Promise { + async createWriteCheckpointMarker(options?: { signal?: AbortSignal }): Promise { await this.performRequest({ method: 'POST', - path: '/api/streaming_import/import_airbyte_records', + path: '/api/mutation', body: { - tables: options.tables, - messages: options.messages + path: `${CONVEX_CHECKPOINT_TABLE}:createCheckpoint`, + args: {}, + format: 'json' }, - signal: options.signal, + signal: options?.signal, extraHeaders: { - 'Content-Type': 'application/json', - 'Convex-Client': 'streaming-import-1.0.0' + 'Content-Type': 'application/json' }, includeJsonFormat: false }); } - async createWriteCheckpointMarker(options?: { headCursor?: string; signal?: AbortSignal }): Promise { - const marker = { - powersync_lsn: options?.headCursor ?? (await this.getHeadCursor({ signal: options?.signal })), - powersync_created_at: Date.now(), - powersync_source: 'powersync' - }; - - await this.importAirbyteRecords({ - tables: { - [CONVEX_CHECKPOINT_TABLE]: { - jsonSchema: POWERSYNC_CHECKPOINT_SCHEMA - } - }, - messages: [ - { - tableName: CONVEX_CHECKPOINT_TABLE, - data: marker - } - ], - signal: options?.signal - }); - } - private async requestJson(options: { endpoint: string; params?: Record; @@ -488,18 +452,6 @@ const RESERVED_SCHEMA_KEYS = new Set([ 'errors' ]); -const POWERSYNC_CHECKPOINT_SCHEMA = { - type: 'object', - properties: { - powersync_lsn: { type: 'string' }, - powersync_created_at: { type: 'number' }, - powersync_source: { type: 'string' } - }, - additionalProperties: false, - required: ['powersync_lsn', 'powersync_created_at', 'powersync_source'], - $schema: 'http://json-schema.org/draft-07/schema#' -} as const; - function looksLikeJsonSchema(value: unknown): boolean { if (!isRecord(value)) { return false; diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index 4574f31a6..098e10d38 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { ConvexApiClient } from '@module/client/ConvexApiClient.js'; +import { CONVEX_CHECKPOINT_TABLE } from '@module/common/ConvexCheckpoints.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; const baseConfig = normalizeConnectionConfig({ @@ -177,24 +178,24 @@ describe('ConvexApiClient', () => { }); }); - it('creates write checkpoint markers via streaming import', async () => { - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('', { status: 200 })); + it('creates write checkpoint markers via mutation', async () => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(JSON.stringify({ status: 'success' }), { status: 200 }) + ); const client = new ConvexApiClient(baseConfig); - await client.createWriteCheckpointMarker({ headCursor: '123' }); + await client.createWriteCheckpointMarker(); expect(fetchSpy).toHaveBeenCalledTimes(1); const [url, init] = fetchSpy.mock.calls[0]!; - expect(String(url)).toContain('/api/streaming_import/import_airbyte_records'); + expect(String(url)).toContain('/api/mutation'); expect(init?.method).toBe('POST'); expect((init?.headers as Record).Authorization).toBe('Convex test-key'); - expect((init?.headers as Record)['Convex-Client']).toBe('streaming-import-1.0.0'); const body = JSON.parse(String(init?.body)); - expect(body.messages).toHaveLength(1); - expect(body.messages[0].tableName).toBe('powersync_checkpoints'); - expect(body.messages[0].data.powersync_lsn).toBe('123'); - expect(body.tables).toHaveProperty('powersync_checkpoints'); + expect(body.path).toBe(`${CONVEX_CHECKPOINT_TABLE}:createCheckpoint`); + expect(body.args).toEqual({}); + expect(body.format).toBe('json'); }); it('propagates checkpoint write errors directly (no fallback)', async () => { @@ -206,7 +207,7 @@ describe('ConvexApiClient', () => { ); const client = new ConvexApiClient(baseConfig); - await expect(client.createWriteCheckpointMarker({ headCursor: '123' })).rejects.toMatchObject({ + await expect(client.createWriteCheckpointMarker()).rejects.toMatchObject({ status: 400, retryable: false }); diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index 45c4af584..30b83c130 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -84,7 +84,7 @@ bucket_definitions: expect(getHeadCursor).toHaveBeenCalledTimes(1); expect(getHeadCursor).toHaveBeenCalledWith(); expect(createWriteCheckpointMarker).toHaveBeenCalledTimes(1); - expect(createWriteCheckpointMarker).toHaveBeenCalledWith({ headCursor: '123' }); + expect(createWriteCheckpointMarker).toHaveBeenCalledWith(); await adapter.shutdown(); }); From d7e97ef3792c57f1b1f1b90ed71d61298ba55ecd Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 20 Feb 2026 12:36:06 -0700 Subject: [PATCH 17/86] cleanup --- modules/module-convex/src/common/ConvexLSN.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index 912bbd351..a3c6012f7 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -15,7 +15,7 @@ export class ConvexLSN { } const [first, second] = comparable.split(DELIMITER, 2); - const prefixed = first.match(/^[no]([0-9]+)$/); + const prefixed = first.match(/^n([0-9]+)$/); if (prefixed && second != null) { const decoded = decodeCursor(second); return ConvexLSN.fromCursor(decoded); From 0c7c370049501cfe668ebe6398241f4242787e53 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Mar 2026 10:17:55 -0700 Subject: [PATCH 18/86] remove agents.md entirelyu --- modules/module-convex/AGENTS.md | 126 ------------------------------- modules/module-convex/README.md | 128 ++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 126 deletions(-) delete mode 100644 modules/module-convex/AGENTS.md diff --git a/modules/module-convex/AGENTS.md b/modules/module-convex/AGENTS.md deleted file mode 100644 index e91de5c1c..000000000 --- a/modules/module-convex/AGENTS.md +++ /dev/null @@ -1,126 +0,0 @@ -# Convex Module Agent Guide - -This file is the working contract for agents modifying `module-convex`. - -## 1) Scope -- This module replicates Convex data into PowerSync bucket storage. -- Source API is Convex Streaming Export (`json_schemas`, `list_snapshot`, `document_deltas`). -- Initial scope is default Convex component only. - -## 2) Canonical Behavior -- Initial replication: - 1. Pin one **global snapshot** via `list_snapshot` **without** `tableName` and `cursor`. - 2. Snapshot each selected sync-rules table with that fixed `snapshot`. - 3. First per-table snapshot call omits `cursor`; pagination cursor is only for later pages in the same run. - 4. Commit snapshot LSN, then switch to deltas. -- Streaming replication: - - Start from persisted resume LSN. - - Poll `document_deltas`. - - Always stream globally (no `tableName` filter), then filter locally by selected sync-rules tables. - -## 3) Hard Invariants (Do Not Break) -- `snapshot` is the consistency boundary; page `cursor` is pagination state. -- All table snapshots in a run must use the same pinned `snapshot`; if response snapshot differs, fail fast. -- On restart during initial replication: - - Reuse persisted snapshot LSN boundary. - - Restart table page walk from first page (do not resume per-table `lastKey`). -- Delta streaming starts from resume LSN (snapshot boundary), not from table page cursor. -- `tablePattern.connectionTag` and schema must match before table selection. -- Source table replica identity is `_id`. -- The overall system must ensure causal consistency of replicated data in bucket storage. - -## 4) LSN and Cursor Rules -- Convex snapshot and delta cursors are always `i64` timestamps (serialized as decimal numeric strings in JSON). -- `ConvexLSN.fromCursor` rejects non-numeric cursors. -- The `list_snapshot` pagination cursor is a separate JSON-serialized `{tablet, id}` string — it is pagination state, not a replication cursor. -- Persist LSNs using `ConvexLSN` comparable format (`n` + zero-padded timestamp + `|` + base64url-encoded cursor). -- Keep raw cursor round-trip safe. - -## 5) API Client Contract -- Auth header: `Authorization: Convex `. -- Always request `format=json`. -- Fallback path support: `/api/streaming_export/...` when `/api/...` returns `404`. -- Parse large numeric JSON using `JSONBig`. -- `json_schemas` must support: - - array/object under `tables`, - - self-host top-level table map shape. -- Retry classification: - - retryable: network, timeout, 429, 5xx. - - non-retryable: malformed responses, auth/config issues. - -## 6) Sync Rules and Connection Semantics -- Default schema is `convex`. -- SQL literal reminder for sync rules: string literals must use single quotes (`'sally'`), not double quotes. - -## 7) Schema Change Caveat -- Convex `json_schemas` does not provide a schema change token or revision cursor that can be checkpointed. -- Current behavior uses `json_schemas` for discovery/debug, but does not continuously diff source schema versions. -- Operational caveat: if Convex schema changes (tables or columns), developers must review and redeploy sync rules manually. -- Future improvement: cache a canonicalized `json_schemas` hash, poll periodically, and raise diagnostics when schema drift is detected. - -## 8) Datatype Mapping -- Current runtime mapping in stream writer: - - `number` integer -> `BigInt` (SQLite INTEGER), - - `number` float -> REAL, - - `boolean` -> `1n`/`0n`, - - `Uint8Array` -> BLOB, - - `object/array` -> JSON TEXT. - NOTE: - - Preserve bytes as BLOB in sync rules; use explicit base64 conversion in rules only when needed by consumers. - -## 9) Checkpointing and Consistency -- `createReplicationHead` must: - 1. resolve global head cursor, - 2. write a Convex checkpoint marker via `POST /api/mutation` (calls `powersync_checkpoints:createCheckpoint`), - 3. then pass the head to callback. -- Source marker table: `powersync_checkpoints` - - Convex rejects table names starting with `_`, so no leading-underscore variant is used. - - The table has a single `last_updated` field; the mutation upserts one row (bounded to one row total). - - The developer must deploy the `powersync_checkpoints` schema and mutation to their Convex project (see README). -- Stream handling requirement: - - checkpoint marker tables must always be excluded from replicated source tables and ignored in delta row application. - - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). - -## 10) Logging Policy -- Keep logs high-signal and bounded. -- Required snapshot logs: - - pinned or reused global snapshot, - - table snapshot start, - - snapshot completion and resume LSN. -- Avoid noisy per-row debug logs unless behind explicit debug gating. - -## 11) Known Non-Goals (For Now) -- Convex component targeting beyond default component. -- A true push stream transport (module is polling deltas). -- Read-only Convex deploy key model (Convex deploy key is full access; use network isolation and secret controls). -- Cross-source transactional guarantees beyond Convex stream ordering. - -## 12) Conformance Suite (Must Stay Green) -- Unit tests: - - `test/src/ConvexApiClient.test.ts` - - `test/src/ConvexCheckpoints.test.ts` - - `test/src/ConvexLSN.test.ts` - - `test/src/ConvexRouteAPIAdapter.test.ts` - - `test/src/ConvexStream.test.ts` - - `test/src/types.test.ts` -- Required assertions include: - - global snapshot pin call is unfiltered, - - first per-table snapshot call omits cursor, - - pagination cursor used only for subsequent pages, - - snapshot mismatch fails fast, - - route adapter head resolves global snapshot cursor and writes a source marker, - - marker-only pages advance LSN via immediate keepalive. -- Manual smoke test (self-host acceptable): - - initial snapshot visible in buckets, - - inserts/updates/deletes propagate via deltas, - - no table-not-found due to schema/tag mismatch. - -## 13) Change Checklist for Agents -- If changing snapshot/delta flow: - - update `ConvexStream` tests first. -- If changing cursor/LSN encoding: - - update `ConvexLSN` tests and backward-compat coverage. -- If changing API response parsing: - - include self-host payload fixtures in `ConvexApiClient` tests. -- If changing public behavior: - - update `README.md` and this `AGENTS.md` in the same PR. diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index a2a6b75a4..3e530ecd2 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -104,3 +104,131 @@ The mutation upserts a single row (insert on first call, patch thereafter), so t - Delta polling is global, so marker rows are observed even when not referenced in sync rules. - Checkpoint marker tables are always ignored during snapshot and delta replication. - Marker-only pages trigger immediate keepalive checkpoint advancement (not minute-throttled), reducing write-checkpoint acknowledgement latency. + + +# Technical notes + +The content below is written in an agents.md style describing the behavior of `module-convex`. + +## 1) Scope +- This module replicates Convex data into PowerSync bucket storage. +- Source API is Convex Streaming Export (`json_schemas`, `list_snapshot`, `document_deltas`). +- Initial scope is default Convex component only. + +## 2) Canonical Behavior +- Initial replication: + 1. Pin one **global snapshot** via `list_snapshot` **without** `tableName` and `cursor`. + 2. Snapshot each selected sync-rules table with that fixed `snapshot`. + 3. First per-table snapshot call omits `cursor`; pagination cursor is only for later pages in the same run. + 4. Commit snapshot LSN, then switch to deltas. +- Streaming replication: + - Start from persisted resume LSN. + - Poll `document_deltas`. + - Always stream globally (no `tableName` filter), then filter locally by selected sync-rules tables. + +## 3) Hard Invariants (Do Not Break) +- `snapshot` is the consistency boundary; page `cursor` is pagination state. +- All table snapshots in a run must use the same pinned `snapshot`; if response snapshot differs, fail fast. +- On restart during initial replication: + - Reuse persisted snapshot LSN boundary. + - Restart table page walk from first page (do not resume per-table `lastKey`). +- Delta streaming starts from resume LSN (snapshot boundary), not from table page cursor. +- `tablePattern.connectionTag` and schema must match before table selection. +- Source table replica identity is `_id`. +- The overall system must ensure causal consistency of replicated data in bucket storage. + +## 4) LSN and Cursor Rules +- Convex snapshot and delta cursors are always `i64` timestamps (serialized as decimal numeric strings in JSON). +- `ConvexLSN.fromCursor` rejects non-numeric cursors. +- The `list_snapshot` pagination cursor is a separate JSON-serialized `{tablet, id}` string — it is pagination state, not a replication cursor. +- Persist LSNs as a zero-padded decimal cursor string so lexicographic ordering matches timestamp ordering. +- `ConvexLSN.fromSerialized` accepts bare numeric cursor strings and zero-padded stored values. + +## 5) API Client Contract +- Auth header: `Authorization: Convex `. +- Always request `format=json`. +- Fallback path support: `/api/streaming_export/...` when `/api/...` returns `404`. +- Parse large numeric JSON using `JSONBig`. +- `json_schemas` must support: + - array/object under `tables`, + - self-host top-level table map shape. +- Retry classification: + - retryable: network, timeout, 429, 5xx. + - non-retryable: malformed responses, auth/config issues. + +## 6) Sync Rules and Connection Semantics +- Default schema is `convex`. +- SQL literal reminder for sync rules: string literals must use single quotes (`'sally'`), not double quotes. + +## 7) Schema Change Caveat +- Convex `json_schemas` does not provide a schema change token or revision cursor that can be checkpointed. +- Current behavior uses `json_schemas` for discovery/debug, but does not continuously diff source schema versions. +- Operational caveat: if Convex schema changes (tables or columns), developers must review and redeploy sync rules manually. +- Future improvement: cache a canonicalized `json_schemas` hash, poll periodically, and raise diagnostics when schema drift is detected. + +## 8) Datatype Mapping +- Current runtime mapping in stream writer: + - `number` integer -> `BigInt` (SQLite INTEGER), + - `number` float -> REAL, + - `boolean` -> `1n`/`0n`, + - `Uint8Array` -> BLOB, + - `object/array` -> JSON TEXT. + NOTE: + - Preserve bytes as BLOB in sync rules; use explicit base64 conversion in rules only when needed by consumers. + +## 9) Checkpointing and Consistency +- `createReplicationHead` must: + 1. resolve global head cursor, + 2. write a Convex checkpoint marker via `POST /api/mutation` (calls `powersync_checkpoints:createCheckpoint`), + 3. then pass the head to callback. +- Source marker table: `powersync_checkpoints` + - Convex rejects table names starting with `_`, so no leading-underscore variant is used. + - The table has a single `last_updated` field; the mutation upserts one row (bounded to one row total). + - The developer must deploy the `powersync_checkpoints` schema and mutation to their Convex project (see README). +- Stream handling requirement: + - checkpoint marker tables must always be excluded from replicated source tables and ignored in delta row application. + - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). + +## 10) Logging Policy +- Keep logs high-signal and bounded. +- Required snapshot logs: + - pinned or reused global snapshot, + - table snapshot start, + - snapshot completion and resume LSN. +- Avoid noisy per-row debug logs unless behind explicit debug gating. + +## 11) Known Non-Goals (For Now) +- Convex component targeting beyond default component. +- A true push stream transport (module is polling deltas). +- Read-only Convex deploy key model (Convex deploy key is full access; use network isolation and secret controls). +- Cross-source transactional guarantees beyond Convex stream ordering. + +## 12) Conformance Suite (Must Stay Green) +- Unit tests: + - `test/src/ConvexApiClient.test.ts` + - `test/src/ConvexCheckpoints.test.ts` + - `test/src/ConvexLSN.test.ts` + - `test/src/ConvexRouteAPIAdapter.test.ts` + - `test/src/ConvexStream.test.ts` + - `test/src/types.test.ts` +- Required assertions include: + - global snapshot pin call is unfiltered, + - first per-table snapshot call omits cursor, + - pagination cursor used only for subsequent pages, + - snapshot mismatch fails fast, + - route adapter head resolves global snapshot cursor and writes a source marker, + - marker-only pages advance LSN via immediate keepalive. +- Manual smoke test (self-host acceptable): + - initial snapshot visible in buckets, + - inserts/updates/deletes propagate via deltas, + - no table-not-found due to schema/tag mismatch. + +## 13) Change Checklist for Agents +- If changing snapshot/delta flow: + - update `ConvexStream` tests first. +- If changing cursor/LSN encoding: + - update `ConvexLSN` tests and backward-compat coverage. +- If changing API response parsing: + - include self-host payload fixtures in `ConvexApiClient` tests. +- If changing public behavior: + - update `README.md` and this `AGENTS.md` in the same PR. From 565d4e75f2ac406c6e84454332586d7d0c2fe990 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Mar 2026 10:18:05 -0700 Subject: [PATCH 19/86] simplify LSN representation --- modules/module-convex/src/common/ConvexLSN.ts | 68 ++++++------------- .../module-convex/test/src/ConvexLSN.test.ts | 18 +++-- 2 files changed, 36 insertions(+), 50 deletions(-) diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index a3c6012f7..287a528fb 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -1,55 +1,26 @@ const CURSOR_WIDTH = 20; -const DELIMITER = '|'; - -export interface ConvexLSNSpecification { - cursor: string; - timestamp: bigint; -} export class ConvexLSN { static ZERO = ConvexLSN.fromCursor('0'); static fromSerialized(comparable: string): ConvexLSN { - if (!comparable.includes(DELIMITER)) { - return ConvexLSN.fromCursor(comparable); - } - - const [first, second] = comparable.split(DELIMITER, 2); - const prefixed = first.match(/^n([0-9]+)$/); - if (prefixed && second != null) { - const decoded = decodeCursor(second); - return ConvexLSN.fromCursor(decoded); - } - - return ConvexLSN.fromCursor(comparable); + return ConvexLSN.fromCursor(parseSerializedCursor(comparable)); } - static fromCursor(cursor: string | number | bigint): ConvexLSN { + static fromCursor(cursor: string | bigint): ConvexLSN { const asString = `${cursor}`; - if (asString.length == 0) { - throw new Error('Convex cursor cannot be empty'); - } - - const ts = parseTimestamp(asString); - return new ConvexLSN({ - cursor: asString, - timestamp: ts - }); + assertValidCursor(asString); + return new ConvexLSN(normalizeCursor(asString)); } - constructor(private readonly options: ConvexLSNSpecification) {} - - get timestamp() { - return this.options.timestamp; - } + private constructor(private readonly value: string) {} get cursor() { - return this.options.cursor; + return this.value; } get comparable() { - const sortPart = this.options.timestamp.toString().padStart(CURSOR_WIDTH, '0'); - return `n${sortPart}${DELIMITER}${encodeCursor(this.options.cursor)}`; + return this.value.padStart(CURSOR_WIDTH, '0'); } toString() { @@ -57,21 +28,26 @@ export class ConvexLSN { } toCursorString() { - return this.options.cursor; + return this.value; } } -function parseTimestamp(cursor: string): bigint { - if (/^[0-9]+$/.test(cursor)) { - return BigInt(cursor); - } - throw new Error(`Convex cursor is not a valid numeric timestamp: ${cursor}`); +function parseSerializedCursor(comparable: string): string { + assertValidCursor(comparable); + return normalizeCursor(comparable); } -function encodeCursor(value: string): string { - return Buffer.from(value, 'utf8').toString('base64url'); +function assertValidCursor(cursor: string) { + if (cursor.length == 0) { + throw new Error('Convex cursor cannot be empty'); + } + + if (!/^[0-9]+$/.test(cursor)) { + throw new Error(`Convex cursor is not a valid numeric timestamp: ${cursor}`); + } } -function decodeCursor(value: string): string { - return Buffer.from(value, 'base64url').toString('utf8'); +function normalizeCursor(cursor: string): string { + const normalized = cursor.replace(/^0+/, ''); + return normalized == '' ? '0' : normalized; } diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts index 25acea5a0..3e4184e72 100644 --- a/modules/module-convex/test/src/ConvexLSN.test.ts +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -2,11 +2,11 @@ import { describe, expect, it } from 'vitest'; import { ConvexLSN } from '@module/common/ConvexLSN.js'; describe('ConvexLSN', () => { - it('serializes and deserializes cursor and timestamp', () => { + it('serializes and deserializes the numeric cursor', () => { const source = ConvexLSN.fromCursor('12345'); const roundTrip = ConvexLSN.fromSerialized(source.comparable); - expect(roundTrip.timestamp).toBe(12345n); + expect(source.comparable).toBe('00000000000000012345'); expect(roundTrip.toCursorString()).toBe('12345'); }); @@ -17,12 +17,22 @@ describe('ConvexLSN', () => { expect(older < newer).toBe(true); }); - it('handles bare numeric cursor string (no delimiter)', () => { + it('handles bare numeric cursor string', () => { const parsed = ConvexLSN.fromSerialized('42'); - expect(parsed.timestamp).toBe(42n); expect(parsed.toCursorString()).toBe('42'); }); + it('normalizes zero-padded serialized values', () => { + const parsed = ConvexLSN.fromSerialized('00000000000000012345'); + expect(parsed.toCursorString()).toBe('12345'); + }); + + it('rejects delimiter-based serialized payloads', () => { + expect(() => ConvexLSN.fromSerialized('n00000000000000012345|12345')).toThrow( + 'Convex cursor is not a valid numeric timestamp' + ); + }); + it('rejects non-numeric cursors', () => { expect(() => ConvexLSN.fromCursor('{"tablet":"abc","id":"xyz"}')).toThrow( 'Convex cursor is not a valid numeric timestamp' From 92f477e92424b9e3e0004b50254b5cc8da5e5580 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Mar 2026 10:42:04 -0700 Subject: [PATCH 20/86] simplify LSNs even further --- modules/module-convex/README.md | 4 ++-- modules/module-convex/src/common/ConvexLSN.ts | 20 +++++----------- .../module-convex/test/src/ConvexLSN.test.ts | 23 ++++++++++--------- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 3e530ecd2..052dc32f8 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -141,8 +141,8 @@ The content below is written in an agents.md style describing the behavior of `m - Convex snapshot and delta cursors are always `i64` timestamps (serialized as decimal numeric strings in JSON). - `ConvexLSN.fromCursor` rejects non-numeric cursors. - The `list_snapshot` pagination cursor is a separate JSON-serialized `{tablet, id}` string — it is pagination state, not a replication cursor. -- Persist LSNs as a zero-padded decimal cursor string so lexicographic ordering matches timestamp ordering. -- `ConvexLSN.fromSerialized` accepts bare numeric cursor strings and zero-padded stored values. +- Persist LSNs as the raw decimal cursor string returned by Convex. +- `ConvexLSN.fromSerialized` accepts canonical numeric cursor strings only. ## 5) API Client Contract - Auth header: `Authorization: Convex `. diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index 287a528fb..99f46d6dd 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -1,16 +1,14 @@ -const CURSOR_WIDTH = 20; - export class ConvexLSN { static ZERO = ConvexLSN.fromCursor('0'); static fromSerialized(comparable: string): ConvexLSN { - return ConvexLSN.fromCursor(parseSerializedCursor(comparable)); + return ConvexLSN.fromCursor(comparable); } static fromCursor(cursor: string | bigint): ConvexLSN { const asString = `${cursor}`; assertValidCursor(asString); - return new ConvexLSN(normalizeCursor(asString)); + return new ConvexLSN(asString); } private constructor(private readonly value: string) {} @@ -20,7 +18,7 @@ export class ConvexLSN { } get comparable() { - return this.value.padStart(CURSOR_WIDTH, '0'); + return this.value; } toString() { @@ -32,11 +30,6 @@ export class ConvexLSN { } } -function parseSerializedCursor(comparable: string): string { - assertValidCursor(comparable); - return normalizeCursor(comparable); -} - function assertValidCursor(cursor: string) { if (cursor.length == 0) { throw new Error('Convex cursor cannot be empty'); @@ -45,9 +38,8 @@ function assertValidCursor(cursor: string) { if (!/^[0-9]+$/.test(cursor)) { throw new Error(`Convex cursor is not a valid numeric timestamp: ${cursor}`); } -} -function normalizeCursor(cursor: string): string { - const normalized = cursor.replace(/^0+/, ''); - return normalized == '' ? '0' : normalized; + if (cursor.length > 1 && cursor.startsWith('0')) { + throw new Error(`Convex cursor is not a canonical numeric timestamp: ${cursor}`); + } } diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts index 3e4184e72..5f89fe8bc 100644 --- a/modules/module-convex/test/src/ConvexLSN.test.ts +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -3,32 +3,33 @@ import { ConvexLSN } from '@module/common/ConvexLSN.js'; describe('ConvexLSN', () => { it('serializes and deserializes the numeric cursor', () => { - const source = ConvexLSN.fromCursor('12345'); + const source = ConvexLSN.fromCursor('1772817606884944136'); const roundTrip = ConvexLSN.fromSerialized(source.comparable); - expect(source.comparable).toBe('00000000000000012345'); - expect(roundTrip.toCursorString()).toBe('12345'); + expect(source.comparable).toBe('1772817606884944136'); + expect(roundTrip.toCursorString()).toBe('1772817606884944136'); }); it('sorts lexicographically by timestamp', () => { - const older = ConvexLSN.fromCursor('9').comparable; - const newer = ConvexLSN.fromCursor('10').comparable; + const older = ConvexLSN.fromCursor('1772817606884944136').comparable; + const newer = ConvexLSN.fromCursor('1772817606884944137').comparable; expect(older < newer).toBe(true); }); it('handles bare numeric cursor string', () => { - const parsed = ConvexLSN.fromSerialized('42'); - expect(parsed.toCursorString()).toBe('42'); + const parsed = ConvexLSN.fromSerialized('1772817606884944136'); + expect(parsed.toCursorString()).toBe('1772817606884944136'); }); - it('normalizes zero-padded serialized values', () => { - const parsed = ConvexLSN.fromSerialized('00000000000000012345'); - expect(parsed.toCursorString()).toBe('12345'); + it('rejects padded serialized payloads', () => { + expect(() => ConvexLSN.fromSerialized('0001772817606884944136')).toThrow( + 'Convex cursor is not a canonical numeric timestamp' + ); }); it('rejects delimiter-based serialized payloads', () => { - expect(() => ConvexLSN.fromSerialized('n00000000000000012345|12345')).toThrow( + expect(() => ConvexLSN.fromSerialized('1772817606884944136|1772817606884944136')).toThrow( 'Convex cursor is not a valid numeric timestamp' ); }); From 84ba4756d21e8aa1dd767fac9f0f9cc91b0164e9 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Mar 2026 10:56:22 -0700 Subject: [PATCH 21/86] simplify convexLSN into oblivion --- modules/module-convex/README.md | 6 +-- .../src/api/ConvexRouteAPIAdapter.ts | 4 +- modules/module-convex/src/common/ConvexLSN.ts | 38 +++++-------------- .../src/replication/ConvexStream.ts | 24 ++++++------ .../module-convex/test/src/ConvexLSN.test.ts | 28 +++++++------- .../test/src/ConvexRouteAPIAdapter.test.ts | 4 +- .../test/src/ConvexStream.test.ts | 22 +++++------ modules/module-convex/test/src/util.ts | 4 +- 8 files changed, 55 insertions(+), 75 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 052dc32f8..e4b9f78b3 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -139,10 +139,10 @@ The content below is written in an agents.md style describing the behavior of `m ## 4) LSN and Cursor Rules - Convex snapshot and delta cursors are always `i64` timestamps (serialized as decimal numeric strings in JSON). -- `ConvexLSN.fromCursor` rejects non-numeric cursors. +- Convex LSN helpers reject non-numeric cursors. - The `list_snapshot` pagination cursor is a separate JSON-serialized `{tablet, id}` string — it is pagination state, not a replication cursor. - Persist LSNs as the raw decimal cursor string returned by Convex. -- `ConvexLSN.fromSerialized` accepts canonical numeric cursor strings only. +- Persisted Convex LSNs must be canonical numeric cursor strings. ## 5) API Client Contract - Auth header: `Authorization: Convex `. @@ -227,7 +227,7 @@ The content below is written in an agents.md style describing the behavior of `m - If changing snapshot/delta flow: - update `ConvexStream` tests first. - If changing cursor/LSN encoding: - - update `ConvexLSN` tests and backward-compat coverage. + - update the Convex LSN helper tests and any format-compat coverage. - If changing API response parsing: - include self-host payload fixtures in `ConvexApiClient` tests. - If changing public behavior: diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index 87608b574..1217760d4 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -1,7 +1,7 @@ import { api, ParseSyncRulesOptions, ReplicationHeadCallback, ReplicationLagOptions, SourceTable } from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; -import { ConvexLSN } from '../common/ConvexLSN.js'; +import { toConvexLsn } from '../common/ConvexLSN.js'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; import * as types from '../types/types.js'; @@ -122,7 +122,7 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { async createReplicationHead(callback: ReplicationHeadCallback): Promise { const head = await this.connectionManager.client.getHeadCursor(); await this.connectionManager.client.createWriteCheckpointMarker(); - return await callback(ConvexLSN.fromCursor(head).comparable); + return await callback(toConvexLsn(head)); } async getConnectionSchema(): Promise { diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index 99f46d6dd..c7b43a7e2 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -1,36 +1,16 @@ -export class ConvexLSN { - static ZERO = ConvexLSN.fromCursor('0'); +export const ZERO_LSN = '0'; - static fromSerialized(comparable: string): ConvexLSN { - return ConvexLSN.fromCursor(comparable); - } - - static fromCursor(cursor: string | bigint): ConvexLSN { - const asString = `${cursor}`; - assertValidCursor(asString); - return new ConvexLSN(asString); - } - - private constructor(private readonly value: string) {} - - get cursor() { - return this.value; - } - - get comparable() { - return this.value; - } - - toString() { - return this.comparable; - } +export function parseConvexLsn(lsn: string): string { + return toConvexLsn(lsn); +} - toCursorString() { - return this.value; - } +export function toConvexLsn(cursor: string | bigint): string { + const asString = `${cursor}`; + assertValidConvexLsn(asString); + return asString; } -function assertValidCursor(cursor: string) { +function assertValidConvexLsn(cursor: string) { if (cursor.length == 0) { throw new Error('Convex cursor cannot be empty'); } diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index e74eedcc0..065b7aff7 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -25,7 +25,7 @@ import { isCursorExpiredError } from '../client/ConvexApiClient.js'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; -import { ConvexLSN } from '../common/ConvexLSN.js'; +import { parseConvexLsn, toConvexLsn, ZERO_LSN } from '../common/ConvexLSN.js'; import { ConvexConnectionManager } from './ConvexConnectionManager.js'; export interface ConvexStreamOptions { @@ -117,7 +117,7 @@ export class ConvexStream { await this.storage.startBatch( { logger: this.logger, - zeroLSN: ConvexLSN.ZERO.comparable, + zeroLSN: ZERO_LSN, defaultSchema: this.defaultSchema, storeCurrentData: false, skipExistingRows: false @@ -131,7 +131,7 @@ export class ConvexStream { // Resolve source tables up-front to warm table metadata and sync-rule matching. await this.resolveAllSourceTables(batch); - let cursor = ConvexLSN.fromSerialized(resumeFromLsn).toCursorString(); + let cursor = parseConvexLsn(resumeFromLsn); while (!this.abortSignal.aborted) { const page = await this.connections.client @@ -147,7 +147,7 @@ export class ConvexStream { }); const nextCursor = page.cursor; - const pageLsn = ConvexLSN.fromCursor(nextCursor).comparable; + const pageLsn = toConvexLsn(nextCursor); let changesInPage = 0; let sawCheckpointMarker = false; @@ -251,15 +251,15 @@ export class ConvexStream { const flushResult = await this.storage.startBatch( { logger: this.logger, - zeroLSN: ConvexLSN.ZERO.comparable, + zeroLSN: ZERO_LSN, defaultSchema: this.defaultSchema, storeCurrentData: false, skipExistingRows: true }, async (batch) => { const snapshotCursor = await this.resolveSnapshotBoundary(snapshotLsn); - const snapshotComparable = ConvexLSN.fromCursor(snapshotCursor).comparable; - await batch.setResumeLsn(snapshotComparable); + const snapshotLsnValue = toConvexLsn(snapshotCursor); + await batch.setResumeLsn(snapshotLsnValue); const sourceTables = await this.resolveAllSourceTables(batch); @@ -280,9 +280,9 @@ export class ConvexStream { await this.snapshotTable(batch, tableWithProgress, snapshotCursor); } - await batch.commit(snapshotComparable); + await batch.commit(snapshotLsnValue); - this.logger.info(`Snapshot done. Need to replicate from ${snapshotComparable} for consistency.`); + this.logger.info(`Snapshot done. Need to replicate from ${snapshotLsnValue} for consistency.`); } ); @@ -369,8 +369,8 @@ export class ConvexStream { throw new ReplicationAbortedError('Initial replication interrupted'); } - const snapshotComparable = ConvexLSN.fromCursor(snapshotCursor).comparable; - const [doneTable] = await batch.markSnapshotDone([latestTable], snapshotComparable); + const snapshotLsnValue = toConvexLsn(snapshotCursor); + const [doneTable] = await batch.markSnapshotDone([latestTable], snapshotLsnValue); this.relationCache.update(doneTable); return { @@ -380,7 +380,7 @@ export class ConvexStream { private async resolveSnapshotBoundary(snapshotLsn: string | null): Promise { if (snapshotLsn != null) { - const snapshotCursor = ConvexLSN.fromSerialized(snapshotLsn).toCursorString(); + const snapshotCursor = parseConvexLsn(snapshotLsn); this.logger.info(`Using existing global snapshot ${snapshotCursor}`); return snapshotCursor; } diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts index 5f89fe8bc..db6752aa9 100644 --- a/modules/module-convex/test/src/ConvexLSN.test.ts +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -1,41 +1,41 @@ import { describe, expect, it } from 'vitest'; -import { ConvexLSN } from '@module/common/ConvexLSN.js'; +import { parseConvexLsn, toConvexLsn } from '@module/common/ConvexLSN.js'; -describe('ConvexLSN', () => { - it('serializes and deserializes the numeric cursor', () => { - const source = ConvexLSN.fromCursor('1772817606884944136'); - const roundTrip = ConvexLSN.fromSerialized(source.comparable); +describe('Convex cursor LSN helpers', () => { + it('validates and round-trips the numeric cursor', () => { + const source = toConvexLsn('1772817606884944136'); + const roundTrip = parseConvexLsn(source); - expect(source.comparable).toBe('1772817606884944136'); - expect(roundTrip.toCursorString()).toBe('1772817606884944136'); + expect(source).toBe('1772817606884944136'); + expect(roundTrip).toBe('1772817606884944136'); }); it('sorts lexicographically by timestamp', () => { - const older = ConvexLSN.fromCursor('1772817606884944136').comparable; - const newer = ConvexLSN.fromCursor('1772817606884944137').comparable; + const older = toConvexLsn('1772817606884944136'); + const newer = toConvexLsn('1772817606884944137'); expect(older < newer).toBe(true); }); it('handles bare numeric cursor string', () => { - const parsed = ConvexLSN.fromSerialized('1772817606884944136'); - expect(parsed.toCursorString()).toBe('1772817606884944136'); + const parsed = parseConvexLsn('1772817606884944136'); + expect(parsed).toBe('1772817606884944136'); }); it('rejects padded serialized payloads', () => { - expect(() => ConvexLSN.fromSerialized('0001772817606884944136')).toThrow( + expect(() => parseConvexLsn('0001772817606884944136')).toThrow( 'Convex cursor is not a canonical numeric timestamp' ); }); it('rejects delimiter-based serialized payloads', () => { - expect(() => ConvexLSN.fromSerialized('1772817606884944136|1772817606884944136')).toThrow( + expect(() => parseConvexLsn('1772817606884944136|1772817606884944136')).toThrow( 'Convex cursor is not a valid numeric timestamp' ); }); it('rejects non-numeric cursors', () => { - expect(() => ConvexLSN.fromCursor('{"tablet":"abc","id":"xyz"}')).toThrow( + expect(() => toConvexLsn('{"tablet":"abc","id":"xyz"}')).toThrow( 'Convex cursor is not a valid numeric timestamp' ); }); diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index 30b83c130..8bd40b294 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -1,7 +1,7 @@ import { SqlSyncRules } from '@powersync/service-sync-rules'; import { describe, expect, it, vi } from 'vitest'; import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; -import { ConvexLSN } from '@module/common/ConvexLSN.js'; +import { toConvexLsn } from '@module/common/ConvexLSN.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; function createAdapter() { @@ -80,7 +80,7 @@ bucket_definitions: (adapter as any).connectionManager.client.createWriteCheckpointMarker = createWriteCheckpointMarker; const result = await adapter.createReplicationHead(async (head) => head); - expect(result).toBe(ConvexLSN.fromCursor('123').comparable); + expect(result).toBe(toConvexLsn('123')); expect(getHeadCursor).toHaveBeenCalledTimes(1); expect(getHeadCursor).toHaveBeenCalledWith(); expect(createWriteCheckpointMarker).toHaveBeenCalledTimes(1); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 136e66b70..ea0b2942e 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -1,7 +1,7 @@ import { SaveOperationTag, SourceTable } from '@powersync/service-core'; import { TablePattern } from '@powersync/service-sync-rules'; import { describe, expect, it, vi } from 'vitest'; -import { ConvexLSN } from '@module/common/ConvexLSN.js'; +import { toConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; import { ConvexStream } from '@module/replication/ConvexStream.js'; function createFakeStorage(options?: { @@ -40,7 +40,7 @@ function createFakeStorage(options?: { const batch: any = { lastCheckpointLsn: null, resumeFromLsn: options?.resumeFromLsn ?? null, - noCheckpointBeforeLsn: ConvexLSN.ZERO.comparable, + noCheckpointBeforeLsn: ZERO_LSN, async save(record: any) { saves.push(record); return null; @@ -95,7 +95,7 @@ function createFakeStorage(options?: { return { active: true, snapshot_done: options?.snapshotDone ?? false, - checkpoint_lsn: options?.snapshotDone ? ConvexLSN.fromCursor('100').comparable : null, + checkpoint_lsn: options?.snapshotDone ? toConvexLsn('100') : null, snapshot_lsn: options?.snapshotLsn ?? null }; }, @@ -181,12 +181,12 @@ describe('ConvexStream', () => { expect(context.saves.length).toBe(1); expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); expect(context.resumeLsnUpdates.length).toBe(1); - expect(context.commits.at(-1)).toBe(ConvexLSN.fromCursor('100').comparable); + expect(context.commits.at(-1)).toBe(toConvexLsn('100')); }); it('starts each table snapshot from first page, then paginates within the run', async () => { const context = createFakeStorage({ - snapshotLsn: ConvexLSN.fromCursor('200').comparable, + snapshotLsn: toConvexLsn('200'), tableSnapshotStatus: { replicatedCount: 99, totalEstimatedCount: -1, @@ -254,7 +254,7 @@ describe('ConvexStream', () => { it('fails when table snapshots return a different snapshot boundary', async () => { const context = createFakeStorage({ - snapshotLsn: ConvexLSN.fromCursor('300').comparable + snapshotLsn: toConvexLsn('300') }); const abortController = new AbortController(); @@ -291,7 +291,7 @@ describe('ConvexStream', () => { it('streams deltas and commits checkpoint', async () => { const context = createFakeStorage({ snapshotDone: true, - resumeFromLsn: ConvexLSN.fromCursor('100').comparable + resumeFromLsn: toConvexLsn('100') }); const abortController = new AbortController(); @@ -337,14 +337,14 @@ describe('ConvexStream', () => { expect(context.saves.length).toBe(2); expect(context.saves[0]?.tag).toBe(SaveOperationTag.UPDATE); expect(context.saves[1]?.tag).toBe(SaveOperationTag.DELETE); - expect(context.commits.at(-1)).toBe(ConvexLSN.fromCursor('101').comparable); + expect(context.commits.at(-1)).toBe(toConvexLsn('101')); expect(deltaCalls[0]?.tableName).toBeUndefined(); }); it('keeps alive immediately when only checkpoint marker rows are streamed', async () => { const context = createFakeStorage({ snapshotDone: true, - resumeFromLsn: ConvexLSN.fromCursor('100').comparable + resumeFromLsn: toConvexLsn('100') }); const abortController = new AbortController(); let calls = 0; @@ -391,8 +391,8 @@ describe('ConvexStream', () => { expect(context.saves.length).toBe(0); expect(context.commits.length).toBe(0); expect(context.keepalives).toEqual([ - ConvexLSN.fromCursor('101').comparable, - ConvexLSN.fromCursor('102').comparable + toConvexLsn('101'), + toConvexLsn('102') ]); }); }); diff --git a/modules/module-convex/test/src/util.ts b/modules/module-convex/test/src/util.ts index 04c690616..19d17fdef 100644 --- a/modules/module-convex/test/src/util.ts +++ b/modules/module-convex/test/src/util.ts @@ -13,7 +13,7 @@ import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import { ConvexConnectionManager } from '@module/replication/ConvexConnectionManager.js'; import { ConvexStream, ConvexStreamOptions } from '@module/replication/ConvexStream.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; -import { ConvexLSN } from '@module/common/ConvexLSN.js'; +import { ZERO_LSN } from '@module/common/ConvexLSN.js'; import { env } from './env.js'; export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ @@ -169,7 +169,7 @@ export class ConvexStreamTestContext { await this.storage!.startBatch( { defaultSchema: 'convex', - zeroLSN: ConvexLSN.ZERO.comparable, + zeroLSN: ZERO_LSN, storeCurrentData: false, skipExistingRows: false }, From 78f821a06c8b3da68b709c1c30dbefa5e75adab5 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Mar 2026 11:49:10 -0700 Subject: [PATCH 22/86] remove configurable request timeout hardcode to 60 --- modules/module-convex/src/client/ConvexApiClient.ts | 6 ++++-- modules/module-convex/src/types/types.ts | 3 --- modules/module-convex/test/src/ConvexApiClient.test.ts | 3 +-- modules/module-convex/test/src/types.test.ts | 1 - modules/module-convex/test/src/util.ts | 3 +-- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index 9555208d5..e05517bd2 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -3,6 +3,8 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { NormalizedConvexConnectionConfig } from '../types/types.js'; import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; +const CONVEX_REQUEST_TIMEOUT_MS = 60_000; + export interface ConvexRawDocument { _id?: string; _table?: string; @@ -207,10 +209,10 @@ export class ConvexApiClient { } const timeout = new AbortController(); - const timeoutPromise = delay(this.config.requestTimeoutMs, undefined, { + const timeoutPromise = delay(CONVEX_REQUEST_TIMEOUT_MS, undefined, { signal: timeout.signal }).then(() => { - timeout.abort(new Error(`Convex API request timed out after ${this.config.requestTimeoutMs}ms`)); + timeout.abort(new Error(`Convex API request timed out after ${CONVEX_REQUEST_TIMEOUT_MS}ms`)); }); const signals = [timeout.signal]; diff --git a/modules/module-convex/src/types/types.ts b/modules/module-convex/src/types/types.ts index db8a23405..f3648df51 100644 --- a/modules/module-convex/src/types/types.ts +++ b/modules/module-convex/src/types/types.ts @@ -14,7 +14,6 @@ export interface NormalizedConvexConnectionConfig { debugApi: boolean; pollingIntervalMs: number; - requestTimeoutMs: number; lookup?: LookupFunction; } @@ -25,7 +24,6 @@ export const ConvexConnectionConfig = service_types.configFile.DataSourceConfig. deployment_url: t.string, deploy_key: t.string, polling_interval_ms: t.number.optional(), - request_timeout_ms: t.number.optional(), reject_ip_ranges: t.array(t.string).optional() }) ); @@ -73,7 +71,6 @@ export function normalizeConnectionConfig(options: ConvexConnectionConfig): Norm debugApi: options.debug_api ?? false, pollingIntervalMs: options.polling_interval_ms ?? 1_000, - requestTimeoutMs: options.request_timeout_ms ?? 30_000, lookup }; diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index 098e10d38..46553bd40 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -6,8 +6,7 @@ import { normalizeConnectionConfig } from '@module/types/types.js'; const baseConfig = normalizeConnectionConfig({ type: 'convex', deployment_url: 'https://example.convex.cloud', - deploy_key: 'test-key', - request_timeout_ms: 5000 + deploy_key: 'test-key' }); describe('ConvexApiClient', () => { diff --git a/modules/module-convex/test/src/types.test.ts b/modules/module-convex/test/src/types.test.ts index ab7fd4e51..7a4c9866f 100644 --- a/modules/module-convex/test/src/types.test.ts +++ b/modules/module-convex/test/src/types.test.ts @@ -12,7 +12,6 @@ describe('Convex connection config', () => { expect(config.id).toBe('default'); expect(config.tag).toBe('default'); expect(config.pollingIntervalMs).toBe(1000); - expect(config.requestTimeoutMs).toBe(30000); expect(config.deploymentUrl).toBe('https://example.convex.cloud'); }); diff --git a/modules/module-convex/test/src/util.ts b/modules/module-convex/test/src/util.ts index 19d17fdef..c38882187 100644 --- a/modules/module-convex/test/src/util.ts +++ b/modules/module-convex/test/src/util.ts @@ -30,8 +30,7 @@ export function makeConvexConnectionManager() { type: 'convex' as const, deployment_url: env.CONVEX_URL, deploy_key: env.CONVEX_DEPLOY_KEY, - polling_interval_ms: 200, - request_timeout_ms: 15_000 + polling_interval_ms: 200 }; const config = { ...rawConfig, ...normalizeConnectionConfig(rawConfig) }; From 596c1aa399e8e90907ccf9aad65b29a2f06da86f Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Mar 2026 11:54:21 -0700 Subject: [PATCH 23/86] resumable initial replication --- modules/module-convex/README.md | 8 +- .../src/replication/ConvexStream.ts | 126 +++++++++++++++--- .../test/src/ConvexStream.test.ts | 71 +++++++--- 3 files changed, 162 insertions(+), 43 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index e4b9f78b3..5cf90e1ca 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -13,7 +13,6 @@ replication: deployment_url: https://.convex.cloud deploy_key: polling_interval_ms: 1000 - request_timeout_ms: 30000 ``` ## Manual smoke test @@ -27,8 +26,8 @@ replication: - Initial replication pins a global Convex snapshot boundary using `list_snapshot` without `tableName`. - Table hydration then runs per selected sync-rule table using that fixed snapshot boundary. -- Each table snapshot starts from the first page (cursor omitted on the first call), and only uses cursor pagination within the current run. -- If initial replication is interrupted, restart resumes from the stored snapshot boundary but restarts table paging from page 1. +- Each table snapshot starts from the first page when there is no persisted progress, and otherwise resumes from the stored page cursor. +- If initial replication is interrupted, restart resumes from the stored snapshot boundary and any persisted per-table snapshot progress. ## Convex Source Setup @@ -131,7 +130,8 @@ The content below is written in an agents.md style describing the behavior of `m - All table snapshots in a run must use the same pinned `snapshot`; if response snapshot differs, fail fast. - On restart during initial replication: - Reuse persisted snapshot LSN boundary. - - Restart table page walk from first page (do not resume per-table `lastKey`). + - Resume table page walk from the persisted per-table `lastKey` cursor when available. + - If the last page was already flushed before interruption, mark the table snapshot done without re-reading rows. - Delta streaming starts from resume LSN (snapshot boundary), not from table page cursor. - `tablePattern.connectionTag` and schema must match before table selection. - Source table replica identity is `_id`. diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 065b7aff7..3ca5eb5ec 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -269,13 +269,15 @@ export class ConvexStream { continue; } - const tableWithProgress = await batch.updateTableProgress(sourceTable, { - totalEstimatedCount: -1, - replicatedCount: 0, - lastKey: null - }); + const tableWithProgress = + sourceTable.snapshotStatus == null + ? await batch.updateTableProgress(sourceTable, { + totalEstimatedCount: -1, + replicatedCount: 0, + lastKey: null + }) + : sourceTable; this.relationCache.update(tableWithProgress); - this.logger.info(`Starting table snapshot from first page for [${tableWithProgress.qualifiedName}]`); await this.snapshotTable(batch, tableWithProgress, snapshotCursor); } @@ -296,18 +298,35 @@ export class ConvexStream { table: SourceTable, snapshotCursor: string ): Promise<{ table: SourceTable }> { - let pageCursor: string | null = null; - let replicatedCount = 0; + const snapshotProgress = decodeSnapshotProgressCursor(table.snapshotStatus?.lastKey); + let pageCursor = snapshotProgress.cursor; + let replicatedCount = table.snapshotStatus?.replicatedCount ?? 0; let latestTable = table; - let firstPage = true; + + if (snapshotProgress.finished) { + this.logger.info(`Finishing table snapshot from persisted progress for [${table.qualifiedName}]`); + } else if (pageCursor != null) { + this.logger.info(`Resuming table snapshot from persisted cursor for [${table.qualifiedName}]`); + } else { + this.logger.info(`Starting table snapshot from first page for [${table.qualifiedName}]`); + } + + if (this.abortSignal.aborted) { + throw new ReplicationAbortedError('Initial replication interrupted'); + } + + if (snapshotProgress.finished) { + return { + table: await this.markSnapshotDone(batch, latestTable, snapshotCursor) + }; + } while (!this.abortSignal.aborted) { - const requestCursor: string | undefined = firstPage ? undefined : (pageCursor ?? undefined); const page: ConvexListSnapshotResult = await this.connections.client .listSnapshot({ tableName: table.name, snapshot: snapshotCursor, - cursor: requestCursor, + cursor: pageCursor ?? undefined, signal: this.abortSignal }) .catch((error) => { @@ -316,7 +335,6 @@ export class ConvexStream { } throw error; }); - firstPage = false; if (snapshotCursor != page.snapshot) { throw new ReplicationAssertionError( @@ -354,7 +372,10 @@ export class ConvexStream { latestTable = await batch.updateTableProgress(latestTable, { replicatedCount, totalEstimatedCount: -1, - lastKey: encodeSnapshotProgressCursor(pageCursor) + lastKey: encodeSnapshotProgressCursor({ + cursor: pageCursor, + finished: !page.hasMore + }) }); this.relationCache.update(latestTable); @@ -369,15 +390,22 @@ export class ConvexStream { throw new ReplicationAbortedError('Initial replication interrupted'); } - const snapshotLsnValue = toConvexLsn(snapshotCursor); - const [doneTable] = await batch.markSnapshotDone([latestTable], snapshotLsnValue); - this.relationCache.update(doneTable); - return { - table: doneTable + table: await this.markSnapshotDone(batch, latestTable, snapshotCursor) }; } + private async markSnapshotDone( + batch: storage.BucketStorageBatch, + table: SourceTable, + snapshotCursor: string + ): Promise { + const snapshotLsnValue = toConvexLsn(snapshotCursor); + const [doneTable] = await batch.markSnapshotDone([table], snapshotLsnValue); + this.relationCache.update(doneTable); + return doneTable; + } + private async resolveSnapshotBoundary(snapshotLsn: string | null): Promise { if (snapshotLsn != null) { const snapshotCursor = parseConvexLsn(snapshotLsn); @@ -631,10 +659,66 @@ function toConvexSyncValue(value: unknown): any { return null; } -function encodeSnapshotProgressCursor(cursor: string | null): Uint8Array | null { - if (cursor == null) { +interface ConvexSnapshotProgressCursor { + cursor: string | null; + finished: boolean; +} + +const SNAPSHOT_PROGRESS_PREFIX = 'convex-snapshot-progress:'; + +function encodeSnapshotProgressCursor(progress: ConvexSnapshotProgressCursor): Uint8Array | null { + if (!progress.finished && progress.cursor == null) { return null; } - return Buffer.from(cursor, 'utf8'); + if (!progress.finished) { + return Buffer.from(progress.cursor!, 'utf8'); + } + + return Buffer.from(`${SNAPSHOT_PROGRESS_PREFIX}${JSON.stringify(progress)}`, 'utf8'); +} + +function decodeSnapshotProgressCursor(value: Uint8Array | null | undefined): ConvexSnapshotProgressCursor { + if (value == null) { + return { + cursor: null, + finished: false + }; + } + + const serialized = Buffer.from(value).toString('utf8'); + if (!serialized.startsWith(SNAPSHOT_PROGRESS_PREFIX)) { + return { + cursor: serialized, + finished: false + }; + } + + let parsed: unknown; + try { + parsed = JSON.parse(serialized.slice(SNAPSHOT_PROGRESS_PREFIX.length)); + } catch (error) { + throw new ReplicationAssertionError( + `Convex snapshot progress cursor is not valid JSON: ${error instanceof Error ? error.message : `${error}`}` + ); + } + + if (typeof parsed != 'object' || parsed == null || Array.isArray(parsed)) { + throw new ReplicationAssertionError('Convex snapshot progress cursor must decode to an object'); + } + + const parsedProgress = parsed as { cursor?: unknown; finished?: unknown }; + const cursor = parsedProgress.cursor; + const finished = parsedProgress.finished; + if (cursor != null && typeof cursor != 'string') { + throw new ReplicationAssertionError('Convex snapshot progress cursor must contain a string cursor or null'); + } + if (typeof finished != 'boolean') { + throw new ReplicationAssertionError('Convex snapshot progress cursor must contain a boolean finished flag'); + } + + return { + cursor: cursor ?? null, + finished + }; } diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index ea0b2942e..c3e8babb2 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -184,13 +184,13 @@ describe('ConvexStream', () => { expect(context.commits.at(-1)).toBe(toConvexLsn('100')); }); - it('starts each table snapshot from first page, then paginates within the run', async () => { + it('resumes table snapshots from the persisted page cursor', async () => { const context = createFakeStorage({ snapshotLsn: toConvexLsn('200'), tableSnapshotStatus: { - replicatedCount: 99, + replicatedCount: 1, totalEstimatedCount: -1, - lastKey: Buffer.from('stale-cursor', 'utf8') + lastKey: Buffer.from('page-2', 'utf8') } }); const abortController = new AbortController(); @@ -198,14 +198,6 @@ describe('ConvexStream', () => { const snapshotCalls: any[] = []; const listSnapshot = vi.fn(async (options: any) => { snapshotCalls.push(options ?? {}); - if (snapshotCalls.length == 1) { - return { - snapshot: '200', - cursor: 'page-2', - hasMore: true, - values: [{ _table: 'users', _id: 'u1', name: 'Alice' }] - }; - } return { snapshot: '200', cursor: null, @@ -241,15 +233,58 @@ describe('ConvexStream', () => { await stream.initReplication(); expect(getGlobalSnapshotCursor).not.toHaveBeenCalled(); - expect(snapshotCalls.length).toBe(2); + expect(snapshotCalls.length).toBe(1); expect(snapshotCalls[0]?.snapshot).toBe('200'); - expect(snapshotCalls[0]?.cursor).toBeUndefined(); - expect(snapshotCalls[1]?.cursor).toBe('page-2'); - expect(context.tableProgressUpdates[0]).toMatchObject({ - replicatedCount: 0, - lastKey: null, - totalEstimatedCount: -1 + expect(snapshotCalls[0]?.cursor).toBe('page-2'); + expect(context.saves.length).toBe(1); + expect(context.tableProgressUpdates).toHaveLength(1); + expect(context.tableProgressUpdates[0]?.replicatedCount).toBe(2); + }); + + it('marks snapshot done without re-reading rows when the final page was already flushed', async () => { + const context = createFakeStorage({ + snapshotLsn: toConvexLsn('200'), + tableSnapshotStatus: { + replicatedCount: 2, + totalEstimatedCount: -1, + lastKey: Buffer.from('convex-snapshot-progress:{"cursor":null,"finished":true}', 'utf8') + } }); + const abortController = new AbortController(); + const listSnapshot = vi.fn(async () => ({ + snapshot: '200', + cursor: null, + hasMore: false, + values: [] + })); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: {} }], + raw: {} + }), + listSnapshot, + getGlobalSnapshotCursor: async () => 'should-not-be-called' + } + } as any + }); + + await stream.initReplication(); + + expect(listSnapshot).not.toHaveBeenCalled(); + expect(context.saves.length).toBe(0); + expect(context.table.snapshotComplete).toBe(true); }); it('fails when table snapshots return a different snapshot boundary', async () => { From f0c83b6e7a5708c22292d1c9d101f94b1f452c9a Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Mar 2026 13:20:22 -0700 Subject: [PATCH 24/86] inline snapshotting for new tables also cleaned up handling of convex types --- modules/module-convex/README.md | 5 +- .../src/replication/ConvexStream.ts | 38 +++-- .../test/src/ConvexStream.test.ts | 133 +++++++++++++++--- 3 files changed, 142 insertions(+), 34 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 5cf90e1ca..d27f3562f 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -124,6 +124,7 @@ The content below is written in an agents.md style describing the behavior of `m - Start from persisted resume LSN. - Poll `document_deltas`. - Always stream globally (no `tableName` filter), then filter locally by selected sync-rules tables. + - If a table is first seen in a `document_deltas` page and matches sync rules, snapshot it inline at that page boundary and skip that table's delta rows from the same page, because the snapshot already includes them. ## 3) Hard Invariants (Do Not Break) - `snapshot` is the consistency boundary; page `cursor` is pagination state. @@ -171,10 +172,10 @@ The content below is written in an agents.md style describing the behavior of `m - `number` integer -> `BigInt` (SQLite INTEGER), - `number` float -> REAL, - `boolean` -> `1n`/`0n`, - - `Uint8Array` -> BLOB, + - bytes over the JSON API -> base64 `string`, - `object/array` -> JSON TEXT. NOTE: - - Preserve bytes as BLOB in sync rules; use explicit base64 conversion in rules only when needed by consumers. + - Convex does not expose a native `Date` wire type here; timestamps arrive as `number` or `string`. ## 9) Checkpointing and Consistency - `createReplicationHead` must: diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 3ca5eb5ec..a67c12d12 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -151,6 +151,7 @@ export class ConvexStream { let changesInPage = 0; let sawCheckpointMarker = false; + const snapshottedTablesInPage = new Set(); for (const change of page.values) { if (this.abortSignal.aborted) { throw new ReplicationAbortedError('Replication interrupted'); @@ -166,10 +167,13 @@ export class ConvexStream { continue; } - const table = await this.getOrResolveTable(batch, tableName); + const table = await this.getOrResolveTable(batch, tableName, nextCursor, snapshottedTablesInPage); if (table == null || !table.syncAny) { continue; } + if (snapshottedTablesInPage.has(tableName)) { + continue; + } const changed = await this.writeChange(batch, table, change); if (!changed) { @@ -472,7 +476,12 @@ export class ConvexStream { return resolved; } - private async getOrResolveTable(batch: storage.BucketStorageBatch, tableName: string): Promise { + private async getOrResolveTable( + batch: storage.BucketStorageBatch, + tableName: string, + snapshotCursor: string, + snapshottedTablesInPage: Set + ): Promise { const descriptor: SourceEntityDescriptor = { schema: this.defaultSchema, name: tableName, @@ -489,10 +498,21 @@ export class ConvexStream { return null; } - // Refresh schema cache when we discover a new table while streaming. - await this.getAllTableSchemas({ force: true }); + let table = await this.processTable(batch, descriptor); + if (!table.snapshotComplete && table.syncAny) { + this.logger.info(`New table discovered while streaming: [${table.qualifiedName}]`); + await batch.truncate([table]); + table = await batch.updateTableProgress(table, { + totalEstimatedCount: -1, + replicatedCount: 0, + lastKey: null + }); + this.relationCache.update(table); + table = (await this.snapshotTable(batch, table, snapshotCursor)).table; + snapshottedTablesInPage.add(tableName); + } - return await this.processTable(batch, descriptor); + return table; } private isTableSelectedBySyncRules(tableName: string): boolean { @@ -644,14 +664,6 @@ function toConvexSyncValue(value: unknown): any { return value ? 1n : 0n; } - if (value instanceof Uint8Array) { - return value; - } - - if (value instanceof Date) { - return value.toISOString(); - } - if (Array.isArray(value) || typeof value == 'object') { return JSON.stringify(value); } diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index c3e8babb2..38b8d5320 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -8,6 +8,7 @@ function createFakeStorage(options?: { snapshotDone?: boolean; snapshotLsn?: string | null; resumeFromLsn?: string | null; + sourcePatterns?: TablePattern[]; tableSnapshotStatus?: { totalEstimatedCount?: number; replicatedCount?: number; @@ -20,22 +21,48 @@ function createFakeStorage(options?: { const resumeLsnUpdates: string[] = []; const tableProgressUpdates: any[] = []; - const table = new SourceTable({ - id: 1, - connectionTag: 'default', - objectId: 'users', - schema: 'convex', - name: 'users', - replicaIdColumns: [{ name: '_id' }], - snapshotComplete: false + const tables = new Map(); + let nextTableId = 1; + const getOrCreateTable = ( + name: string, + tableOptions?: { + snapshotStatus?: { + totalEstimatedCount?: number; + replicatedCount?: number; + lastKey?: Uint8Array | null; + }; + snapshotComplete?: boolean; + } + ) => { + const existing = tables.get(name); + if (existing != null) { + return existing; + } + + const table = new SourceTable({ + id: nextTableId++, + connectionTag: 'default', + objectId: name, + schema: 'convex', + name, + replicaIdColumns: [{ name: '_id' }], + snapshotComplete: tableOptions?.snapshotComplete ?? false + }); + if (tableOptions?.snapshotStatus) { + table.snapshotStatus = { + totalEstimatedCount: tableOptions.snapshotStatus.totalEstimatedCount ?? -1, + replicatedCount: tableOptions.snapshotStatus.replicatedCount ?? 0, + lastKey: tableOptions.snapshotStatus.lastKey ?? null + }; + } + + tables.set(name, table); + return table; + }; + + const table = getOrCreateTable('users', { + snapshotStatus: options?.tableSnapshotStatus }); - if (options?.tableSnapshotStatus) { - table.snapshotStatus = { - totalEstimatedCount: options.tableSnapshotStatus.totalEstimatedCount ?? -1, - replicatedCount: options.tableSnapshotStatus.replicatedCount ?? 0, - lastKey: options.tableSnapshotStatus.lastKey ?? null - }; - } const batch: any = { lastCheckpointLsn: null, @@ -75,7 +102,10 @@ function createFakeStorage(options?: { return tables; }, async updateTableProgress(sourceTable: SourceTable, progress: any) { - tableProgressUpdates.push(progress); + tableProgressUpdates.push({ + tableName: sourceTable.name, + ...progress + }); sourceTable.snapshotStatus = { totalEstimatedCount: progress.totalEstimatedCount ?? sourceTable.snapshotStatus?.totalEstimatedCount ?? -1, replicatedCount: progress.replicatedCount ?? sourceTable.snapshotStatus?.replicatedCount ?? 0, @@ -88,7 +118,7 @@ function createFakeStorage(options?: { const storage = { group_id: 1, getParsedSyncRules: () => ({ - getSourceTables: () => [new TablePattern('convex', 'users')], + getSourceTables: () => options?.sourcePatterns ?? [new TablePattern('convex', 'users')], applyRowContext: (row: Record) => row }), async getStatus() { @@ -101,8 +131,8 @@ function createFakeStorage(options?: { }, clear: vi.fn(async () => undefined), populatePersistentChecksumCache: vi.fn(async () => ({ buckets: 0 })), - resolveTable: vi.fn(async () => ({ - table, + resolveTable: vi.fn(async ({ entity_descriptor }: any) => ({ + table: getOrCreateTable(entity_descriptor.name), dropTables: [] })), startBatch: vi.fn(async (_options: any, callback: (batch: any) => Promise) => { @@ -116,6 +146,7 @@ function createFakeStorage(options?: { storage, batch, table, + tables, saves, commits, keepalives, @@ -376,6 +407,70 @@ describe('ConvexStream', () => { expect(deltaCalls[0]?.tableName).toBeUndefined(); }); + it('snapshots a newly discovered wildcard-matched table inline without refreshing metadata', async () => { + const context = createFakeStorage({ + snapshotDone: true, + resumeFromLsn: toConvexLsn('100'), + sourcePatterns: [new TablePattern('convex', 'projects%')] + }); + const abortController = new AbortController(); + + let calls = 0; + const getJsonSchemas = vi.fn(async () => ({ + tables: [{ tableName: 'users', schema: {} }], + raw: {} + })); + const listSnapshot = vi.fn(async (options: any) => ({ + snapshot: '101', + cursor: null, + hasMore: false, + values: [{ _table: 'projects_archive', _id: 'p1', name: 'From snapshot' }] + })); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas, + listSnapshot, + documentDeltas: async () => { + calls += 1; + setTimeout(() => abortController.abort(), 0); + return { + cursor: '101', + hasMore: false, + values: [{ _table: 'projects_archive', _id: 'p1', name: 'From delta' }] + }; + } + } + } as any + }); + + await stream.streamChanges(); + + expect(calls).toBeGreaterThan(0); + expect(getJsonSchemas).toHaveBeenCalledTimes(1); + expect(listSnapshot).toHaveBeenCalledTimes(1); + expect(listSnapshot).toHaveBeenCalledWith({ + tableName: 'projects_archive', + snapshot: '101', + cursor: undefined, + signal: abortController.signal + }); + expect(context.saves.length).toBe(1); + expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); + expect(context.saves[0]?.sourceTable.name).toBe('projects_archive'); + expect(context.tables.get('projects_archive')?.snapshotComplete).toBe(true); + }); + it('keeps alive immediately when only checkpoint marker rows are streamed', async () => { const context = createFakeStorage({ snapshotDone: true, From 05945b914d18e4846fff3c4516903bcbbcc5d78b Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Mar 2026 16:15:25 -0700 Subject: [PATCH 25/86] clean up type mapping --- modules/module-convex/README.md | 12 +- .../src/api/ConvexRouteAPIAdapter.ts | 66 +---- .../src/common/convex-to-sqlite.ts | 225 ++++++++++++++++++ .../src/replication/ConvexStream.ts | 62 ++--- .../test/src/ConvexRouteAPIAdapter.test.ts | 9 +- .../test/src/ConvexStream.test.ts | 61 ++++- .../test/src/convex-to-sqlite.test.ts | 61 +++++ 7 files changed, 381 insertions(+), 115 deletions(-) create mode 100644 modules/module-convex/src/common/convex-to-sqlite.ts create mode 100644 modules/module-convex/test/src/convex-to-sqlite.test.ts diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index d27f3562f..931dee6f1 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -169,13 +169,15 @@ The content below is written in an agents.md style describing the behavior of `m ## 8) Datatype Mapping - Current runtime mapping in stream writer: - - `number` integer -> `BigInt` (SQLite INTEGER), - - `number` float -> REAL, - - `boolean` -> `1n`/`0n`, - - bytes over the JSON API -> base64 `string`, - - `object/array` -> JSON TEXT. + - `id` / `string` -> TEXT, + - `int64` / integer `number` -> `BigInt` (SQLite INTEGER), + - `float64` / non-integer `number` -> REAL, + - `boolean` -> `1n`/`0n` (SQLite INTEGER), + - `bytes` over the JSON API -> base64 `string` on the wire -> `Uint8Array` (SQLite BLOB), + - `array` / `object` / `record` -> JSON TEXT. NOTE: - Convex does not expose a native `Date` wire type here; timestamps arrive as `number` or `string`. + - BLOB values are valid row values but are not valid bucket parameter values. ## 9) Checkpointing and Consistency - `createReplicationHead` must: diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index 1217760d4..dd47370dd 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -3,6 +3,7 @@ import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import { toConvexLsn } from '../common/ConvexLSN.js'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; +import { extractProperties, readConvexFieldType, toExpressionTypeFromConvexType } from '../common/convex-to-sqlite.js'; import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; import * as types from '../types/types.js'; @@ -136,13 +137,13 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { .map((table) => ({ name: table.tableName, columns: Object.entries({ - _id: { type: 'string' }, - ...extractProperties(table.schema) + ...extractProperties(table.schema), + _id: { type: 'id' } }) .sort(([a], [b]) => a.localeCompare(b)) .map(([columnName, property]) => { - const jsonType = readJsonSchemaType(property); - const sqliteType = toSqliteType(jsonType); + const jsonType = readConvexFieldType(property); + const sqliteType = toExpressionTypeFromConvexType(jsonType); return { name: columnName, @@ -212,60 +213,3 @@ function createTableInfo(options: { errors: options.errors ?? [] }; } - -function extractProperties(schema: Record) { - const direct = schema.properties; - if (isRecord(direct)) { - return direct; - } - - const nested = schema.schema?.properties; - if (isRecord(nested)) { - return nested; - } - - return {}; -} - -function readJsonSchemaType(value: unknown): string { - if (!isRecord(value)) { - return 'unknown'; - } - - const type = value.type; - if (typeof type == 'string') { - return type; - } - - if (Array.isArray(type)) { - const firstString = type.find((entry) => typeof entry == 'string'); - if (firstString) { - return firstString; - } - } - - return 'unknown'; -} - -function toSqliteType(type: string): sync_rules.ExpressionType { - switch (type) { - case 'integer': - return sync_rules.ExpressionType.INTEGER; - case 'number': - return sync_rules.ExpressionType.REAL; - case 'boolean': - return sync_rules.ExpressionType.INTEGER; - case 'null': - return sync_rules.ExpressionType.NONE; - case 'array': - case 'object': - return sync_rules.ExpressionType.TEXT; - case 'string': - default: - return sync_rules.ExpressionType.TEXT; - } -} - -function isRecord(value: unknown): value is Record { - return typeof value == 'object' && value != null && !Array.isArray(value); -} diff --git a/modules/module-convex/src/common/convex-to-sqlite.ts b/modules/module-convex/src/common/convex-to-sqlite.ts new file mode 100644 index 000000000..e4c5bdd22 --- /dev/null +++ b/modules/module-convex/src/common/convex-to-sqlite.ts @@ -0,0 +1,225 @@ +import { + DatabaseInputValue, + DatabaseInputRow, + ExpressionType, + SqliteInputRow, + toSyncRulesRow +} from '@powersync/service-sync-rules'; +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; +import { ConvexRawDocument } from '../client/ConvexApiClient.js'; + +export function toSqliteInputRow(change: ConvexRawDocument, properties?: Record): SqliteInputRow { + const row: DatabaseInputRow = {}; + + for (const [key, value] of Object.entries(change)) { + if (key == '_table' || key == '_deleted') { + continue; + } + + row[key] = toConvexDatabaseValue(value, readConvexFieldType(properties?.[key])); + } + + return toSyncRulesRow(row); +} + +export function toExpressionTypeFromConvexType(type: string | undefined): ExpressionType { + switch (normalizeConvexType(type)) { + case 'int64': + return ExpressionType.INTEGER; + case 'float64': + return ExpressionType.REAL; + case 'boolean': + return ExpressionType.INTEGER; + case 'bytes': + return ExpressionType.BLOB; + case 'null': + return ExpressionType.NONE; + case 'array': + case 'object': + case 'record': + return ExpressionType.TEXT; + case 'id': + case 'string': + case 'unknown': + default: + return ExpressionType.TEXT; + } +} + +export function extractProperties(schema: Record) { + const direct = schema.properties; + if (isRecord(direct)) { + return direct; + } + + const nested = schema.schema?.properties; + if (isRecord(nested)) { + return nested; + } + + return {}; +} + +export function readConvexFieldType(value: unknown): string { + if (!isRecord(value)) { + return 'unknown'; + } + + const format = typeof value.format == 'string' ? normalizeConvexType(value.format) : null; + if (format == 'bytes' || format == 'id') { + return format; + } + + const contentEncoding = typeof value.contentEncoding == 'string' ? value.contentEncoding.toLowerCase() : null; + if (contentEncoding == 'base64') { + return 'bytes'; + } + + const directType = typeof value.type == 'string' ? value.type : null; + if (directType != null) { + return normalizeConvexType(directType); + } + + if (Array.isArray(value.type)) { + const firstString = value.type.find((entry) => typeof entry == 'string'); + if (typeof firstString == 'string') { + return normalizeConvexType(firstString); + } + } + + const alternateType = + typeof value.valueType == 'string' + ? value.valueType + : typeof value.fieldType == 'string' + ? value.fieldType + : typeof value.kind == 'string' + ? value.kind + : null; + if (alternateType != null) { + return normalizeConvexType(alternateType); + } + + return 'unknown'; +} + +function toConvexDatabaseValue(value: unknown, type: string): DatabaseInputValue { + switch (normalizeConvexType(type)) { + case 'bytes': + return toBytesValue(value); + case 'boolean': + if (typeof value == 'boolean') { + return value; + } + break; + default: + break; + } + + if (value == null) { + return null; + } + + if (typeof value == 'string') { + return value; + } + + if (typeof value == 'number') { + if (Number.isInteger(value)) { + return BigInt(value); + } + return value; + } + + if (typeof value == 'bigint') { + return value; + } + + if (typeof value == 'boolean') { + return value; + } + + if (value instanceof Uint8Array) { + return value; + } + + if (ArrayBuffer.isView(value)) { + return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); + } + + if (value instanceof ArrayBuffer) { + return new Uint8Array(value); + } + + if (Array.isArray(value) || typeof value == 'object') { + return value as DatabaseInputValue; + } + + return null; +} + +function toBytesValue(value: unknown): Uint8Array | null { + if (value == null) { + return null; + } + + if (value instanceof Uint8Array) { + return value; + } + + if (ArrayBuffer.isView(value)) { + return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); + } + + if (value instanceof ArrayBuffer) { + return new Uint8Array(value); + } + + if (typeof value != 'string') { + throw new ServiceError( + ErrorCode.PSYNC_S1004, + `Convex bytes value must be a base64 string or binary buffer, got ${typeof value}` + ); + } + + const normalized = value.replace(/\s+/g, ''); + const buffer = Buffer.from(normalized, 'base64'); + const canonical = buffer.toString('base64').replace(/=+$/g, ''); + if (normalized != '' && canonical != normalized.replace(/=+$/g, '')) { + throw new ServiceError(ErrorCode.PSYNC_S1004, 'Convex bytes value is not valid base64'); + } + + return new Uint8Array(buffer); +} + +function normalizeConvexType(type: string | undefined): string { + const normalized = type?.trim().toLowerCase(); + switch (normalized) { + case 'id': + case 'string': + case 'bytes': + case 'array': + case 'object': + case 'record': + case 'null': + return normalized; + case 'integer': + case 'int64': + return 'int64'; + case 'number': + case 'float': + case 'float64': + return 'float64'; + case 'bool': + case 'boolean': + return 'boolean'; + case 'bytea': + case 'blob': + return 'bytes'; + default: + return normalized ?? 'unknown'; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value == 'object' && value != null && !Array.isArray(value); +} diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index a67c12d12..2a4a5d592 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -15,7 +15,7 @@ import { SourceTable, storage } from '@powersync/service-core'; -import { HydratedSyncRules, SqliteInputRow, TablePattern } from '@powersync/service-sync-rules'; +import { HydratedSyncRules, TablePattern } from '@powersync/service-sync-rules'; import { ReplicationMetric } from '@powersync/service-types'; import { setTimeout as delay } from 'timers/promises'; import { @@ -26,6 +26,7 @@ import { } from '../client/ConvexApiClient.js'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { parseConvexLsn, toConvexLsn, ZERO_LSN } from '../common/ConvexLSN.js'; +import { extractProperties, toSqliteInputRow } from '../common/convex-to-sqlite.js'; import { ConvexConnectionManager } from './ConvexConnectionManager.js'; export interface ConvexStreamOptions { @@ -51,6 +52,7 @@ export class ConvexStream { private readonly relationCache = new RelationCache(getCacheIdentifier); private tableSchemaCache: ConvexTableSchema[] | null = null; + private tableSchemaPropertiesByName = new Map>(); private oldestUncommittedChange: Date | null = null; private lastKeepaliveAt = 0; @@ -302,6 +304,7 @@ export class ConvexStream { table: SourceTable, snapshotCursor: string ): Promise<{ table: SourceTable }> { + const tableProperties = this.getTableSchemaProperties(table.name); const snapshotProgress = decodeSnapshotProgressCursor(table.snapshotStatus?.lastKey); let pageCursor = snapshotProgress.cursor; let replicatedCount = table.snapshotStatus?.replicatedCount ?? 0; @@ -357,7 +360,7 @@ export class ConvexStream { continue; } - const row = this.toSqliteRow(rawDocument); + const row = this.toSqliteRow(rawDocument, tableProperties); await batch.save({ tag: SaveOperationTag.INSERT, sourceTable: latestTable, @@ -498,6 +501,8 @@ export class ConvexStream { return null; } + await this.getAllTableSchemas({ force: true }); + let table = await this.processTable(batch, descriptor); if (!table.snapshotComplete && table.syncAny) { this.logger.info(`New table discovered while streaming: [${table.qualifiedName}]`); @@ -580,7 +585,7 @@ export class ConvexStream { return true; } - const after = this.toSqliteRow(change); + const after = this.toSqliteRow(change, this.getTableSchemaProperties(table.name)); await batch.save({ tag: SaveOperationTag.UPDATE, sourceTable: table, @@ -593,17 +598,8 @@ export class ConvexStream { return true; } - private toSqliteRow(change: ConvexRawDocument) { - const row: SqliteInputRow = {}; - - for (const [key, value] of Object.entries(change)) { - if (key == '_table' || key == '_deleted') { - continue; - } - row[key] = toConvexSyncValue(value); - } - - return this.syncRules.applyRowContext(row); + private toSqliteRow(change: ConvexRawDocument, properties?: Record) { + return this.syncRules.applyRowContext(toSqliteInputRow(change, properties)); } private async getAllTableSchemas(options?: { force?: boolean }): Promise { @@ -613,9 +609,16 @@ export class ConvexStream { const schema = await this.connections.client.getJsonSchemas({ signal: this.abortSignal }); this.tableSchemaCache = schema.tables; + this.tableSchemaPropertiesByName = new Map( + schema.tables.map((table) => [table.tableName, extractProperties(table.schema)]) + ); return schema.tables; } + private getTableSchemaProperties(tableName: string): Record | undefined { + return this.tableSchemaPropertiesByName.get(tableName); + } + private touch() { if (performance.now() - this.lastTouchedAt < 1_000) { return; @@ -640,37 +643,6 @@ function readTableName(change: ConvexRawDocument): string | null { return table; } -function toConvexSyncValue(value: unknown): any { - if (value == null) { - return null; - } - - if (typeof value == 'string') { - return value; - } - - if (typeof value == 'number') { - if (Number.isInteger(value)) { - return BigInt(value); - } - return value; - } - - if (typeof value == 'bigint') { - return value; - } - - if (typeof value == 'boolean') { - return value ? 1n : 0n; - } - - if (Array.isArray(value) || typeof value == 'object') { - return JSON.stringify(value); - } - - return null; -} - interface ConvexSnapshotProgressCursor { cursor: string | null; finished: boolean; diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index 8bd40b294..1d00e9303 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -1,4 +1,4 @@ -import { SqlSyncRules } from '@powersync/service-sync-rules'; +import { ExpressionType, SqlSyncRules } from '@powersync/service-sync-rules'; import { describe, expect, it, vi } from 'vitest'; import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; import { toConvexLsn } from '@module/common/ConvexLSN.js'; @@ -26,7 +26,8 @@ function createAdapter() { schema: { properties: { _id: { type: 'string' }, - age: { type: 'integer' } + age: { type: 'integer' }, + avatar: { type: 'bytes' } } } } @@ -47,6 +48,10 @@ describe('ConvexRouteAPIAdapter', () => { const schema = await adapter.getConnectionSchema(); expect(schema[0]?.name).toBe('convex'); expect(schema[0]?.tables[0]?.name).toBe('users'); + expect(schema[0]?.tables[0]?.columns.find((column) => column.name == '_id')?.type).toBe('id'); + expect(schema[0]?.tables[0]?.columns.find((column) => column.name == 'avatar')?.sqlite_type).toBe( + ExpressionType.BLOB.typeFlags + ); await adapter.shutdown(); }); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 38b8d5320..d663c4150 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -215,6 +215,63 @@ describe('ConvexStream', () => { expect(context.commits.at(-1)).toBe(toConvexLsn('100')); }); + it('decodes bytes fields to Uint8Array during snapshot hydration', async () => { + const context = createFakeStorage(); + const abortController = new AbortController(); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [ + { + tableName: 'users', + schema: { + properties: { + avatar: { type: 'bytes' } + } + } + } + ], + raw: {} + }), + listSnapshot: async (options: any) => { + if (options?.tableName == null) { + return { + snapshot: '100', + cursor: null, + hasMore: false, + values: [] + }; + } + + return { + snapshot: '100', + cursor: null, + hasMore: false, + values: [{ _table: 'users', _id: 'u1', avatar: 'AQID' }] + }; + }, + getGlobalSnapshotCursor: async () => '100' + } + } as any + }); + + await stream.initReplication(); + + expect(context.saves).toHaveLength(1); + expect(context.saves[0]?.after.avatar).toEqual(Uint8Array.of(1, 2, 3)); + }); + it('resumes table snapshots from the persisted page cursor', async () => { const context = createFakeStorage({ snapshotLsn: toConvexLsn('200'), @@ -407,7 +464,7 @@ describe('ConvexStream', () => { expect(deltaCalls[0]?.tableName).toBeUndefined(); }); - it('snapshots a newly discovered wildcard-matched table inline without refreshing metadata', async () => { + it('refreshes metadata before snapshotting a newly discovered wildcard-matched table inline', async () => { const context = createFakeStorage({ snapshotDone: true, resumeFromLsn: toConvexLsn('100'), @@ -457,7 +514,7 @@ describe('ConvexStream', () => { await stream.streamChanges(); expect(calls).toBeGreaterThan(0); - expect(getJsonSchemas).toHaveBeenCalledTimes(1); + expect(getJsonSchemas).toHaveBeenCalledTimes(2); expect(listSnapshot).toHaveBeenCalledTimes(1); expect(listSnapshot).toHaveBeenCalledWith({ tableName: 'projects_archive', diff --git a/modules/module-convex/test/src/convex-to-sqlite.test.ts b/modules/module-convex/test/src/convex-to-sqlite.test.ts new file mode 100644 index 000000000..383d329a4 --- /dev/null +++ b/modules/module-convex/test/src/convex-to-sqlite.test.ts @@ -0,0 +1,61 @@ +import { + applyRowContext, + CompatibilityContext, + CompatibilityEdition, + ExpressionType, + isJsonValue +} from '@powersync/service-sync-rules'; +import { describe, expect, it } from 'vitest'; +import { + readConvexFieldType, + toExpressionTypeFromConvexType, + toSqliteInputRow +} from '@module/common/convex-to-sqlite.js'; + +const context = new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }); + +describe('convex-to-sqlite', () => { + it('maps Convex types to SQLite expression types', () => { + expect(toExpressionTypeFromConvexType('id')).toBe(ExpressionType.TEXT); + expect(toExpressionTypeFromConvexType('bytes')).toBe(ExpressionType.BLOB); + expect(toExpressionTypeFromConvexType('int64')).toBe(ExpressionType.INTEGER); + expect(toExpressionTypeFromConvexType('float64')).toBe(ExpressionType.REAL); + expect(toExpressionTypeFromConvexType('record')).toBe(ExpressionType.TEXT); + expect(toExpressionTypeFromConvexType('null')).toBe(ExpressionType.NONE); + }); + + it('detects bytes and id field types from schema metadata', () => { + expect(readConvexFieldType({ type: 'string', format: 'bytes' })).toBe('bytes'); + expect(readConvexFieldType({ type: 'string', format: 'id' })).toBe('id'); + expect(readConvexFieldType({ valueType: 'record' })).toBe('record'); + expect(readConvexFieldType({ contentEncoding: 'base64', type: 'string' })).toBe('bytes'); + }); + + it('decodes bytes to Uint8Array and keeps them out of JSON-compatible values', () => { + const row = applyRowContext( + toSqliteInputRow( + { + _id: 'doc1', + payload: 'AQID', + plain_text: 'AQID' + }, + { + _id: { type: 'id' }, + payload: { type: 'bytes' }, + plain_text: { type: 'string' } + } + ), + context + ); + + const payload = row.payload; + expect(row._id).toBe('doc1'); + expect(payload).toEqual(Uint8Array.of(1, 2, 3)); + expect(payload instanceof Uint8Array).toBe(true); + if (!(payload instanceof Uint8Array)) { + throw new Error('Expected payload to be Uint8Array'); + } + expect(isJsonValue(payload)).toBe(false); + expect(row.plain_text).toBe('AQID'); + }); +}); From 9b1d02f042b2b073ed44dec9d782f19f52ac5eb5 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Mar 2026 21:25:01 -0700 Subject: [PATCH 26/86] pnpmlock --- pnpm-lock.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 22bf665dd..a295dbe9e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -205,6 +205,9 @@ importers: '@powersync/service-core-tests': specifier: workspace:* version: link:../../packages/service-core-tests + '@powersync/service-module-mongodb-storage': + specifier: workspace:* + version: link:../module-mongodb-storage modules/module-core: dependencies: From 1831687b21e15d4b334687dc5fd5e09a46c09b6f Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Fri, 6 Mar 2026 21:43:10 -0700 Subject: [PATCH 27/86] upstream api change --- .../src/api/ConvexRouteAPIAdapter.ts | 4 +- .../src/replication/ConvexStream.ts | 2 +- .../test/src/ConvexRouteAPIAdapter.test.ts | 2 +- .../test/src/ConvexStream.test.ts | 4 +- modules/module-convex/test/src/util.ts | 42 +++++++++++++++---- 5 files changed, 39 insertions(+), 15 deletions(-) diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index dd47370dd..1daee5392 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -42,7 +42,7 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { async getDebugTablesInfo( tablePatterns: sync_rules.TablePattern[], - sqlSyncRules: sync_rules.SqlSyncRules + sqlSyncRules: sync_rules.SyncConfig ): Promise { const schema = await this.connectionManager.client.getJsonSchemas(); const tablesByName = new Map(schema.tables.map((table) => [table.tableName, table])); @@ -194,7 +194,7 @@ function createTableInfo(options: { const tableName = options.tableName ?? (options.tablePattern.isWildcard ? options.tablePattern.tablePrefix : options.tablePattern.name); const sourceTable = new SourceTable({ - id: 0, + id: tableName, connectionTag: options.connectionTag, objectId: tableName, schema: options.tablePattern.schema, diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 2a4a5d592..a044b58d1 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -408,7 +408,7 @@ export class ConvexStream { snapshotCursor: string ): Promise { const snapshotLsnValue = toConvexLsn(snapshotCursor); - const [doneTable] = await batch.markSnapshotDone([table], snapshotLsnValue); + const [doneTable] = await batch.markTableSnapshotDone([table], snapshotLsnValue); this.relationCache.update(doneTable); return doneTable; } diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index 1d00e9303..c8257daf7 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -71,7 +71,7 @@ bucket_definitions: } ); - const result = await adapter.getDebugTablesInfo(syncRules.getSourceTables(), syncRules); + const result = await adapter.getDebugTablesInfo(syncRules.config.getSourceTables(), syncRules.config); expect(result[0]?.table?.name).toBe('users'); await adapter.shutdown(); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index d663c4150..fc199d90b 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -40,7 +40,7 @@ function createFakeStorage(options?: { } const table = new SourceTable({ - id: nextTableId++, + id: `${nextTableId++}`, connectionTag: 'default', objectId: name, schema: 'convex', @@ -95,7 +95,7 @@ function createFakeStorage(options?: { resumeLsnUpdates.push(lsn); this.resumeFromLsn = lsn; }, - async markSnapshotDone(tables: SourceTable[], _lsn: string) { + async markTableSnapshotDone(tables: SourceTable[], _lsn: string) { for (const sourceTable of tables) { sourceTable.snapshotComplete = true; } diff --git a/modules/module-convex/test/src/util.ts b/modules/module-convex/test/src/util.ts index c38882187..6a4e2bf91 100644 --- a/modules/module-convex/test/src/util.ts +++ b/modules/module-convex/test/src/util.ts @@ -6,7 +6,8 @@ import { InternalOpId, OplogEntry, storage, - SyncRulesBucketStorage + SyncRulesBucketStorage, + updateSyncRulesFromYaml } from '@powersync/service-core'; import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; @@ -42,9 +43,11 @@ export class ConvexStreamTestContext { private _stream?: ConvexStream; private abortController = new AbortController(); private streamPromise?: Promise; + private syncRulesContent?: storage.PersistedSyncRulesContent; public storage?: SyncRulesBucketStorage; - static async open(factory: storage.TestStorageFactory) { + static async open(factoryOrConfig: storage.TestStorageFactory | storage.TestStorageConfig) { + const factory = typeof factoryOrConfig === 'function' ? factoryOrConfig : factoryOrConfig.factory; const f = await factory({}); const connectionManager = makeConvexConnectionManager(); return new ConvexStreamTestContext(f, connectionManager); @@ -77,8 +80,16 @@ export class ConvexStreamTestContext { return this.connectionManager.client; } + private requireSyncRulesContent(): storage.PersistedSyncRulesContent { + if (this.syncRulesContent == null) { + throw new Error('Call updateSyncRules() first'); + } + return this.syncRulesContent; + } + async updateSyncRules(content: string) { - const syncRules = await this.factory.updateSyncRules({ content, validate: true }); + const syncRules = await this.factory.updateSyncRules(updateSyncRulesFromYaml(content, { validate: true })); + this.syncRulesContent = syncRules; this.storage = this.factory.getInstance(syncRules); return this.storage!; } @@ -128,17 +139,18 @@ export class ConvexStreamTestContext { async getBucketData(bucket: string, options?: { timeout?: number }): Promise { const checkpoint = await this.waitForCheckpoint(options); - const map = new Map([[bucket, 0n]]); + const syncRules = this.requireSyncRulesContent(); + let request = test_utils.bucketRequest(syncRules, bucket, 0n); let data: OplogEntry[] = []; while (true) { - const batch = this.storage!.getBucketDataBatch(checkpoint, map); + const batch = this.storage!.getBucketDataBatch(checkpoint, [request]); const batches = await test_utils.fromAsync(batch); data = data.concat(batches[0]?.chunkData.data ?? []); if (batches.length == 0 || !batches[0]!.chunkData.has_more) { break; } - map.set(bucket, BigInt(batches[0]!.chunkData.next_after)); + request = test_utils.bucketRequest(syncRules, bucket, BigInt(batches[0]!.chunkData.next_after)); } return data; @@ -146,13 +158,25 @@ export class ConvexStreamTestContext { async getChecksums(buckets: string[], options?: { timeout?: number }) { const checkpoint = await this.waitForCheckpoint(options); - return this.storage!.getChecksums(checkpoint, buckets); + const syncRules = this.requireSyncRulesContent(); + return this.storage!.getChecksums( + checkpoint, + buckets.map((bucket) => { + const request = test_utils.bucketRequest(syncRules, bucket, 0n); + return { + bucket: request.bucket, + source: request.source + }; + }) + ); } async getChecksum(bucket: string, options?: { timeout?: number }) { const checkpoint = await this.waitForCheckpoint(options); - const map = await this.storage!.getChecksums(checkpoint, [bucket]); - return map.get(bucket); + const syncRules = this.requireSyncRulesContent(); + const request = test_utils.bucketRequest(syncRules, bucket, 0n); + const map = await this.storage!.getChecksums(checkpoint, [{ bucket: request.bucket, source: request.source }]); + return map.get(request.bucket); } /** From 96d9be99591e81034156266e63bccbe2882f19e9 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Mon, 9 Mar 2026 11:40:33 -0600 Subject: [PATCH 28/86] fix regression in snapshotting --- modules/module-convex/src/replication/ConvexStream.ts | 2 ++ modules/module-convex/test/src/ConvexStream.test.ts | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index a044b58d1..f944849a5 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -288,6 +288,8 @@ export class ConvexStream { await this.snapshotTable(batch, tableWithProgress, snapshotCursor); } + await batch.markAllSnapshotDone(snapshotLsnValue); + await batch.commit(snapshotLsnValue); this.logger.info(`Snapshot done. Need to replicate from ${snapshotLsnValue} for consistency.`); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index fc199d90b..3be8953b4 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -18,6 +18,7 @@ function createFakeStorage(options?: { const saves: any[] = []; const commits: string[] = []; const keepalives: string[] = []; + const allSnapshotDoneLsns: string[] = []; const resumeLsnUpdates: string[] = []; const tableProgressUpdates: any[] = []; @@ -95,6 +96,9 @@ function createFakeStorage(options?: { resumeLsnUpdates.push(lsn); this.resumeFromLsn = lsn; }, + async markAllSnapshotDone(lsn: string) { + allSnapshotDoneLsns.push(lsn); + }, async markTableSnapshotDone(tables: SourceTable[], _lsn: string) { for (const sourceTable of tables) { sourceTable.snapshotComplete = true; @@ -150,6 +154,7 @@ function createFakeStorage(options?: { saves, commits, keepalives, + allSnapshotDoneLsns, resumeLsnUpdates, tableProgressUpdates }; @@ -212,6 +217,7 @@ describe('ConvexStream', () => { expect(context.saves.length).toBe(1); expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); expect(context.resumeLsnUpdates.length).toBe(1); + expect(context.allSnapshotDoneLsns).toEqual([toConvexLsn('100')]); expect(context.commits.at(-1)).toBe(toConvexLsn('100')); }); From c0dc0d8b2661514a9cd0ea99a5d7ff9585691211 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Tue, 10 Mar 2026 13:14:59 -0600 Subject: [PATCH 29/86] fix test deps --- modules/module-convex/test/tsconfig.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/module-convex/test/tsconfig.json b/modules/module-convex/test/tsconfig.json index 7d296f30d..0e532cba3 100644 --- a/modules/module-convex/test/tsconfig.json +++ b/modules/module-convex/test/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../../tsconfig.base.json", + "extends": "../../../tsconfig.tests.json", "compilerOptions": { "rootDir": "src", "baseUrl": "./", @@ -20,6 +20,12 @@ }, { "path": "../../../packages/service-core-tests" + }, + { + "path": "../../module-mongodb-storage" + }, + { + "path": "../../module-postgres-storage" } ] -} +} \ No newline at end of file From 39b7cd97baa99793f016f9df9784285d7fc18c89 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Tue, 10 Mar 2026 13:23:27 -0600 Subject: [PATCH 30/86] add missing dep --- modules/module-convex/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json index ba31d3708..80e0767d3 100644 --- a/modules/module-convex/package.json +++ b/modules/module-convex/package.json @@ -37,6 +37,7 @@ }, "devDependencies": { "@powersync/service-core-tests": "workspace:*", - "@powersync/service-module-mongodb-storage": "workspace:*" + "@powersync/service-module-mongodb-storage": "workspace:*", + "@powersync/service-module-postgres-storage": "workspace:*" } } From 9f6f81cd7ee7ecec81d4a8587f56f2ddaff9669e Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Tue, 10 Mar 2026 13:26:46 -0600 Subject: [PATCH 31/86] update pnpm-lock.yaml --- pnpm-lock.yaml | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ee85fe73d..4714d0456 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -214,6 +214,9 @@ importers: '@powersync/service-module-mongodb-storage': specifier: workspace:* version: link:../module-mongodb-storage + '@powersync/service-module-postgres-storage': + specifier: workspace:* + version: link:../module-postgres-storage modules/module-core: dependencies: @@ -8797,14 +8800,6 @@ snapshots: optionalDependencies: vite: 7.1.5(@types/node@24.10.9)(yaml@2.8.2) - '@vitest/mocker@4.0.16(vite@7.1.5(@types/node@22.16.2)(yaml@2.4.5))': - dependencies: - '@vitest/spy': 4.0.16 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 7.1.5(@types/node@22.16.2)(yaml@2.4.5) - '@vitest/mocker@4.0.16(vite@7.1.5(@types/node@22.16.2)(yaml@2.8.2))': dependencies: '@vitest/spy': 4.0.16 @@ -8813,14 +8808,6 @@ snapshots: optionalDependencies: vite: 7.1.5(@types/node@22.16.2)(yaml@2.8.2) - '@vitest/mocker@4.0.16(vite@7.1.5(@types/node@24.10.9)(yaml@2.8.2))': - dependencies: - '@vitest/spy': 4.0.16 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 7.1.5(@types/node@24.10.9)(yaml@2.8.2) - '@vitest/pretty-format@3.2.4': dependencies: tinyrainbow: 2.0.0 @@ -8867,7 +8854,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vitest: 4.0.16(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.0.16)(yaml@2.4.5) + vitest: 4.0.16(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.0.16)(yaml@2.8.2) '@vitest/utils@3.2.4': dependencies: @@ -12408,7 +12395,7 @@ snapshots: vitest@4.0.16(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.0.16)(yaml@2.4.5): dependencies: '@vitest/expect': 4.0.16 - '@vitest/mocker': 4.0.16(vite@7.1.5(@types/node@22.16.2)(yaml@2.4.5)) + '@vitest/mocker': 4.0.16(vite@7.1.5(@types/node@22.16.2)(yaml@2.8.2)) '@vitest/pretty-format': 4.0.16 '@vitest/runner': 4.0.16 '@vitest/snapshot': 4.0.16 @@ -12486,7 +12473,7 @@ snapshots: vitest@4.0.16(@opentelemetry/api@1.9.0)(@types/node@24.10.9)(@vitest/ui@4.0.16)(yaml@2.8.2): dependencies: '@vitest/expect': 4.0.16 - '@vitest/mocker': 4.0.16(vite@7.1.5(@types/node@24.10.9)(yaml@2.8.2)) + '@vitest/mocker': 4.0.16(vite@7.1.5(@types/node@22.16.2)(yaml@2.8.2)) '@vitest/pretty-format': 4.0.16 '@vitest/runner': 4.0.16 '@vitest/snapshot': 4.0.16 From 8cddab753bdb09aad634ff5722c8c5133c388f98 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Tue, 10 Mar 2026 13:51:50 -0600 Subject: [PATCH 32/86] prettier -_- --- modules/module-convex/README.md | 32 +++-- .../src/api/ConvexRouteAPIAdapter.ts | 11 +- .../src/client/ConvexApiClient.ts | 13 +- .../module-convex/src/module/ConvexModule.ts | 4 +- .../src/replication/ConvexStream.ts | 5 +- .../test/src/ConvexApiClient.test.ts | 11 +- .../module-convex/test/src/ConvexLSN.test.ts | 4 +- .../test/src/ConvexStream.test.ts | 5 +- .../module-convex/test/src/slow_tests.test.ts | 126 +++++++++--------- modules/module-convex/test/tsconfig.json | 2 +- 10 files changed, 108 insertions(+), 105 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 931dee6f1..f784367d2 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -38,15 +38,15 @@ PowerSync requires a `powersync_checkpoints` table and a mutation function deplo In your Convex project's `convex/schema.ts`, add the `powersync_checkpoints` table: ```typescript -import { defineSchema, defineTable } from "convex/server"; -import { v } from "convex/values"; +import { defineSchema, defineTable } from 'convex/server'; +import { v } from 'convex/values'; export default defineSchema({ // ... your existing tables ... powersync_checkpoints: defineTable({ - last_updated: v.float64(), - }), + last_updated: v.float64() + }) }); ``` @@ -55,19 +55,19 @@ export default defineSchema({ Create `convex/powersync_checkpoints.ts`: ```typescript -import { mutation } from "./_generated/server"; +import { mutation } from './_generated/server'; export const createCheckpoint = mutation({ args: {}, handler: async (ctx) => { - const existing = await ctx.db.query("powersync_checkpoints").first(); + const existing = await ctx.db.query('powersync_checkpoints').first(); if (existing) { await ctx.db.patch(existing._id, { last_updated: Date.now() }); } else { - await ctx.db.insert("powersync_checkpoints", { last_updated: Date.now() }); + await ctx.db.insert('powersync_checkpoints', { last_updated: Date.now() }); } - }, + } }); ``` @@ -104,17 +104,18 @@ The mutation upserts a single row (insert on first call, patch thereafter), so t - Checkpoint marker tables are always ignored during snapshot and delta replication. - Marker-only pages trigger immediate keepalive checkpoint advancement (not minute-throttled), reducing write-checkpoint acknowledgement latency. - # Technical notes The content below is written in an agents.md style describing the behavior of `module-convex`. ## 1) Scope + - This module replicates Convex data into PowerSync bucket storage. - Source API is Convex Streaming Export (`json_schemas`, `list_snapshot`, `document_deltas`). - Initial scope is default Convex component only. ## 2) Canonical Behavior + - Initial replication: 1. Pin one **global snapshot** via `list_snapshot` **without** `tableName` and `cursor`. 2. Snapshot each selected sync-rules table with that fixed `snapshot`. @@ -127,6 +128,7 @@ The content below is written in an agents.md style describing the behavior of `m - If a table is first seen in a `document_deltas` page and matches sync rules, snapshot it inline at that page boundary and skip that table's delta rows from the same page, because the snapshot already includes them. ## 3) Hard Invariants (Do Not Break) + - `snapshot` is the consistency boundary; page `cursor` is pagination state. - All table snapshots in a run must use the same pinned `snapshot`; if response snapshot differs, fail fast. - On restart during initial replication: @@ -139,6 +141,7 @@ The content below is written in an agents.md style describing the behavior of `m - The overall system must ensure causal consistency of replicated data in bucket storage. ## 4) LSN and Cursor Rules + - Convex snapshot and delta cursors are always `i64` timestamps (serialized as decimal numeric strings in JSON). - Convex LSN helpers reject non-numeric cursors. - The `list_snapshot` pagination cursor is a separate JSON-serialized `{tablet, id}` string — it is pagination state, not a replication cursor. @@ -146,6 +149,7 @@ The content below is written in an agents.md style describing the behavior of `m - Persisted Convex LSNs must be canonical numeric cursor strings. ## 5) API Client Contract + - Auth header: `Authorization: Convex `. - Always request `format=json`. - Fallback path support: `/api/streaming_export/...` when `/api/...` returns `404`. @@ -158,16 +162,19 @@ The content below is written in an agents.md style describing the behavior of `m - non-retryable: malformed responses, auth/config issues. ## 6) Sync Rules and Connection Semantics + - Default schema is `convex`. - SQL literal reminder for sync rules: string literals must use single quotes (`'sally'`), not double quotes. ## 7) Schema Change Caveat + - Convex `json_schemas` does not provide a schema change token or revision cursor that can be checkpointed. - Current behavior uses `json_schemas` for discovery/debug, but does not continuously diff source schema versions. - Operational caveat: if Convex schema changes (tables or columns), developers must review and redeploy sync rules manually. - Future improvement: cache a canonicalized `json_schemas` hash, poll periodically, and raise diagnostics when schema drift is detected. ## 8) Datatype Mapping + - Current runtime mapping in stream writer: - `id` / `string` -> TEXT, - `int64` / integer `number` -> `BigInt` (SQLite INTEGER), @@ -175,11 +182,12 @@ The content below is written in an agents.md style describing the behavior of `m - `boolean` -> `1n`/`0n` (SQLite INTEGER), - `bytes` over the JSON API -> base64 `string` on the wire -> `Uint8Array` (SQLite BLOB), - `array` / `object` / `record` -> JSON TEXT. - NOTE: + NOTE: - Convex does not expose a native `Date` wire type here; timestamps arrive as `number` or `string`. - BLOB values are valid row values but are not valid bucket parameter values. ## 9) Checkpointing and Consistency + - `createReplicationHead` must: 1. resolve global head cursor, 2. write a Convex checkpoint marker via `POST /api/mutation` (calls `powersync_checkpoints:createCheckpoint`), @@ -193,6 +201,7 @@ The content below is written in an agents.md style describing the behavior of `m - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). ## 10) Logging Policy + - Keep logs high-signal and bounded. - Required snapshot logs: - pinned or reused global snapshot, @@ -201,12 +210,14 @@ The content below is written in an agents.md style describing the behavior of `m - Avoid noisy per-row debug logs unless behind explicit debug gating. ## 11) Known Non-Goals (For Now) + - Convex component targeting beyond default component. - A true push stream transport (module is polling deltas). - Read-only Convex deploy key model (Convex deploy key is full access; use network isolation and secret controls). - Cross-source transactional guarantees beyond Convex stream ordering. ## 12) Conformance Suite (Must Stay Green) + - Unit tests: - `test/src/ConvexApiClient.test.ts` - `test/src/ConvexCheckpoints.test.ts` @@ -227,6 +238,7 @@ The content below is written in an agents.md style describing the behavior of `m - no table-not-found due to schema/tag mismatch. ## 13) Change Checklist for Agents + - If changing snapshot/delta flow: - update `ConvexStream` tests first. - If changing cursor/LSN encoding: diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index 1daee5392..4fd07ef16 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -1,4 +1,10 @@ -import { api, ParseSyncRulesOptions, ReplicationHeadCallback, ReplicationLagOptions, SourceTable } from '@powersync/service-core'; +import { + api, + ParseSyncRulesOptions, + ReplicationHeadCallback, + ReplicationLagOptions, + SourceTable +} from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import { toConvexLsn } from '../common/ConvexLSN.js'; @@ -192,7 +198,8 @@ function createTableInfo(options: { errors?: service_types.ReplicationError[]; }) { const tableName = - options.tableName ?? (options.tablePattern.isWildcard ? options.tablePattern.tablePrefix : options.tablePattern.name); + options.tableName ?? + (options.tablePattern.isWildcard ? options.tablePattern.tablePrefix : options.tablePattern.name); const sourceTable = new SourceTable({ id: tableName, connectionTag: options.connectionTag, diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index e05517bd2..d83bb9259 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -52,13 +52,7 @@ export class ConvexApiError extends Error { readonly retryable: boolean; readonly body?: unknown; - constructor(options: { - message: string; - status?: number; - retryable: boolean; - body?: unknown; - cause?: unknown; - }) { + constructor(options: { message: string; status?: number; retryable: boolean; body?: unknown; cause?: unknown }) { super(options.message, options.cause !== undefined ? { cause: options.cause } : undefined); this.name = 'ConvexApiError'; this.status = options.status; @@ -312,8 +306,9 @@ export function isCursorExpiredError(error: unknown): boolean { const asString = `${error.message} ${JSON.stringify(error.body ?? {})}`.toLowerCase(); return ( - asString.includes('cursor') && (asString.includes('expired') || asString.includes('invalid')) - ) || (asString.includes('snapshot') && asString.includes('expired')); + (asString.includes('cursor') && (asString.includes('expired') || asString.includes('invalid'))) || + (asString.includes('snapshot') && asString.includes('expired')) + ); } function parseHasMore(payload: Record): boolean { diff --git a/modules/module-convex/src/module/ConvexModule.ts b/modules/module-convex/src/module/ConvexModule.ts index b873ed813..7cb5d3381 100644 --- a/modules/module-convex/src/module/ConvexModule.ts +++ b/modules/module-convex/src/module/ConvexModule.ts @@ -60,9 +60,7 @@ export class ConvexModule extends replication.ReplicationModule { + static async testConnection(normalizedConfig: types.ResolvedConvexConnectionConfig): Promise { const connectionManager = new ConvexConnectionManager(normalizedConfig); try { await connectionManager.client.getJsonSchemas(); diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index f944849a5..041c2453d 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -143,7 +143,10 @@ export class ConvexStream { }) .catch((error) => { if (isCursorExpiredError(error)) { - throw new ConvexCursorExpiredError('Convex cursor expired; initial replication restart required', error); + throw new ConvexCursorExpiredError( + 'Convex cursor expired; initial replication restart required', + error + ); } throw error; }); diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index 46553bd40..edfff7471 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -178,9 +178,9 @@ describe('ConvexApiClient', () => { }); it('creates write checkpoint markers via mutation', async () => { - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( - new Response(JSON.stringify({ status: 'success' }), { status: 200 }) - ); + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response(JSON.stringify({ status: 'success' }), { status: 200 })); const client = new ConvexApiClient(baseConfig); await client.createWriteCheckpointMarker(); @@ -199,10 +199,7 @@ describe('ConvexApiClient', () => { it('propagates checkpoint write errors directly (no fallback)', async () => { vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( - new Response( - JSON.stringify({ code: 'SomeError' }), - { status: 400 } - ) + new Response(JSON.stringify({ code: 'SomeError' }), { status: 400 }) ); const client = new ConvexApiClient(baseConfig); diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts index db6752aa9..9fc858bd7 100644 --- a/modules/module-convex/test/src/ConvexLSN.test.ts +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -35,8 +35,6 @@ describe('Convex cursor LSN helpers', () => { }); it('rejects non-numeric cursors', () => { - expect(() => toConvexLsn('{"tablet":"abc","id":"xyz"}')).toThrow( - 'Convex cursor is not a valid numeric timestamp' - ); + expect(() => toConvexLsn('{"tablet":"abc","id":"xyz"}')).toThrow('Convex cursor is not a valid numeric timestamp'); }); }); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 3be8953b4..72535594a 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -583,9 +583,6 @@ describe('ConvexStream', () => { expect(context.saves.length).toBe(0); expect(context.commits.length).toBe(0); - expect(context.keepalives).toEqual([ - toConvexLsn('101'), - toConvexLsn('102') - ]); + expect(context.keepalives).toEqual([toConvexLsn('101'), toConvexLsn('102')]); }); }); diff --git a/modules/module-convex/test/src/slow_tests.test.ts b/modules/module-convex/test/src/slow_tests.test.ts index 5f563dbb8..42f809488 100644 --- a/modules/module-convex/test/src/slow_tests.test.ts +++ b/modules/module-convex/test/src/slow_tests.test.ts @@ -3,94 +3,90 @@ import { describe, expect, test } from 'vitest'; import { ConvexStreamTestContext, INITIALIZED_MONGO_STORAGE_FACTORY, makeConvexConnectionManager } from './util.js'; import { env } from './env.js'; -describe.runIf(env.SLOW_TESTS && !!env.CONVEX_DEPLOY_KEY)( - 'convex slow tests', - { timeout: 120_000 }, - function () { - test('connects to Convex and lists table schemas', async () => { - await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); - const schemas = await context.client.getJsonSchemas(); - - expect(schemas.tables.length).toBeGreaterThan(0); - - const tableNames = schemas.tables.map((t) => t.tableName); - expect(tableNames).toContain('lists'); - }); - - test('list_snapshot returns data from Convex', async () => { - const mgr = makeConvexConnectionManager(); - const page = await mgr.client.listSnapshot({ tableName: 'lists' }); - - expect(page.snapshot).toBeDefined(); - // If this fails, seed data first: - // curl -X POST http://127.0.0.1:3210/api/mutation \ - // -H 'Content-Type: application/json' \ - // -H 'Authorization: Convex ' \ - // -d '{"path":"seed:seedLists","args":{"count":10},"format":"json"}' - expect(page.values.length).toBeGreaterThan(0); - - await mgr.end(); - }); - - test('snapshot replicates existing data into buckets', async () => { - await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); - - await context.updateSyncRules(` +describe.runIf(env.SLOW_TESTS && !!env.CONVEX_DEPLOY_KEY)('convex slow tests', { timeout: 120_000 }, function () { + test('connects to Convex and lists table schemas', async () => { + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); + const schemas = await context.client.getJsonSchemas(); + + expect(schemas.tables.length).toBeGreaterThan(0); + + const tableNames = schemas.tables.map((t) => t.tableName); + expect(tableNames).toContain('lists'); + }); + + test('list_snapshot returns data from Convex', async () => { + const mgr = makeConvexConnectionManager(); + const page = await mgr.client.listSnapshot({ tableName: 'lists' }); + + expect(page.snapshot).toBeDefined(); + // If this fails, seed data first: + // curl -X POST http://127.0.0.1:3210/api/mutation \ + // -H 'Content-Type: application/json' \ + // -H 'Authorization: Convex ' \ + // -d '{"path":"seed:seedLists","args":{"count":10},"format":"json"}' + expect(page.values.length).toBeGreaterThan(0); + + await mgr.end(); + }); + + test('snapshot replicates existing data into buckets', async () => { + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); + + await context.updateSyncRules(` bucket_definitions: global: data: - SELECT _id as id, name FROM "lists" `); - await context.replicateSnapshot(); - await context.markSnapshotQueryable(); + await context.replicateSnapshot(); + await context.markSnapshotQueryable(); - const checksum = await context.getChecksum('global[]'); - expect(checksum).toBeDefined(); - expect(checksum!.count).toBeGreaterThan(0); - }); + const checksum = await context.getChecksum('global[]'); + expect(checksum).toBeDefined(); + expect(checksum!.count).toBeGreaterThan(0); + }); - test('snapshot replicates with wildcard table pattern', async () => { - await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); - await context.updateSyncRules(` + test('snapshot replicates with wildcard table pattern', async () => { + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); + await context.updateSyncRules(` bucket_definitions: global: data: - SELECT _id as id, * FROM "lists" `); - await context.replicateSnapshot(); - await context.markSnapshotQueryable(); + await context.replicateSnapshot(); + await context.markSnapshotQueryable(); - const data = await context.getBucketData('global[]'); - expect(data.length).toBeGreaterThan(0); - }); + const data = await context.getBucketData('global[]'); + expect(data.length).toBeGreaterThan(0); + }); - test('delta streaming starts from snapshot cursor', async () => { - await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); - await context.updateSyncRules(` + test('delta streaming starts from snapshot cursor', async () => { + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); + await context.updateSyncRules(` bucket_definitions: global: data: - SELECT _id as id, name FROM "lists" `); - await context.replicateSnapshot(); + await context.replicateSnapshot(); - // Start streaming in background; it should poll deltas without error. - const streamPromise = context.startStreaming(); + // Start streaming in background; it should poll deltas without error. + const streamPromise = context.startStreaming(); - // Give delta polling a couple of cycles to confirm it doesn't crash. - await new Promise((resolve) => setTimeout(resolve, 2_000)); + // Give delta polling a couple of cycles to confirm it doesn't crash. + await new Promise((resolve) => setTimeout(resolve, 2_000)); - // Abort streaming gracefully. - context.abort(); - await streamPromise.catch(() => {}); + // Abort streaming gracefully. + context.abort(); + await streamPromise.catch(() => {}); - // Checkpoint should still be valid after streaming ran. - const checksum = await context.getChecksum('global[]'); - expect(checksum).toBeDefined(); - expect(checksum!.count).toBeGreaterThan(0); - }); - } -); + // Checkpoint should still be valid after streaming ran. + const checksum = await context.getChecksum('global[]'); + expect(checksum).toBeDefined(); + expect(checksum!.count).toBeGreaterThan(0); + }); +}); diff --git a/modules/module-convex/test/tsconfig.json b/modules/module-convex/test/tsconfig.json index 0e532cba3..e89f5c03b 100644 --- a/modules/module-convex/test/tsconfig.json +++ b/modules/module-convex/test/tsconfig.json @@ -28,4 +28,4 @@ "path": "../../module-postgres-storage" } ] -} \ No newline at end of file +} From f6e245d45afbefaa43bc794b27cb4231546bfb37 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Tue, 10 Mar 2026 21:36:37 -0600 Subject: [PATCH 33/86] README cleanup --- modules/module-convex/README.md | 187 +++--------------- .../src/api/ConvexRouteAPIAdapter.ts | 2 +- 2 files changed, 30 insertions(+), 159 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index f784367d2..7fde2ed8a 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -9,100 +9,15 @@ replication: connections: - type: convex id: default - tag: default deployment_url: https://.convex.cloud deploy_key: polling_interval_ms: 1000 + request_timeout_ms: 30000 ``` ## Manual smoke test -1. Start PowerSync with a Convex source config and valid sync rules that reference `convex.`. -2. Confirm diagnostics reports the Convex source as connected. -3. Verify initial snapshot data appears in the expected bucket(s). -4. Insert/update/delete rows in Convex and verify bucket changes are replicated. - -## Snapshot behavior - -- Initial replication pins a global Convex snapshot boundary using `list_snapshot` without `tableName`. -- Table hydration then runs per selected sync-rule table using that fixed snapshot boundary. -- Each table snapshot starts from the first page when there is no persisted progress, and otherwise resumes from the stored page cursor. -- If initial replication is interrupted, restart resumes from the stored snapshot boundary and any persisted per-table snapshot progress. - -## Convex Source Setup - -PowerSync requires a `powersync_checkpoints` table and a mutation function deployed to your Convex project. This is analogous to setting up a replication publication on Postgres — it enables PowerSync to write checkpoint markers that ensure causal consistency. - -### 1. Add the table to your schema - -In your Convex project's `convex/schema.ts`, add the `powersync_checkpoints` table: - -```typescript -import { defineSchema, defineTable } from 'convex/server'; -import { v } from 'convex/values'; - -export default defineSchema({ - // ... your existing tables ... - - powersync_checkpoints: defineTable({ - last_updated: v.float64() - }) -}); -``` - -### 2. Add the checkpoint mutation - -Create `convex/powersync_checkpoints.ts`: - -```typescript -import { mutation } from './_generated/server'; - -export const createCheckpoint = mutation({ - args: {}, - handler: async (ctx) => { - const existing = await ctx.db.query('powersync_checkpoints').first(); - - if (existing) { - await ctx.db.patch(existing._id, { last_updated: Date.now() }); - } else { - await ctx.db.insert('powersync_checkpoints', { last_updated: Date.now() }); - } - } -}); -``` - -### 3. Deploy - -```bash -npx convex dev --once -``` - -### How it works - -PowerSync calls this mutation via the [Convex HTTP API](https://docs.convex.dev/http-api): - -``` -POST /api/mutation -Authorization: Convex -Content-Type: application/json - -{ - "path": "powersync_checkpoints:createCheckpoint", - "args": {}, - "format": "json" -} -``` - -The mutation upserts a single row (insert on first call, patch thereafter), so the table stays bounded to one row. This write generates a delta event that PowerSync observes to confirm write-checkpoint consistency. - -> **Note:** The `powersync_checkpoints` table is automatically excluded from sync rules — it is never replicated to client devices. - -## Write checkpoint behavior - -- Route write checkpoints create a source marker row in `powersync_checkpoints` via the deployed mutation. -- Delta polling is global, so marker rows are observed even when not referenced in sync rules. -- Checkpoint marker tables are always ignored during snapshot and delta replication. -- Marker-only pages trigger immediate keepalive checkpoint advancement (not minute-throttled), reducing write-checkpoint acknowledgement latency. +1. Simplest is to run the convex demo in the self-host-demo [repo](https://github.com/powersync-ja/self-host-demo) # Technical notes @@ -111,21 +26,21 @@ The content below is written in an agents.md style describing the behavior of `m ## 1) Scope - This module replicates Convex data into PowerSync bucket storage. -- Source API is Convex Streaming Export (`json_schemas`, `list_snapshot`, `document_deltas`). -- Initial scope is default Convex component only. +- Source APIs used are Convex Streaming Export: (`json_schemas`, `list_snapshot`, `document_deltas`). +- Initial scope is default Convex component only, but we could consider support for custom components in the future if we can figure out consistency. ## 2) Canonical Behavior - Initial replication: - 1. Pin one **global snapshot** via `list_snapshot` **without** `tableName` and `cursor`. - 2. Snapshot each selected sync-rules table with that fixed `snapshot`. - 3. First per-table snapshot call omits `cursor`; pagination cursor is only for later pages in the same run. - 4. Commit snapshot LSN, then switch to deltas. + - Initial replication pins a global Convex snapshot boundary using `list_snapshot`. If this is omitted, it provides the global snapshot boundary [ref](https://docs.convex.dev/streaming-export-api#get-apilist_snapshot). + - Snapshot each selected Sync Streams table with that fixed `snapshot`. + - First per-table snapshot call omits `cursor`; pagination cursor is only for later pages in the same run. + - Commit snapshot LSN, then switch to deltas. - Streaming replication: - Start from persisted resume LSN. - - Poll `document_deltas`. - - Always stream globally (no `tableName` filter), then filter locally by selected sync-rules tables. - - If a table is first seen in a `document_deltas` page and matches sync rules, snapshot it inline at that page boundary and skip that table's delta rows from the same page, because the snapshot already includes them. + - Poll `document_deltas` using frequency configured in `polling_interval_ms` + - Always stream globally (no `tableName` filter), then filter locally by selected Sync Streams tables. + - If a table is first seen in a `document_deltas` page and matches Sync Streams, snapshot it inline at that page boundary and skip that table's delta rows from the same page, because the snapshot already includes them. ## 3) Hard Invariants (Do Not Break) @@ -143,9 +58,7 @@ The content below is written in an agents.md style describing the behavior of `m ## 4) LSN and Cursor Rules - Convex snapshot and delta cursors are always `i64` timestamps (serialized as decimal numeric strings in JSON). -- Convex LSN helpers reject non-numeric cursors. - The `list_snapshot` pagination cursor is a separate JSON-serialized `{tablet, id}` string — it is pagination state, not a replication cursor. -- Persist LSNs as the raw decimal cursor string returned by Convex. - Persisted Convex LSNs must be canonical numeric cursor strings. ## 5) API Client Contract @@ -161,29 +74,35 @@ The content below is written in an agents.md style describing the behavior of `m - retryable: network, timeout, 429, 5xx. - non-retryable: malformed responses, auth/config issues. -## 6) Sync Rules and Connection Semantics +## 6) Sync Streams and Connection Semantics - Default schema is `convex`. -- SQL literal reminder for sync rules: string literals must use single quotes (`'sally'`), not double quotes. ## 7) Schema Change Caveat - Convex `json_schemas` does not provide a schema change token or revision cursor that can be checkpointed. - Current behavior uses `json_schemas` for discovery/debug, but does not continuously diff source schema versions. -- Operational caveat: if Convex schema changes (tables or columns), developers must review and redeploy sync rules manually. +- Operational caveat: if Convex schema changes (tables or columns), developers must review and redeploy Sync Streams manually. - Future improvement: cache a canonicalized `json_schemas` hash, poll periodically, and raise diagnostics when schema drift is detected. ## 8) Datatype Mapping - Current runtime mapping in stream writer: - - `id` / `string` -> TEXT, - - `int64` / integer `number` -> `BigInt` (SQLite INTEGER), - - `float64` / non-integer `number` -> REAL, - - `boolean` -> `1n`/`0n` (SQLite INTEGER), - - `bytes` over the JSON API -> base64 `string` on the wire -> `Uint8Array` (SQLite BLOB), - - `array` / `object` / `record` -> JSON TEXT. - NOTE: - - Convex does not expose a native `Date` wire type here; timestamps arrive as `number` or `string`. + +| Convex Type | TS/JS Type | SQLite type | +| ----------- | ----------- | --------------------------------- | +| Id | string | text | +| Null | null | null | +| Int64 | bigint | integer | +| Float64 | number | real | +| Boolean | boolean | Up to developer - string or number | +| String | string | text | +| Bytes | ArrayBuffer | text | +| Array | Array | text | +| Object | Object | text | +| Record | Record | text | + + - Convex does not expose a native `Date` wire type; timestamps arrive as `number` or `string`. - BLOB values are valid row values but are not valid bucket parameter values. ## 9) Checkpointing and Consistency @@ -198,52 +117,4 @@ The content below is written in an agents.md style describing the behavior of `m - The developer must deploy the `powersync_checkpoints` schema and mutation to their Convex project (see README). - Stream handling requirement: - checkpoint marker tables must always be excluded from replicated source tables and ignored in delta row application. - - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). - -## 10) Logging Policy - -- Keep logs high-signal and bounded. -- Required snapshot logs: - - pinned or reused global snapshot, - - table snapshot start, - - snapshot completion and resume LSN. -- Avoid noisy per-row debug logs unless behind explicit debug gating. - -## 11) Known Non-Goals (For Now) - -- Convex component targeting beyond default component. -- A true push stream transport (module is polling deltas). -- Read-only Convex deploy key model (Convex deploy key is full access; use network isolation and secret controls). -- Cross-source transactional guarantees beyond Convex stream ordering. - -## 12) Conformance Suite (Must Stay Green) - -- Unit tests: - - `test/src/ConvexApiClient.test.ts` - - `test/src/ConvexCheckpoints.test.ts` - - `test/src/ConvexLSN.test.ts` - - `test/src/ConvexRouteAPIAdapter.test.ts` - - `test/src/ConvexStream.test.ts` - - `test/src/types.test.ts` -- Required assertions include: - - global snapshot pin call is unfiltered, - - first per-table snapshot call omits cursor, - - pagination cursor used only for subsequent pages, - - snapshot mismatch fails fast, - - route adapter head resolves global snapshot cursor and writes a source marker, - - marker-only pages advance LSN via immediate keepalive. -- Manual smoke test (self-host acceptable): - - initial snapshot visible in buckets, - - inserts/updates/deletes propagate via deltas, - - no table-not-found due to schema/tag mismatch. - -## 13) Change Checklist for Agents - -- If changing snapshot/delta flow: - - update `ConvexStream` tests first. -- If changing cursor/LSN encoding: - - update the Convex LSN helper tests and any format-compat coverage. -- If changing API response parsing: - - include self-host payload fixtures in `ConvexApiClient` tests. -- If changing public behavior: - - update `README.md` and this `AGENTS.md` in the same PR. + - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). \ No newline at end of file diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index 4fd07ef16..2d7095e03 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -80,7 +80,7 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { const matchedTableNames = [...tablesByName.keys()] .filter((name) => { - //Convex doesn't support user-defined schemas, so this is more a forwards compatibility check for when multiple connections are supported + //Convex doesn't support user-defined schemas, so this is more of a forwards compatibility check for when multiple connections are supported if (tablePattern.schema != this.connectionManager.schema) { return false; } From b6c623b773a59cce0fd69d387e1250cda86390b1 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Wed, 11 Mar 2026 15:20:34 -0600 Subject: [PATCH 34/86] stop using deprecated startBatch() --- .../src/replication/ConvexStream.ts | 269 +++++++++--------- 1 file changed, 130 insertions(+), 139 deletions(-) diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 041c2453d..366a37623 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -116,117 +116,111 @@ export class ConvexStream { } async streamChanges() { - await this.storage.startBatch( - { - logger: this.logger, - zeroLSN: ZERO_LSN, - defaultSchema: this.defaultSchema, - storeCurrentData: false, - skipExistingRows: false - }, - async (batch) => { - let resumeFromLsn = batch.resumeFromLsn; - if (resumeFromLsn == null) { - throw new ReplicationAssertionError(`No LSN found to resume replication from.`); - } + await using batch = await this.storage.createWriter({ + logger: this.logger, + zeroLSN: ZERO_LSN, + defaultSchema: this.defaultSchema, + storeCurrentData: false, //convex currently has a hard document limit of 1MB per document + skipExistingRows: false + }); - // Resolve source tables up-front to warm table metadata and sync-rule matching. - await this.resolveAllSourceTables(batch); + let resumeFromLsn = batch.resumeFromLsn; + if (resumeFromLsn == null) { + throw new ReplicationAssertionError(`No LSN found to resume replication from.`); + } - let cursor = parseConvexLsn(resumeFromLsn); + // Resolve source tables up-front to warm table metadata and sync-rule matching. + await this.resolveAllSourceTables(batch); - while (!this.abortSignal.aborted) { - const page = await this.connections.client - .documentDeltas({ - cursor, - signal: this.abortSignal - }) - .catch((error) => { - if (isCursorExpiredError(error)) { - throw new ConvexCursorExpiredError( - 'Convex cursor expired; initial replication restart required', - error - ); - } - throw error; - }); - - const nextCursor = page.cursor; - const pageLsn = toConvexLsn(nextCursor); - - let changesInPage = 0; - let sawCheckpointMarker = false; - const snapshottedTablesInPage = new Set(); - for (const change of page.values) { - if (this.abortSignal.aborted) { - throw new ReplicationAbortedError('Replication interrupted'); - } - - const tableName = readTableName(change); - if (tableName == null) { - continue; - } - - if (isConvexCheckpointTable(tableName)) { - sawCheckpointMarker = true; - continue; - } - - const table = await this.getOrResolveTable(batch, tableName, nextCursor, snapshottedTablesInPage); - if (table == null || !table.syncAny) { - continue; - } - if (snapshottedTablesInPage.has(tableName)) { - continue; - } - - const changed = await this.writeChange(batch, table, change); - if (!changed) { - continue; - } - - changesInPage += 1; - if (this.oldestUncommittedChange == null) { - this.oldestUncommittedChange = new Date(); - } - } + let cursor = parseConvexLsn(resumeFromLsn); - if (changesInPage > 0) { - const didCommit = await batch.commit(pageLsn, { - createEmptyCheckpoints: false, - oldestUncommittedChange: this.oldestUncommittedChange - }); - - if (didCommit) { - this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); - this.oldestUncommittedChange = null; - this.isStartingReplication = false; - } - } else if (sawCheckpointMarker) { - await batch.keepalive(pageLsn); - this.lastKeepaliveAt = Date.now(); - this.isStartingReplication = false; - } else if (nextCursor != cursor && Date.now() - this.lastKeepaliveAt > 60_000) { - await batch.keepalive(pageLsn); - this.lastKeepaliveAt = Date.now(); - this.isStartingReplication = false; + while (!this.abortSignal.aborted) { + const page = await this.connections.client + .documentDeltas({ + cursor, + signal: this.abortSignal + }) + .catch((error) => { + if (isCursorExpiredError(error)) { + throw new ConvexCursorExpiredError('Convex cursor expired; initial replication restart required', error); } + throw error; + }); - cursor = nextCursor; + const nextCursor = page.cursor; + const pageLsn = toConvexLsn(nextCursor); - if (!page.hasMore) { - await delay(this.pollingIntervalMs, undefined, { signal: this.abortSignal }).catch((error) => { - if (this.abortSignal.aborted) { - return; - } - throw error; - }); - } + let changesInPage = 0; + let sawCheckpointMarker = false; + const snapshottedTablesInPage = new Set(); + for (const change of page.values) { + if (this.abortSignal.aborted) { + throw new ReplicationAbortedError('Replication interrupted'); + } - this.touch(); + const tableName = readTableName(change); + if (tableName == null) { + continue; + } + + if (isConvexCheckpointTable(tableName)) { + sawCheckpointMarker = true; + continue; + } + + const table = await this.getOrResolveTable(batch, tableName, nextCursor, snapshottedTablesInPage); + if (table == null || !table.syncAny) { + continue; + } + if (snapshottedTablesInPage.has(tableName)) { + continue; + } + + const changed = await this.writeChange(batch, table, change); + if (!changed) { + continue; + } + + changesInPage += 1; + if (this.oldestUncommittedChange == null) { + this.oldestUncommittedChange = new Date(); } } - ); + + if (changesInPage > 0) { + const didCommit = await batch.commit(pageLsn, { + createEmptyCheckpoints: false, + oldestUncommittedChange: this.oldestUncommittedChange + }); + + if (didCommit) { + this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); + this.oldestUncommittedChange = null; + this.isStartingReplication = false; + } + } else if (sawCheckpointMarker) { + await batch.keepalive(pageLsn); + this.lastKeepaliveAt = Date.now(); + this.isStartingReplication = false; + } else if (nextCursor != cursor && Date.now() - this.lastKeepaliveAt > 60_000) { + await batch.keepalive(pageLsn); + this.lastKeepaliveAt = Date.now(); + this.isStartingReplication = false; + } + + cursor = nextCursor; + + if (!page.hasMore) { + await delay(this.pollingIntervalMs, undefined, { signal: this.abortSignal }).catch((error) => { + if (this.abortSignal.aborted) { + return; + } + throw error; + }); + } + + this.touch(); + } } async getReplicationLagMillis(): Promise { @@ -257,50 +251,47 @@ export class ConvexStream { } private async initialReplication(snapshotLsn: string | null) { - const flushResult = await this.storage.startBatch( - { - logger: this.logger, - zeroLSN: ZERO_LSN, - defaultSchema: this.defaultSchema, - storeCurrentData: false, - skipExistingRows: true - }, - async (batch) => { - const snapshotCursor = await this.resolveSnapshotBoundary(snapshotLsn); - const snapshotLsnValue = toConvexLsn(snapshotCursor); - await batch.setResumeLsn(snapshotLsnValue); - - const sourceTables = await this.resolveAllSourceTables(batch); - - for (const sourceTable of sourceTables) { - if (sourceTable.snapshotComplete) { - this.logger.info(`Skipping table [${sourceTable.qualifiedName}] - snapshot already done.`); - continue; - } - - const tableWithProgress = - sourceTable.snapshotStatus == null - ? await batch.updateTableProgress(sourceTable, { - totalEstimatedCount: -1, - replicatedCount: 0, - lastKey: null - }) - : sourceTable; - this.relationCache.update(tableWithProgress); - - await this.snapshotTable(batch, tableWithProgress, snapshotCursor); - } + await using batch = await this.storage.createWriter({ + logger: this.logger, + zeroLSN: ZERO_LSN, + defaultSchema: this.defaultSchema, + storeCurrentData: false, + skipExistingRows: true + }); - await batch.markAllSnapshotDone(snapshotLsnValue); + const snapshotCursor = await this.resolveSnapshotBoundary(snapshotLsn); + const snapshotLsnValue = toConvexLsn(snapshotCursor); + await batch.setResumeLsn(snapshotLsnValue); - await batch.commit(snapshotLsnValue); + const sourceTables = await this.resolveAllSourceTables(batch); - this.logger.info(`Snapshot done. Need to replicate from ${snapshotLsnValue} for consistency.`); + for (const sourceTable of sourceTables) { + if (sourceTable.snapshotComplete) { + this.logger.info(`Skipping table [${sourceTable.qualifiedName}] - snapshot already done.`); + continue; } - ); + + const tableWithProgress = + sourceTable.snapshotStatus == null + ? await batch.updateTableProgress(sourceTable, { + totalEstimatedCount: -1, + replicatedCount: 0, + lastKey: null + }) + : sourceTable; + this.relationCache.update(tableWithProgress); + + await this.snapshotTable(batch, tableWithProgress, snapshotCursor); + } + + await batch.markAllSnapshotDone(snapshotLsnValue); + + await batch.commit(snapshotLsnValue); + + this.logger.info(`Snapshot done. Need to replicate from ${snapshotLsnValue} for consistency.`); return { - lastOpId: flushResult?.flushed_op + lastOpId: batch.last_flushed_op }; } From 682f9a05c87fc6f60a478613cda41e7344dac77d Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Wed, 11 Mar 2026 15:20:51 -0600 Subject: [PATCH 35/86] README updates --- modules/module-convex/README.md | 54 +++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 7fde2ed8a..befcb3c79 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -26,7 +26,7 @@ The content below is written in an agents.md style describing the behavior of `m ## 1) Scope - This module replicates Convex data into PowerSync bucket storage. -- Source APIs used are Convex Streaming Export: (`json_schemas`, `list_snapshot`, `document_deltas`). +- Source APIs used are Convex [Streaming Export](https://docs.convex.dev/streaming-export-api): (`json_schemas`, `list_snapshot`, `document_deltas`). - Initial scope is default Convex component only, but we could consider support for custom components in the future if we can figure out consistency. ## 2) Canonical Behavior @@ -38,7 +38,7 @@ The content below is written in an agents.md style describing the behavior of `m - Commit snapshot LSN, then switch to deltas. - Streaming replication: - Start from persisted resume LSN. - - Poll `document_deltas` using frequency configured in `polling_interval_ms` + - Poll `document_deltas` using frequency configured in `polling_interval_ms` - Always stream globally (no `tableName` filter), then filter locally by selected Sync Streams tables. - If a table is first seen in a `document_deltas` page and matches Sync Streams, snapshot it inline at that page boundary and skip that table's delta rows from the same page, because the snapshot already includes them. @@ -74,38 +74,34 @@ The content below is written in an agents.md style describing the behavior of `m - retryable: network, timeout, 429, 5xx. - non-retryable: malformed responses, auth/config issues. -## 6) Sync Streams and Connection Semantics - -- Default schema is `convex`. - -## 7) Schema Change Caveat +## 6) Schema Change Caveat - Convex `json_schemas` does not provide a schema change token or revision cursor that can be checkpointed. - Current behavior uses `json_schemas` for discovery/debug, but does not continuously diff source schema versions. - Operational caveat: if Convex schema changes (tables or columns), developers must review and redeploy Sync Streams manually. - Future improvement: cache a canonicalized `json_schemas` hash, poll periodically, and raise diagnostics when schema drift is detected. -## 8) Datatype Mapping +## 7) Datatype Mapping - Current runtime mapping in stream writer: -| Convex Type | TS/JS Type | SQLite type | -| ----------- | ----------- | --------------------------------- | -| Id | string | text | -| Null | null | null | -| Int64 | bigint | integer | -| Float64 | number | real | +| Convex Type | TS/JS Type | SQLite type | +| ----------- | ----------- | ---------------------------------- | +| Id | string | text | +| Null | null | null | +| Int64 | bigint | integer | +| Float64 | number | real | | Boolean | boolean | Up to developer - string or number | -| String | string | text | -| Bytes | ArrayBuffer | text | -| Array | Array | text | -| Object | Object | text | -| Record | Record | text | +| String | string | text | +| Bytes | ArrayBuffer | text | +| Array | Array | text | +| Object | Object | text | +| Record | Record | text | - - Convex does not expose a native `Date` wire type; timestamps arrive as `number` or `string`. - - BLOB values are valid row values but are not valid bucket parameter values. +- Convex does not expose a native `Date` wire type; timestamps arrive as `number` or `string`. +- BLOB values are valid row values but are not valid bucket parameter values. -## 9) Checkpointing and Consistency +## 8) Checkpointing and Consistency - `createReplicationHead` must: 1. resolve global head cursor, @@ -117,4 +113,16 @@ The content below is written in an agents.md style describing the behavior of `m - The developer must deploy the `powersync_checkpoints` schema and mutation to their Convex project (see README). - Stream handling requirement: - checkpoint marker tables must always be excluded from replicated source tables and ignored in delta row application. - - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). \ No newline at end of file + - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). + +## 9) Convex-specific notes + +- The default schema is `convex` + +- **Mutation Transaction Atomicity in** `document_deltas` + - The `cursor` in `/api/document_deltas` is a Convex commit **timestamp** (`i64`), not a per-operation counter. + - Every Convex mutation is an ACID transaction that commits with a single timestamp; all writes within that mutation share the same `_ts` value in the delta stream. + - Therefore, the cursor advances **once per mutation**, not once per individual CRUD operation inside it. + - Example: a mutation that deletes 5 documents and updates 3 produces 8 entries in `document_deltas`, all with identical `_ts`. + - The Convex backend enforces this by never splitting a page mid-timestamp: when the row limit is reached mid-transaction, the page extends until all rows at that `_ts` are included before stopping. + - Consequence for replication: all writes from a single mutation always appear in the same `document_deltas` page and are committed to bucket storage atomically as one batch. From bcfcfdee457616d08a3eff40ba7d8fe190b372fc Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Mon, 16 Mar 2026 12:55:18 -0600 Subject: [PATCH 36/86] update fake storage in tests for new createWriter() storage API --- modules/module-convex/test/src/ConvexStream.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 72535594a..83dc98a05 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -69,6 +69,7 @@ function createFakeStorage(options?: { lastCheckpointLsn: null, resumeFromLsn: options?.resumeFromLsn ?? null, noCheckpointBeforeLsn: ZERO_LSN, + async [Symbol.asyncDispose]() {}, async save(record: any) { saves.push(record); return null; @@ -139,6 +140,7 @@ function createFakeStorage(options?: { table: getOrCreateTable(entity_descriptor.name), dropTables: [] })), + createWriter: vi.fn(async (_options: any) => batch), startBatch: vi.fn(async (_options: any, callback: (batch: any) => Promise) => { await callback(batch); return { flushed_op: 1n }; From 776f35de346477e78e4e03786ce5adf1f5ab0dcb Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Tue, 17 Mar 2026 15:57:14 -0600 Subject: [PATCH 37/86] document assumptions about convex LSN length also update tests to be more realistic --- modules/module-convex/README.md | 2 +- modules/module-convex/src/common/ConvexLSN.ts | 12 +++- .../test/src/ConvexApiClient.test.ts | 15 ++--- .../module-convex/test/src/ConvexLSN.test.ts | 12 +++- .../test/src/ConvexRouteAPIAdapter.test.ts | 8 ++- .../test/src/ConvexStream.test.ts | 61 +++++++++++-------- 6 files changed, 69 insertions(+), 41 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index befcb3c79..4a98198e2 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -59,7 +59,7 @@ The content below is written in an agents.md style describing the behavior of `m - Convex snapshot and delta cursors are always `i64` timestamps (serialized as decimal numeric strings in JSON). - The `list_snapshot` pagination cursor is a separate JSON-serialized `{tablet, id}` string — it is pagination state, not a replication cursor. -- Persisted Convex LSNs must be canonical numeric cursor strings. +- Persisted Convex LSNs must be canonical 19-digit numeric cursor strings. `ZERO_LSN = "0"` remains the internal sentinel. ## 5) API Client Contract diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index c7b43a7e2..4a770d810 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -1,4 +1,6 @@ export const ZERO_LSN = '0'; +// Convex replication cursors are fixed-width decimal timestamps. +const CONVEX_TIMESTAMP_DIGITS = 19; export function parseConvexLsn(lsn: string): string { return toConvexLsn(lsn); @@ -19,7 +21,15 @@ function assertValidConvexLsn(cursor: string) { throw new Error(`Convex cursor is not a valid numeric timestamp: ${cursor}`); } - if (cursor.length > 1 && cursor.startsWith('0')) { + if (cursor == ZERO_LSN) { + return; + } + + if (cursor.startsWith('0')) { throw new Error(`Convex cursor is not a canonical numeric timestamp: ${cursor}`); } + + if (cursor.length != CONVEX_TIMESTAMP_DIGITS) { + throw new Error(`Convex cursor is not a 19-digit numeric timestamp: ${cursor}`); + } } diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index edfff7471..329d1a667 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -8,6 +8,7 @@ const baseConfig = normalizeConnectionConfig({ deployment_url: 'https://example.convex.cloud', deploy_key: 'test-key' }); +const SNAPSHOT_CURSOR = '1770335566197683000'; describe('ConvexApiClient', () => { afterEach(() => { @@ -44,8 +45,8 @@ describe('ConvexApiClient', () => { .mockResolvedValueOnce( new Response( JSON.stringify({ - snapshot: '100', - cursor: '100', + snapshot: SNAPSHOT_CURSOR, + cursor: 'next-page', has_more: false, values: [] }), @@ -56,7 +57,7 @@ describe('ConvexApiClient', () => { const client = new ConvexApiClient(baseConfig); const page = await client.listSnapshot({ tableName: 'users' }); - expect(page.snapshot).toBe('100'); + expect(page.snapshot).toBe(SNAPSHOT_CURSOR); expect(fetchSpy.mock.calls.length).toBe(2); expect(String(fetchSpy.mock.calls[1]![0])).toContain('/api/streaming_export/list_snapshot'); }); @@ -75,11 +76,11 @@ describe('ConvexApiClient', () => { const client = new ConvexApiClient(baseConfig); const page = await client.listSnapshot({ - snapshot: '1770335566197683', + snapshot: SNAPSHOT_CURSOR, tableName: 'lists' }); - expect(page.snapshot).toBe('1770335566197683'); + expect(page.snapshot).toBe(SNAPSHOT_CURSOR); expect(page.cursor).toBe('next-page'); }); @@ -150,7 +151,7 @@ describe('ConvexApiClient', () => { const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( new Response( JSON.stringify({ - snapshot: '100', + snapshot: SNAPSHOT_CURSOR, cursor: null, has_more: false, values: [] @@ -160,7 +161,7 @@ describe('ConvexApiClient', () => { ); const client = new ConvexApiClient(baseConfig); - await client.listSnapshot({ tableName: 'lists', snapshot: '100' }); + await client.listSnapshot({ tableName: 'lists', snapshot: SNAPSHOT_CURSOR }); const url = String(fetchSpy.mock.calls[0]![0]); expect(url).toContain('table_name=lists'); diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts index 9fc858bd7..d3591902c 100644 --- a/modules/module-convex/test/src/ConvexLSN.test.ts +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { parseConvexLsn, toConvexLsn } from '@module/common/ConvexLSN.js'; +import { parseConvexLsn, toConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; describe('Convex cursor LSN helpers', () => { it('validates and round-trips the numeric cursor', () => { @@ -10,7 +10,7 @@ describe('Convex cursor LSN helpers', () => { expect(roundTrip).toBe('1772817606884944136'); }); - it('sorts lexicographically by timestamp', () => { + it('sorts lexicographically after validating fixed-width timestamps', () => { const older = toConvexLsn('1772817606884944136'); const newer = toConvexLsn('1772817606884944137'); @@ -22,6 +22,14 @@ describe('Convex cursor LSN helpers', () => { expect(parsed).toBe('1772817606884944136'); }); + it('allows the zero sentinel', () => { + expect(parseConvexLsn(ZERO_LSN)).toBe(ZERO_LSN); + }); + + it('rejects non-19-digit numeric cursors', () => { + expect(() => parseConvexLsn('1770335566197683')).toThrow('Convex cursor is not a 19-digit numeric timestamp'); + }); + it('rejects padded serialized payloads', () => { expect(() => parseConvexLsn('0001772817606884944136')).toThrow( 'Convex cursor is not a canonical numeric timestamp' diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index c8257daf7..3a79d0edc 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -4,6 +4,8 @@ import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; import { toConvexLsn } from '@module/common/ConvexLSN.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; +const HEAD_CURSOR = '1772817606884944123'; + function createAdapter() { const config = normalizeConnectionConfig({ type: 'convex', @@ -34,7 +36,7 @@ function createAdapter() { ], raw: {} }), - getHeadCursor: async () => '123', + getHeadCursor: async () => HEAD_CURSOR, createWriteCheckpointMarker: async () => undefined }; @@ -79,13 +81,13 @@ bucket_definitions: it('creates replication head from the global snapshot cursor', async () => { const adapter = createAdapter(); - const getHeadCursor = vi.fn(async (_options?: any) => '123'); + const getHeadCursor = vi.fn(async (_options?: any) => HEAD_CURSOR); const createWriteCheckpointMarker = vi.fn(async (_options?: any) => undefined); (adapter as any).connectionManager.client.getHeadCursor = getHeadCursor; (adapter as any).connectionManager.client.createWriteCheckpointMarker = createWriteCheckpointMarker; const result = await adapter.createReplicationHead(async (head) => head); - expect(result).toBe(toConvexLsn('123')); + expect(result).toBe(toConvexLsn(HEAD_CURSOR)); expect(getHeadCursor).toHaveBeenCalledTimes(1); expect(getHeadCursor).toHaveBeenCalledWith(); expect(createWriteCheckpointMarker).toHaveBeenCalledTimes(1); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 83dc98a05..b1eb28808 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -4,6 +4,13 @@ import { describe, expect, it, vi } from 'vitest'; import { toConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; import { ConvexStream } from '@module/replication/ConvexStream.js'; +const CURSOR_100 = '1772817606884944100'; +const CURSOR_101 = '1772817606884944101'; +const CURSOR_102 = '1772817606884944102'; +const CURSOR_200 = '1772817606884944200'; +const CURSOR_300 = '1772817606884944300'; +const CURSOR_301 = '1772817606884944301'; + function createFakeStorage(options?: { snapshotDone?: boolean; snapshotLsn?: string | null; @@ -130,7 +137,7 @@ function createFakeStorage(options?: { return { active: true, snapshot_done: options?.snapshotDone ?? false, - checkpoint_lsn: options?.snapshotDone ? toConvexLsn('100') : null, + checkpoint_lsn: options?.snapshotDone ? toConvexLsn(CURSOR_100) : null, snapshot_lsn: options?.snapshotLsn ?? null }; }, @@ -171,7 +178,7 @@ describe('ConvexStream', () => { snapshotCalls.push(options ?? {}); if (options?.tableName == null) { return { - snapshot: '100', + snapshot: CURSOR_100, cursor: null, hasMore: false, values: [] @@ -179,7 +186,7 @@ describe('ConvexStream', () => { } return { - snapshot: '100', + snapshot: CURSOR_100, cursor: null, hasMore: false, values: [{ _table: 'users', _id: 'u1', name: 'Alice' }] @@ -215,12 +222,12 @@ describe('ConvexStream', () => { expect(snapshotCalls[0]?.cursor).toBeUndefined(); expect(snapshotCalls[1]?.tableName).toBe('users'); expect(snapshotCalls[1]?.cursor).toBeUndefined(); - expect(snapshotCalls[1]?.snapshot).toBe('100'); + expect(snapshotCalls[1]?.snapshot).toBe(CURSOR_100); expect(context.saves.length).toBe(1); expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); expect(context.resumeLsnUpdates.length).toBe(1); - expect(context.allSnapshotDoneLsns).toEqual([toConvexLsn('100')]); - expect(context.commits.at(-1)).toBe(toConvexLsn('100')); + expect(context.allSnapshotDoneLsns).toEqual([toConvexLsn(CURSOR_100)]); + expect(context.commits.at(-1)).toBe(toConvexLsn(CURSOR_100)); }); it('decodes bytes fields to Uint8Array during snapshot hydration', async () => { @@ -255,7 +262,7 @@ describe('ConvexStream', () => { listSnapshot: async (options: any) => { if (options?.tableName == null) { return { - snapshot: '100', + snapshot: CURSOR_100, cursor: null, hasMore: false, values: [] @@ -263,13 +270,13 @@ describe('ConvexStream', () => { } return { - snapshot: '100', + snapshot: CURSOR_100, cursor: null, hasMore: false, values: [{ _table: 'users', _id: 'u1', avatar: 'AQID' }] }; }, - getGlobalSnapshotCursor: async () => '100' + getGlobalSnapshotCursor: async () => CURSOR_100 } } as any }); @@ -282,7 +289,7 @@ describe('ConvexStream', () => { it('resumes table snapshots from the persisted page cursor', async () => { const context = createFakeStorage({ - snapshotLsn: toConvexLsn('200'), + snapshotLsn: toConvexLsn(CURSOR_200), tableSnapshotStatus: { replicatedCount: 1, totalEstimatedCount: -1, @@ -295,7 +302,7 @@ describe('ConvexStream', () => { const listSnapshot = vi.fn(async (options: any) => { snapshotCalls.push(options ?? {}); return { - snapshot: '200', + snapshot: CURSOR_200, cursor: null, hasMore: false, values: [{ _table: 'users', _id: 'u2', name: 'Bob' }] @@ -330,7 +337,7 @@ describe('ConvexStream', () => { expect(getGlobalSnapshotCursor).not.toHaveBeenCalled(); expect(snapshotCalls.length).toBe(1); - expect(snapshotCalls[0]?.snapshot).toBe('200'); + expect(snapshotCalls[0]?.snapshot).toBe(CURSOR_200); expect(snapshotCalls[0]?.cursor).toBe('page-2'); expect(context.saves.length).toBe(1); expect(context.tableProgressUpdates).toHaveLength(1); @@ -339,7 +346,7 @@ describe('ConvexStream', () => { it('marks snapshot done without re-reading rows when the final page was already flushed', async () => { const context = createFakeStorage({ - snapshotLsn: toConvexLsn('200'), + snapshotLsn: toConvexLsn(CURSOR_200), tableSnapshotStatus: { replicatedCount: 2, totalEstimatedCount: -1, @@ -348,7 +355,7 @@ describe('ConvexStream', () => { }); const abortController = new AbortController(); const listSnapshot = vi.fn(async () => ({ - snapshot: '200', + snapshot: CURSOR_200, cursor: null, hasMore: false, values: [] @@ -385,7 +392,7 @@ describe('ConvexStream', () => { it('fails when table snapshots return a different snapshot boundary', async () => { const context = createFakeStorage({ - snapshotLsn: toConvexLsn('300') + snapshotLsn: toConvexLsn(CURSOR_300) }); const abortController = new AbortController(); @@ -407,7 +414,7 @@ describe('ConvexStream', () => { }), getGlobalSnapshotCursor: async () => 'should-not-be-called', listSnapshot: async () => ({ - snapshot: '301', + snapshot: CURSOR_301, cursor: null, hasMore: false, values: [{ _table: 'users', _id: 'u1', name: 'Alice' }] @@ -422,7 +429,7 @@ describe('ConvexStream', () => { it('streams deltas and commits checkpoint', async () => { const context = createFakeStorage({ snapshotDone: true, - resumeFromLsn: toConvexLsn('100') + resumeFromLsn: toConvexLsn(CURSOR_100) }); const abortController = new AbortController(); @@ -450,7 +457,7 @@ describe('ConvexStream', () => { deltaCalls.push(options ?? {}); setTimeout(() => abortController.abort(), 0); return { - cursor: '101', + cursor: CURSOR_101, hasMore: false, values: [ { _table: 'users', _id: 'u1', name: 'Updated' }, @@ -468,14 +475,14 @@ describe('ConvexStream', () => { expect(context.saves.length).toBe(2); expect(context.saves[0]?.tag).toBe(SaveOperationTag.UPDATE); expect(context.saves[1]?.tag).toBe(SaveOperationTag.DELETE); - expect(context.commits.at(-1)).toBe(toConvexLsn('101')); + expect(context.commits.at(-1)).toBe(toConvexLsn(CURSOR_101)); expect(deltaCalls[0]?.tableName).toBeUndefined(); }); it('refreshes metadata before snapshotting a newly discovered wildcard-matched table inline', async () => { const context = createFakeStorage({ snapshotDone: true, - resumeFromLsn: toConvexLsn('100'), + resumeFromLsn: toConvexLsn(CURSOR_100), sourcePatterns: [new TablePattern('convex', 'projects%')] }); const abortController = new AbortController(); @@ -486,7 +493,7 @@ describe('ConvexStream', () => { raw: {} })); const listSnapshot = vi.fn(async (options: any) => ({ - snapshot: '101', + snapshot: CURSOR_101, cursor: null, hasMore: false, values: [{ _table: 'projects_archive', _id: 'p1', name: 'From snapshot' }] @@ -510,7 +517,7 @@ describe('ConvexStream', () => { calls += 1; setTimeout(() => abortController.abort(), 0); return { - cursor: '101', + cursor: CURSOR_101, hasMore: false, values: [{ _table: 'projects_archive', _id: 'p1', name: 'From delta' }] }; @@ -526,7 +533,7 @@ describe('ConvexStream', () => { expect(listSnapshot).toHaveBeenCalledTimes(1); expect(listSnapshot).toHaveBeenCalledWith({ tableName: 'projects_archive', - snapshot: '101', + snapshot: CURSOR_101, cursor: undefined, signal: abortController.signal }); @@ -539,7 +546,7 @@ describe('ConvexStream', () => { it('keeps alive immediately when only checkpoint marker rows are streamed', async () => { const context = createFakeStorage({ snapshotDone: true, - resumeFromLsn: toConvexLsn('100') + resumeFromLsn: toConvexLsn(CURSOR_100) }); const abortController = new AbortController(); let calls = 0; @@ -564,7 +571,7 @@ describe('ConvexStream', () => { calls += 1; if (calls == 1) { return { - cursor: '101', + cursor: CURSOR_101, hasMore: true, values: [] }; @@ -572,7 +579,7 @@ describe('ConvexStream', () => { setTimeout(() => abortController.abort(), 0); return { - cursor: '102', + cursor: CURSOR_102, hasMore: false, values: [{ _table: 'powersync_checkpoints', _id: 'cp1' }] }; @@ -585,6 +592,6 @@ describe('ConvexStream', () => { expect(context.saves.length).toBe(0); expect(context.commits.length).toBe(0); - expect(context.keepalives).toEqual([toConvexLsn('101'), toConvexLsn('102')]); + expect(context.keepalives).toEqual([toConvexLsn(CURSOR_101), toConvexLsn(CURSOR_102)]); }); }); From 5b53f413c277df3539a1563b5a4c34ffb1852541 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Wed, 18 Mar 2026 12:49:08 -0600 Subject: [PATCH 38/86] filter out additional metadata fields during replication --- .../module-convex/src/common/convex-to-sqlite.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/module-convex/src/common/convex-to-sqlite.ts b/modules/module-convex/src/common/convex-to-sqlite.ts index e4c5bdd22..980a10609 100644 --- a/modules/module-convex/src/common/convex-to-sqlite.ts +++ b/modules/module-convex/src/common/convex-to-sqlite.ts @@ -8,11 +8,23 @@ import { import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; import { ConvexRawDocument } from '../client/ConvexApiClient.js'; + +/** + * From Convex docs: + * Every document in Convex automatically has two system fields: + + _id - a unique document ID with validator v.id("tableName") + _creationTime - a creation timestamp with validator v.number() + + Technically _creationTime should be handlded differently as the other metadata excludes defined below, but we include it here for convenience since it's not necessary for replication. + */ +const INTERNAL_KEYS = new Set(['_table', '_deleted', '_ts', '_component', '_creationTime']); + export function toSqliteInputRow(change: ConvexRawDocument, properties?: Record): SqliteInputRow { const row: DatabaseInputRow = {}; for (const [key, value] of Object.entries(change)) { - if (key == '_table' || key == '_deleted') { + if (INTERNAL_KEYS.has(key)) { continue; } From 94a4ec2319c4492d4a1d98016e27435ac7c2659a Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Wed, 18 Mar 2026 13:02:21 -0600 Subject: [PATCH 39/86] Enforce reject_ip_ranges for write checkpoint creation --- .../src/client/ConvexApiClient.ts | 2 ++ .../test/src/ConvexApiClient.test.ts | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index d83bb9259..d3e52e03e 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -131,6 +131,8 @@ export class ConvexApiClient { } async createWriteCheckpointMarker(options?: { signal?: AbortSignal }): Promise { + await this.assertHostAllowed(); + await this.performRequest({ method: 'POST', path: '/api/mutation', diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index 329d1a667..3ee372e94 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -209,4 +209,22 @@ describe('ConvexApiClient', () => { retryable: false }); }); + + it('checks the hostname policy before creating checkpoint markers', async () => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response(JSON.stringify({ status: 'success' }), { status: 200 }) + ); + const lookup = vi.fn((_hostname: string, _options: any, callback: (error: Error) => void) => { + callback(new Error('blocked by reject_ip_ranges')); + }); + + const client = new ConvexApiClient({ + ...baseConfig, + lookup + }); + + await expect(client.createWriteCheckpointMarker()).rejects.toThrow('blocked by reject_ip_ranges'); + expect(lookup).toHaveBeenCalledTimes(1); + expect(fetchSpy).not.toHaveBeenCalled(); + }); }); From 07385777be081610d4203e84afbad3caa11132d6 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Wed, 18 Mar 2026 13:12:17 -0600 Subject: [PATCH 40/86] linting --- modules/module-convex/src/common/convex-to-sqlite.ts | 3 +-- modules/module-convex/test/src/ConvexApiClient.test.ts | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/module-convex/src/common/convex-to-sqlite.ts b/modules/module-convex/src/common/convex-to-sqlite.ts index 980a10609..8113be849 100644 --- a/modules/module-convex/src/common/convex-to-sqlite.ts +++ b/modules/module-convex/src/common/convex-to-sqlite.ts @@ -8,7 +8,6 @@ import { import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; import { ConvexRawDocument } from '../client/ConvexApiClient.js'; - /** * From Convex docs: * Every document in Convex automatically has two system fields: @@ -18,7 +17,7 @@ import { ConvexRawDocument } from '../client/ConvexApiClient.js'; Technically _creationTime should be handlded differently as the other metadata excludes defined below, but we include it here for convenience since it's not necessary for replication. */ -const INTERNAL_KEYS = new Set(['_table', '_deleted', '_ts', '_component', '_creationTime']); +const INTERNAL_KEYS = new Set(['_table', '_deleted', '_ts', '_component', '_creationTime']); export function toSqliteInputRow(change: ConvexRawDocument, properties?: Record): SqliteInputRow { const row: DatabaseInputRow = {}; diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index 3ee372e94..0b3cbcdca 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -211,9 +211,9 @@ describe('ConvexApiClient', () => { }); it('checks the hostname policy before creating checkpoint markers', async () => { - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( - new Response(JSON.stringify({ status: 'success' }), { status: 200 }) - ); + const fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(new Response(JSON.stringify({ status: 'success' }), { status: 200 })); const lookup = vi.fn((_hostname: string, _options: any, callback: (error: Error) => void) => { callback(new Error('blocked by reject_ip_ranges')); }); From 2d8745872380f2dbb36b05bc835ef21d3e7d2ba4 Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Wed, 18 Mar 2026 13:19:47 -0600 Subject: [PATCH 41/86] cast mock to fix overloaded signatures --- modules/module-convex/test/src/ConvexApiClient.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index 0b3cbcdca..54f49138b 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -216,7 +216,7 @@ describe('ConvexApiClient', () => { .mockResolvedValue(new Response(JSON.stringify({ status: 'success' }), { status: 200 })); const lookup = vi.fn((_hostname: string, _options: any, callback: (error: Error) => void) => { callback(new Error('blocked by reject_ip_ranges')); - }); + }) as unknown as import('node:net').LookupFunction; const client = new ConvexApiClient({ ...baseConfig, From 4b39e8a0bce14b055f3d893d9f021a2d9e1f5cef Mon Sep 17 00:00:00 2001 From: Kobie Botha Date: Wed, 18 Mar 2026 23:08:51 -0600 Subject: [PATCH 42/86] cleanup README --- modules/module-convex/README.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 4a98198e2..98ef66271 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -8,7 +8,6 @@ Convex replication module for PowerSync. replication: connections: - type: convex - id: default deployment_url: https://.convex.cloud deploy_key: polling_interval_ms: 1000 @@ -28,6 +27,7 @@ The content below is written in an agents.md style describing the behavior of `m - This module replicates Convex data into PowerSync bucket storage. - Source APIs used are Convex [Streaming Export](https://docs.convex.dev/streaming-export-api): (`json_schemas`, `list_snapshot`, `document_deltas`). - Initial scope is default Convex component only, but we could consider support for custom components in the future if we can figure out consistency. +- Deploy keys grant root access (read/write on all tables), components could address this later. ## 2) Canonical Behavior @@ -58,18 +58,14 @@ The content below is written in an agents.md style describing the behavior of `m ## 4) LSN and Cursor Rules - Convex snapshot and delta cursors are always `i64` timestamps (serialized as decimal numeric strings in JSON). -- The `list_snapshot` pagination cursor is a separate JSON-serialized `{tablet, id}` string — it is pagination state, not a replication cursor. +- The `list_snapshot` pagination cursor is a separate JSON-serialized `{table, id}` string — it is pagination state, not a replication cursor. - Persisted Convex LSNs must be canonical 19-digit numeric cursor strings. `ZERO_LSN = "0"` remains the internal sentinel. ## 5) API Client Contract - Auth header: `Authorization: Convex `. - Always request `format=json`. -- Fallback path support: `/api/streaming_export/...` when `/api/...` returns `404`. - Parse large numeric JSON using `JSONBig`. -- `json_schemas` must support: - - array/object under `tables`, - - self-host top-level table map shape. - Retry classification: - retryable: network, timeout, 429, 5xx. - non-retryable: malformed responses, auth/config issues. @@ -110,14 +106,15 @@ The content below is written in an agents.md style describing the behavior of `m - Source marker table: `powersync_checkpoints` - Convex rejects table names starting with `_`, so no leading-underscore variant is used. - The table has a single `last_updated` field; the mutation upserts one row (bounded to one row total). - - The developer must deploy the `powersync_checkpoints` schema and mutation to their Convex project (see README). + - The developer must deploy the `powersync_checkpoints` schema and mutation to their Convex project. - Stream handling requirement: - checkpoint marker tables must always be excluded from replicated source tables and ignored in delta row application. - marker-only delta pages must trigger immediate `keepalive` checkpoint advancement (do not wait for 60s throttle). -## 9) Convex-specific notes +## 9) Other Convex-specific notes - The default schema is `convex` +- On an idle system, multiple successive calls to `/api/document_deltas` will return the same cursor value i.e. the cursor is not wall clock based. - **Mutation Transaction Atomicity in** `document_deltas` - The `cursor` in `/api/document_deltas` is a Convex commit **timestamp** (`i64`), not a per-operation counter. From a18bf9d5b5824bcbb715b303d754ebb9fef107a2 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 28 Apr 2026 13:20:28 +0200 Subject: [PATCH 43/86] wip: use ReplicationLagTracker --- .../src/replication/ConvexReplicationJob.ts | 2 +- .../src/replication/ConvexReplicator.ts | 27 +++++++------ .../src/replication/ConvexStream.ts | 40 +++++++++---------- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/modules/module-convex/src/replication/ConvexReplicationJob.ts b/modules/module-convex/src/replication/ConvexReplicationJob.ts index 5cb7058f7..cc77b5265 100644 --- a/modules/module-convex/src/replication/ConvexReplicationJob.ts +++ b/modules/module-convex/src/replication/ConvexReplicationJob.ts @@ -69,7 +69,7 @@ export class ConvexReplicationJob extends replication.AbstractReplicationJob { } } - async getReplicationLagMillis(): Promise { + getReplicationLagMillis(): number | undefined { return this.lastStream?.getReplicationLagMillis(); } } diff --git a/modules/module-convex/src/replication/ConvexReplicator.ts b/modules/module-convex/src/replication/ConvexReplicator.ts index 26cb90cc3..a50671311 100644 --- a/modules/module-convex/src/replication/ConvexReplicator.ts +++ b/modules/module-convex/src/replication/ConvexReplicator.ts @@ -39,24 +39,25 @@ export class ConvexReplicator extends replication.AbstractReplicator { - const lag = await super.getReplicationLagMillis(); + getReplicationLagMillis(): number | undefined { + const lag = super.getReplicationLagMillis(); if (lag != null) { return lag; } - const content = await this.storage.getActiveSyncRulesContent(); - if (content == null) { - return undefined; - } + // TODO fixme + // const content = await this.storage.getActiveSyncRulesContent(); + // if (content == null) { + // return undefined; + // } - const checkpointTs = content.last_checkpoint_ts?.getTime() ?? 0; - const keepaliveTs = content.last_keepalive_ts?.getTime() ?? 0; - const latestTs = Math.max(checkpointTs, keepaliveTs); - if (latestTs == 0) { - return undefined; - } + // const checkpointTs = content.last_checkpoint_ts?.getTime() ?? 0; + // const keepaliveTs = content.last_keepalive_ts?.getTime() ?? 0; + // const latestTs = Math.max(checkpointTs, keepaliveTs); + // if (latestTs == 0) { + // return undefined; + // } - return Date.now() - latestTs; + // return Date.now() - latestTs; } } diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 366a37623..fc22d2790 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -1,15 +1,16 @@ import { container, DatabaseConnectionError, + logger as defaultLogger, ErrorCode, Logger, - logger as defaultLogger, ReplicationAbortedError, ReplicationAssertionError } from '@powersync/lib-services-framework'; import { MetricsEngine, RelationCache, + ReplicationLagTracker, SaveOperationTag, SourceEntityDescriptor, SourceTable, @@ -50,13 +51,12 @@ export class ConvexStream { private readonly logger: Logger; private readonly relationCache = new RelationCache(getCacheIdentifier); + private replicationLag = new ReplicationLagTracker(); private tableSchemaCache: ConvexTableSchema[] | null = null; private tableSchemaPropertiesByName = new Map>(); - private oldestUncommittedChange: Date | null = null; private lastKeepaliveAt = 0; - private isStartingReplication = true; private lastTouchedAt = performance.now(); constructor(private readonly options: ConvexStreamOptions) { @@ -66,6 +66,10 @@ export class ConvexStream { this.logger = options.logger ?? defaultLogger; } + get isStartingReplication() { + return this.replicationLag.isStartingReplication; + } + private get connections() { return this.options.connections; } @@ -153,6 +157,10 @@ export class ConvexStream { let changesInPage = 0; let sawCheckpointMarker = false; const snapshottedTablesInPage = new Set(); + + // track the begin of a batch/"transaction?" + // TODO(steven) verify this for real + this.replicationLag.trackUncommittedChange(new Date(nextCursor)); for (const change of page.values) { if (this.abortSignal.aborted) { throw new ReplicationAbortedError('Replication interrupted'); @@ -182,30 +190,27 @@ export class ConvexStream { } changesInPage += 1; - if (this.oldestUncommittedChange == null) { - this.oldestUncommittedChange = new Date(); - } } if (changesInPage > 0) { + // TODO(steven) Should this be commiting at this point? If this is paged? + // Where are the transactio boundaries const didCommit = await batch.commit(pageLsn, { createEmptyCheckpoints: false, - oldestUncommittedChange: this.oldestUncommittedChange + oldestUncommittedChange: this.replicationLag.oldestUncommittedChange }); - if (didCommit) { + if (!didCommit.checkpointBlocked) { this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); - this.oldestUncommittedChange = null; - this.isStartingReplication = false; + this.replicationLag.markCommitted(); } } else if (sawCheckpointMarker) { await batch.keepalive(pageLsn); this.lastKeepaliveAt = Date.now(); - this.isStartingReplication = false; } else if (nextCursor != cursor && Date.now() - this.lastKeepaliveAt > 60_000) { await batch.keepalive(pageLsn); + this.replicationLag.markStarted(); this.lastKeepaliveAt = Date.now(); - this.isStartingReplication = false; } cursor = nextCursor; @@ -223,15 +228,8 @@ export class ConvexStream { } } - async getReplicationLagMillis(): Promise { - if (this.oldestUncommittedChange == null) { - if (this.isStartingReplication) { - return undefined; - } - return 0; - } - - return Date.now() - this.oldestUncommittedChange.getTime(); + getReplicationLagMillis(): number | undefined { + return this.replicationLag.getLagMillis(); } private async initSlot(): Promise<{ needsInitialSync: boolean; snapshotLsn: string | null }> { From b5347153866f46ac809ffb23d36fb7f1a7653a15 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 4 May 2026 11:41:40 +0200 Subject: [PATCH 44/86] update for replication lag tracking. Add write checkpoint notes. --- docs/convex-write-checkpoints.md | 205 ++++++++++++++++++ .../src/api/ConvexRouteAPIAdapter.ts | 5 +- .../src/replication/ConvexStream.ts | 31 ++- 3 files changed, 233 insertions(+), 8 deletions(-) create mode 100644 docs/convex-write-checkpoints.md diff --git a/docs/convex-write-checkpoints.md b/docs/convex-write-checkpoints.md new file mode 100644 index 000000000..e534c3f87 --- /dev/null +++ b/docs/convex-write-checkpoints.md @@ -0,0 +1,205 @@ +# Convex write checkpoints + +This note looks at whether the Convex implementation needs to write to the +`powersync_checkpoints` collection from +`createWriteCheckpointMarker(options?: { signal?: AbortSignal }): Promise`. + +## Background + +Write checkpoints let a client wait until its own uploaded write has been +replicated back through PowerSync before applying the corresponding synced data. +This avoids flicker and avoids accepting a local write as durable before the +backend-side effects of that write are visible in the sync stream. + +The managed write checkpoint flow is: + +1. The client or backend writes to the source database. +2. The PowerSync API creates a managed write checkpoint for the user/client. +3. The source connector records a source replication position that should include + the write. +4. PowerSync stores a generated write checkpoint id together with that source + position. +5. When replication commits or keeps alive at or past that source position, + sync can expose the write checkpoint id to the client. + +For Convex, the source position is the Convex replication cursor. The current +implementation in `createReplicationHead`: + +1. calls `getHeadCursor()` to read the current global Convex head, +2. calls `createWriteCheckpointMarker()` to run the + `powersync_checkpoints:createCheckpoint` Convex mutation, +3. invokes the callback with the original head cursor. + +The callback stores the managed write checkpoint in bucket storage with the +original head as the replication head. The marker write is intentionally not the +write checkpoint position. It is a later Convex mutation whose job is to advance +the Convex delta stream beyond the stored head. + +The key invariant is that PowerSync must observe a checkpoint update at or past +the stored head after the managed write checkpoint mapping exists. Other source +connectors make this ordering explicit by creating the mapping in the callback +and then triggering the stream. For Convex, the marker is still the mechanism +that creates the later source event, but the implementation should preserve the +same effective ordering: the client must not be left with a stored write +checkpoint id and no later observable checkpoint update. + +## How the marker is used by replication + +Convex `document_deltas` only advances when there is a source-side change. The +stream implementation ignores rows from the `powersync_checkpoints` table as +user data, but still treats marker-only delta pages as meaningful replication +progress: + +- normal user-data delta pages are committed with `batch.commit(pageLsn)`, +- marker-only pages call `batch.keepalive(pageLsn)` immediately, +- the marker table is excluded from source schema discovery and row application. + +That keepalive is enough to advance the stored checkpoint LSN. Once the stored +checkpoint LSN is at or past the managed write checkpoint head, the sync stream +can acknowledge the write checkpoint to the client. + +## Case 1: no checkpoint collection + +If there is no `powersync_checkpoints` collection, `createReplicationHead` can +still read the current Convex head and create a managed write checkpoint in +PowerSync storage. The problem is that nothing guarantees the Convex delta stream +will advance beyond that head. + +### Case 1.1: no replication lag + +If there is no replication lag, the latest Convex head may already include the +client's source write by the time `/write-checkpoint2.json` runs. PowerSync can +therefore associate the generated write checkpoint id with a correct source +head. + +However, correctness of the association is not enough. The connected sync client +only learns about acknowledged write checkpoints when PowerSync sends a later +sync checkpoint whose source LSN is at or beyond the stored write checkpoint +head. + +If the replicator has already processed past that head before the write +checkpoint mapping is created, there may be no future replication event to cause +a commit or keepalive. On an idle Convex app, repeated `document_deltas` calls can +return the same cursor, so the Convex head does not progress just because time +passes. The client can then wait indefinitely for a checkpoint acknowledgement +that is logically already true but is never published through a newer sync +checkpoint. If the backend mutation made additional changes, the client may also +not observe those changes as the confirmed result of its write. + +### Case 1.2: manual write checkpoint creation + +Manually calling `/write-checkpoint2.json` has the same shape. The write +checkpoint is associated with the current Convex head, but if no later source +change occurs, the stream does not advance. The connected client will not see the +write checkpoint acknowledgement until an unrelated external change moves the +Convex cursor and replication commits or keeps alive at that newer cursor. + +## Case 2: checkpoint collection exists + +With the `powersync_checkpoints` collection and mutation deployed, +`createWriteCheckpointMarker()` creates a small source-side write immediately +after the head is captured. + +### Case 2.1: marker advances past the managed write checkpoint head + +The managed write checkpoint is associated with the pre-marker head. The marker +mutation is committed after that head, so its delta cursor is greater than or +equal to the point needed to acknowledge the managed write checkpoint. + +When the replicator sees the marker row in `document_deltas`, it ignores the row +as replicated data but calls `keepalive(pageLsn)`. That advances the PowerSync +checkpoint LSN past the managed write checkpoint head, allowing the generated +write checkpoint id to be sent to the client. + +This works regardless of whether the normal user-data write was already +replicated before `/write-checkpoint2.json` ran, as long as the marker-driven +checkpoint update is observed after the managed write checkpoint mapping exists. +That is the event that makes the stored write checkpoint visible to sync. + +### Case 2.2: marker keeps idle systems moving + +The marker write also handles the idle-source case. Without it, an idle Convex +deployment may keep returning the same delta cursor forever. With it, the marker +adds a real Convex mutation, which bumps the delta stream and gives the +replicator something to process immediately. + +Because marker-only pages trigger immediate keepalive, write checkpoint latency +does not depend on the 60 second idle keepalive throttle and does not depend on +an unrelated application write. + +## Comparison to other sources + +This is the same role played by source-specific keepalive mechanisms elsewhere: + +- Postgres stores the current WAL LSN and emits a logical replication message so + replication proceeds past that LSN. +- MongoDB stores the current `clusterTime` and writes to + `_powersync_checkpoints` so the change stream proceeds past that time. +- Convex stores the current cursor and writes to `powersync_checkpoints` so + `document_deltas` proceeds past that cursor. + +The common requirement is not "write checkpoint data must be replicated as user +data". The requirement is that, after creating the managed write checkpoint +mapping, the source stream must produce an ordered event beyond the stored +position. + +## Filtered replication streams + +There is an additional failure mode for source databases with filtered +replication streams. For example, imagine a Postgres setup where the logical +replication stream only includes table `A`. + +1. The client writes to table `A`. +2. Some backend logic writes to table `B`. +3. The client then requests a write checkpoint. +4. PowerSync reads the current source head after the table `B` write. + +That source head is a valid database LSN, but the filtered replication stream may +never emit the table `B` change. If no later table `A` change occurs, replication +will not observe an event at or beyond the stored write checkpoint LSN, even +though the source database head has advanced. The write checkpoint can then be +stuck waiting for a position that is real in the source database but not +reachable through the filtered stream. + +That is why filtered sources need a marker that is guaranteed to appear in the +replication stream. In Postgres this can be a logical replication message; in +MongoDB this is a write to a collection included in the change stream. The marker +bridges the gap between "the database head advanced" and "the replication stream +observed progress". + +This particular filtered-stream issue does not appear to apply to Convex in the +same way. Convex `document_deltas` is not filtered by table at the API level for +PowerSync; table filtering happens inside the replicator after it receives the +delta page. A write to another Convex table should still advance the Convex delta +cursor visible to the replicator, even if PowerSync later ignores that row +because it is not included in sync rules. + +So for Convex, the checkpoint marker is less about overcoming source-side stream +filtering and more about guaranteeing source-side progress when there are no +other writes after the managed checkpoint mapping is created. The marker is still +required because an idle Convex deployment may otherwise keep returning the same +delta cursor, but the reason is different from a filtered Postgres slot that +cannot observe some source LSNs at all. + +## Conclusion + +With the current Convex replication API and managed write checkpoint design, the +write to `powersync_checkpoints` is required. + +The marker collection is not required because PowerSync needs to sync the marker +document itself. It is required because Convex cursors only advance on source +writes, and managed write checkpoints are only acknowledged after bucket storage +commits or keeps alive at a cursor at or beyond the stored head. Without the +marker write, a write checkpoint can be associated with a correct Convex head but +never become visible to the connected client on an idle source, especially when +replication has already processed past that head before the write checkpoint +mapping was stored. + +Avoiding source writes would require a different Convex capability or a different +PowerSync design, for example an API that can produce an ordered replication +barrier without a mutation, or storage logic that can safely publish a newly +created managed write checkpoint against an already-committed source head. Until +then, `CONVEX_CHECKPOINT_TABLE` is the mechanism that makes managed write +checkpoints reliably observable. The implementation should also preserve the +ordering invariant above; otherwise the marker can be replicated before the +managed mapping exists, recreating the same idle-source stall in a narrower race. diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index 2d7095e03..e69507849 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -7,8 +7,8 @@ import { } from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; -import { toConvexLsn } from '../common/ConvexLSN.js'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; +import { toConvexLsn } from '../common/ConvexLSN.js'; import { extractProperties, readConvexFieldType, toExpressionTypeFromConvexType } from '../common/convex-to-sqlite.js'; import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; import * as types from '../types/types.js'; @@ -128,8 +128,9 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { async createReplicationHead(callback: ReplicationHeadCallback): Promise { const head = await this.connectionManager.client.getHeadCursor(); + const result = await callback(toConvexLsn(head)); await this.connectionManager.client.createWriteCheckpointMarker(); - return await callback(toConvexLsn(head)); + return result; } async getConnectionSchema(): Promise { diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index fc22d2790..75ba69af1 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -158,9 +158,6 @@ export class ConvexStream { let sawCheckpointMarker = false; const snapshottedTablesInPage = new Set(); - // track the begin of a batch/"transaction?" - // TODO(steven) verify this for real - this.replicationLag.trackUncommittedChange(new Date(nextCursor)); for (const change of page.values) { if (this.abortSignal.aborted) { throw new ReplicationAbortedError('Replication interrupted'); @@ -184,6 +181,16 @@ export class ConvexStream { continue; } + /** + * This tracks the begining of a new transaction which is not yet commited. + * This uses the current page's cursor as the timestamp since this is the closes timestamp + * to the mutation. + * We should only track the first op for a new transaction. + * TODO, the page.cursor wont change between ops here, but, maybe we + * should not call trackUncommitedChange many times here. + */ + this.replicationLag.trackUncommittedChange(new Date(Number(BigInt(page.cursor) / 1_000_000n))); + const changed = await this.writeChange(batch, table, change); if (!changed) { continue; @@ -194,7 +201,12 @@ export class ConvexStream { if (changesInPage > 0) { // TODO(steven) Should this be commiting at this point? If this is paged? - // Where are the transactio boundaries + // Where are the transaction boundaries + /** + * It looks like the document deltas api wont split transactions between pages, + * That means it should be safe to commit after each page. + * Each page could contain many smaller transactions - that should also be fine. + */ const didCommit = await batch.commit(pageLsn, { createEmptyCheckpoints: false, oldestUncommittedChange: this.replicationLag.oldestUncommittedChange @@ -205,10 +217,17 @@ export class ConvexStream { this.replicationLag.markCommitted(); } } else if (sawCheckpointMarker) { - await batch.keepalive(pageLsn); + const { checkpointBlocked } = await batch.keepalive(pageLsn); + if (!checkpointBlocked) { + this.replicationLag.clearUncommittedChange(); + } this.lastKeepaliveAt = Date.now(); + // TODO(steven, should this time be configurable?) } else if (nextCursor != cursor && Date.now() - this.lastKeepaliveAt > 60_000) { - await batch.keepalive(pageLsn); + const { checkpointBlocked } = await batch.keepalive(pageLsn); + if (!checkpointBlocked) { + this.replicationLag.clearUncommittedChange(); + } this.replicationLag.markStarted(); this.lastKeepaliveAt = Date.now(); } From d49aacaf375e74a24d3ff8534214db1c6ccaec7c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 4 May 2026 17:02:20 +0200 Subject: [PATCH 45/86] cleanup replication lag timing code --- modules/module-convex/src/common/ConvexLSN.ts | 7 +++++ .../src/replication/ConvexReplicator.ts | 22 ---------------- .../src/replication/ConvexStream.ts | 26 ++++++++++++------- 3 files changed, 23 insertions(+), 32 deletions(-) diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index 4a770d810..2f8e7b286 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -12,6 +12,13 @@ export function toConvexLsn(cursor: string | bigint): string { return asString; } +/** + * Converts the decimal timestamp LSN to a JS Date. + */ +export function lsnToDate(cursor: string): Date { + return new Date(Number(BigInt(cursor) / 1_000_000n)); +} + function assertValidConvexLsn(cursor: string) { if (cursor.length == 0) { throw new Error('Convex cursor cannot be empty'); diff --git a/modules/module-convex/src/replication/ConvexReplicator.ts b/modules/module-convex/src/replication/ConvexReplicator.ts index a50671311..b2db9135f 100644 --- a/modules/module-convex/src/replication/ConvexReplicator.ts +++ b/modules/module-convex/src/replication/ConvexReplicator.ts @@ -38,26 +38,4 @@ export class ConvexReplicator extends replication.AbstractReplicator(); + let didMarkOldestUncommitedChange = false; + for (const change of page.values) { if (this.abortSignal.aborted) { throw new ReplicationAbortedError('Replication interrupted'); @@ -186,10 +189,13 @@ export class ConvexStream { * This uses the current page's cursor as the timestamp since this is the closes timestamp * to the mutation. * We should only track the first op for a new transaction. - * TODO, the page.cursor wont change between ops here, but, maybe we - * should not call trackUncommitedChange many times here. + * Note that the document-deltas aren't filtered, so we only + * mark the start after this point - which means we will have an uncommited change. */ - this.replicationLag.trackUncommittedChange(new Date(Number(BigInt(page.cursor) / 1_000_000n))); + if (!didMarkOldestUncommitedChange) { + this.replicationLag.trackUncommittedChange(lsnToDate(page.cursor)); + didMarkOldestUncommitedChange = true; + } const changed = await this.writeChange(batch, table, change); if (!changed) { @@ -200,29 +206,29 @@ export class ConvexStream { } if (changesInPage > 0) { - // TODO(steven) Should this be commiting at this point? If this is paged? - // Where are the transaction boundaries /** - * It looks like the document deltas api wont split transactions between pages, + * It looks like the document-deltas api won't split transactions between pages, * That means it should be safe to commit after each page. * Each page could contain many smaller transactions - that should also be fine. */ - const didCommit = await batch.commit(pageLsn, { + const { checkpointBlocked } = await batch.commit(pageLsn, { createEmptyCheckpoints: false, oldestUncommittedChange: this.replicationLag.oldestUncommittedChange }); - if (!didCommit.checkpointBlocked) { + if (!checkpointBlocked) { this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); this.replicationLag.markCommitted(); } } else if (sawCheckpointMarker) { + /** + * This is only reached if the checkpoint marker was the only change observed in a page. + */ const { checkpointBlocked } = await batch.keepalive(pageLsn); if (!checkpointBlocked) { this.replicationLag.clearUncommittedChange(); } this.lastKeepaliveAt = Date.now(); - // TODO(steven, should this time be configurable?) } else if (nextCursor != cursor && Date.now() - this.lastKeepaliveAt > 60_000) { const { checkpointBlocked } = await batch.keepalive(pageLsn); if (!checkpointBlocked) { From 104e62808288ad964ed02a105ee924d859fb4d10 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 4 May 2026 17:02:38 +0200 Subject: [PATCH 46/86] add docs for consistency and write checkpoint tests --- docs/{ => convex}/convex-write-checkpoints.md | 75 ++++++++-- docs/convex/snapshot-consistency.md | 136 ++++++++++++++++++ 2 files changed, 200 insertions(+), 11 deletions(-) rename docs/{ => convex}/convex-write-checkpoints.md (78%) create mode 100644 docs/convex/snapshot-consistency.md diff --git a/docs/convex-write-checkpoints.md b/docs/convex/convex-write-checkpoints.md similarity index 78% rename from docs/convex-write-checkpoints.md rename to docs/convex/convex-write-checkpoints.md index e534c3f87..f3848daad 100644 --- a/docs/convex-write-checkpoints.md +++ b/docs/convex/convex-write-checkpoints.md @@ -4,6 +4,8 @@ This note looks at whether the Convex implementation needs to write to the `powersync_checkpoints` collection from `createWriteCheckpointMarker(options?: { signal?: AbortSignal }): Promise`. +TLDR: Yes, we need a `powersync_checkpoints` collection in the source database. + ## Background Write checkpoints let a client wait until its own uploaded write has been @@ -116,17 +118,6 @@ replicated before `/write-checkpoint2.json` ran, as long as the marker-driven checkpoint update is observed after the managed write checkpoint mapping exists. That is the event that makes the stored write checkpoint visible to sync. -### Case 2.2: marker keeps idle systems moving - -The marker write also handles the idle-source case. Without it, an idle Convex -deployment may keep returning the same delta cursor forever. With it, the marker -adds a real Convex mutation, which bumps the delta stream and gives the -replicator something to process immediately. - -Because marker-only pages trigger immediate keepalive, write checkpoint latency -does not depend on the 60 second idle keepalive throttle and does not depend on -an unrelated application write. - ## Comparison to other sources This is the same role played by source-specific keepalive mechanisms elsewhere: @@ -181,6 +172,68 @@ required because an idle Convex deployment may otherwise keep returning the same delta cursor, but the reason is different from a filtered Postgres slot that cannot observe some source LSNs at all. +## Test results + +Several manual tests were run to check whether Convex can safely omit the +`powersync_checkpoints` marker collection. + +### Test 1: direct write checkpoint on an idle source + +When `/write-checkpoint2.json` was called directly without any other client +mutation: + +- without writing to `powersync_checkpoints`, the write checkpoint did not show + up immediately in the client logs or PowerSync service sync logs, +- with the `powersync_checkpoints` marker write enabled, the write checkpoint did + show up immediately. + +This confirms the idle-source failure mode. A write checkpoint can be stored in +PowerSync bucket storage, but without a later Convex delta there may be no sync +checkpoint diff that exposes it to the connected client. + +### Test 2: replication lag + +When replication lag was introduced, the write checkpoint only appeared after +replication caught up. + +This is the desired behavior. The client should not validate a write checkpoint +until PowerSync has replicated to a source position at or beyond the head stored +for that write checkpoint. + +### Test 3: transaction boundaries and unfiltered deltas + +Testing also indicates that `document_deltas` pages contain entire Convex +transactions, or groups of smaller complete transactions. The current replicator +commits after each page that contains changes, so the marker collection does not +appear to be needed for transaction-boundary correctness. + +Testing also supports the expectation that Convex `document_deltas` is not +filtered per table at the API level. PowerSync filters rows inside the +replicator after receiving the delta page. This means Convex should not have the +same source-side filtered-stream problem described above for Postgres table +filters. + +### Test 4: delayed write checkpoint association + +Another test disabled the `powersync_checkpoints` marker write and added a +2-second delay in the Convex API handler before creating the managed write +checkpoint. + +In that setup, the client did not receive a checkpoint associated with the write +checkpoint. The likely sequence is: + +1. the client mutation committed in Convex, +2. the replicator saw the corresponding Convex cursor, +3. PowerSync sent the sync checkpoint to the client before the managed write + checkpoint mapping existed, +4. the API handler created the managed write checkpoint after the delay, +5. no further Convex write occurred, so no later checkpoint diff was sent. + +This reproduces the key race. The source head can be correct and the data can be +replicated, but the connected client can still miss the write checkpoint +acknowledgement if the write checkpoint mapping is created after the relevant +sync checkpoint has already been sent. + ## Conclusion With the current Convex replication API and managed write checkpoint design, the diff --git a/docs/convex/snapshot-consistency.md b/docs/convex/snapshot-consistency.md new file mode 100644 index 000000000..734122807 --- /dev/null +++ b/docs/convex/snapshot-consistency.md @@ -0,0 +1,136 @@ +# Convex snapshot consistency + +The current approach should be consistent: + +1. pin one global Convex snapshot cursor, +2. snapshot each selected table at that same cursor, +3. persist that cursor as the replication resume point, +4. stream `document_deltas` starting from that original cursor. + +The important property is that every table snapshot is read at the same Convex +snapshot cursor, not that all tables are fetched in one HTTP response. Since +`list_snapshot` accepts a `snapshot` parameter, we can snapshot tables +individually while still reading the same database state. + +Filtering specific tables should also be consistent as long as filtering happens +inside the PowerSync replicator after reading the unfiltered Convex delta stream. +It would become more delicate if Convex exposed a table-filtered delta stream and +PowerSync used that as the source of truth, because then some source cursor +positions could be skipped by the stream. + +## Current initial replication flow + +The Convex replicator does not take independent "latest" snapshots per table. +Instead, it resolves a single snapshot boundary and reuses it for every table: + +1. `getGlobalSnapshotCursor()` calls `list_snapshot` without a table filter to + get the latest global snapshot cursor. +2. The replicator stores that cursor as the resume LSN with `setResumeLsn`. +3. Each selected source table is snapshotted with `list_snapshot({ tableName, snapshot: snapshotCursor })`. +4. Each page returned by `list_snapshot` is checked to ensure the returned + `snapshot` still matches the original `snapshotCursor`. +5. After all selected tables are snapshotted, the replicator marks the snapshot + done at that same LSN and commits. +6. Streaming replication starts from the same cursor using `document_deltas`. + +This is the usual snapshot-then-stream consistency pattern. Rows changed while +the snapshot is being read are allowed to be missed by the snapshot because they +will appear in the delta stream after the pinned cursor. + +## Why individual table snapshots are okay + +Fetching each table separately would be unsafe if each request implicitly used +"whatever the latest database state is at request time". In that model, table +`A` could be read at time `T1`, table `B` could be read at time `T2`, and the +combined snapshot could represent no real database state. + +Convex avoids that problem by allowing a fixed snapshot cursor to be supplied to +`list_snapshot`. Once the replicator has pinned cursor `S`, every table snapshot +request is for the state of that table at `S`. + +That means a full "snapshot all tables at once" API is not required for +consistency. It may be convenient, but the consistency boundary is already the +shared snapshot cursor. + +## Relationship to document deltas + +The second half of the consistency guarantee is the relationship between the +snapshot cursor and `document_deltas`. + +For the current design to be correct, the cursor returned by `list_snapshot` +must be usable as the cursor passed to `document_deltas`, and `document_deltas` +must then return all committed changes after that cursor in order. Based on the +behavior tested so far, Convex's streaming export APIs appear to provide this: + +- backend mutations are reflected in the snapshot head after the mutation + succeeds, +- delta pages appear to contain complete Convex transactions, or groups of + complete transactions, +- the delta cursor is a global source position rather than a per-table position. + +With those properties, the snapshot and stream compose cleanly. The snapshot +captures the selected tables at cursor `S`; the delta stream then advances the +replica from `S` onward. + +## Table filtering + +PowerSync can still filter which tables it stores and applies to sync rules. +The key distinction is where filtering happens. + +The safe shape is: + +1. read the global Convex delta stream, +2. observe every Convex cursor in order, +3. ignore rows for tables not selected by sync rules, +4. commit or keepalive based on the cursor that was observed. + +This is the current model: Convex `document_deltas` is not filtered per table at +the API level for PowerSync. The replicator receives delta pages first and then +decides whether each row matches a selected source table. + +That means an unrelated table write can still move the cursor seen by the +replicator, even if PowerSync ignores the row. This is important for checkpoint +progress and avoids the filtered-stream issue seen in sources where the +replication stream itself only includes selected tables. + +## What would be risky + +Consistency would become trickier if PowerSync used a Convex API mode that +filtered deltas before the replicator saw them. + +For example, suppose the source cursor advances for writes to tables `A` and +`B`, but PowerSync only subscribes to a stream of table `A` deltas. If a write to +table `B` advances the global cursor, the source head may move to a position +that the filtered stream never emits. That can make resume points and write +checkpoint heads harder to reason about. + +This does not appear to be the current Convex setup. The current setup filters +after receiving the delta page, so the replicator can still observe cursor +movement caused by ignored tables. + +## Resume behavior + +Because the snapshot cursor is persisted before table snapshotting begins, the +initial snapshot can be resumed against the same global snapshot boundary. Table +progress stores the table page cursor and whether that table's snapshot is +finished. + +On restart, the replicator should continue using the original snapshot cursor +for unfinished tables. Completed tables do not need to be snapshotted again. Once +all tables are marked complete, streaming resumes from the same global cursor and +replays any later deltas. + +This is why resumable per-table snapshots are compatible with consistency: the +resume point is not a new per-table "latest" snapshot. It is the original global +snapshot cursor. + +## Conclusion + +There does not appear to be a consistency issue with snapshotting Convex tables +individually, as long as every table is snapshotted at the same pinned +`list_snapshot` cursor and `document_deltas` starts from that cursor. + +Likewise, filtering selected tables should be safe when filtering happens in the +PowerSync replicator after reading the unfiltered Convex delta stream. The risky +case would be source-side table-filtered deltas, where the global source cursor +can advance for writes the replication stream never exposes. From f93966fcb62f34ef38da5342e76f488036079afd Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 4 May 2026 17:49:36 +0200 Subject: [PATCH 47/86] cleanup replication snapshot resume token --- modules/module-convex/package.json | 1 + .../ConvexSnapshotProgresCursor.ts | 40 +++ .../src/replication/ConvexStream.ts | 68 +--- pnpm-lock.yaml | 299 +----------------- 4 files changed, 48 insertions(+), 360 deletions(-) create mode 100644 modules/module-convex/src/replication/ConvexSnapshotProgresCursor.ts diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json index 80e0767d3..92ba46ed4 100644 --- a/modules/module-convex/package.json +++ b/modules/module-convex/package.json @@ -33,6 +33,7 @@ "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", + "bson": "^6.10.4", "ts-codec": "^1.3.0" }, "devDependencies": { diff --git a/modules/module-convex/src/replication/ConvexSnapshotProgresCursor.ts b/modules/module-convex/src/replication/ConvexSnapshotProgresCursor.ts new file mode 100644 index 000000000..074165ba2 --- /dev/null +++ b/modules/module-convex/src/replication/ConvexSnapshotProgresCursor.ts @@ -0,0 +1,40 @@ +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import bson from 'bson'; +import * as t from 'ts-codec'; + +export const ConvexSnapshotProgressCursor = t.object({ + cursor: t.string.or(t.Null), + finished: t.boolean +}); + +export type ConvexSnapshotProgressCursor = t.Encoded; + +export const DEFAULT_CONVEX_SNAPSHOT_CURSOR: ConvexSnapshotProgressCursor = { + cursor: null, + finished: false +}; + +export const BinaryConvexSnapshotProgressCursor = t.codec( + 'ConvexSnapshotProgressCursor', + (decoded: ConvexSnapshotProgressCursor) => bson.serialize(ConvexSnapshotProgressCursor.encode(decoded)), + (encoded) => ConvexSnapshotProgressCursor.decode(bson.deserialize(encoded) as any) +); + +/** + * Decodes an (optional) encoded ConvexSnapshotProgressCursor. + * @default {DEFAULT_CONVEX_SNAPSHOT_CURSOR} + * @throws {ReplicationAssertionError} if the value could not be decoded + */ +export function decodeSnapshotProgressCursor(value: Uint8Array | null | undefined): ConvexSnapshotProgressCursor { + if (value == null) { + return DEFAULT_CONVEX_SNAPSHOT_CURSOR; + } + + try { + return BinaryConvexSnapshotProgressCursor.decode(value); + } catch (error) { + throw new ReplicationAssertionError( + `Convex snapshot progress cursor is not valid JSON: ${error instanceof Error ? error.message : `${error}`}` + ); + } +} diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index d4aa2dd5d..b37edc6ec 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -29,6 +29,7 @@ import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { lsnToDate, parseConvexLsn, toConvexLsn, ZERO_LSN } from '../common/ConvexLSN.js'; import { extractProperties, toSqliteInputRow } from '../common/convex-to-sqlite.js'; import { ConvexConnectionManager } from './ConvexConnectionManager.js'; +import { BinaryConvexSnapshotProgressCursor, decodeSnapshotProgressCursor } from './ConvexSnapshotProgresCursor.js'; export interface ConvexStreamOptions { connections: ConvexConnectionManager; @@ -278,6 +279,7 @@ export class ConvexStream { logger: this.logger, zeroLSN: ZERO_LSN, defaultSchema: this.defaultSchema, + // TODO(steven) check this storeCurrentData: false, skipExistingRows: true }); @@ -398,7 +400,7 @@ export class ConvexStream { latestTable = await batch.updateTableProgress(latestTable, { replicatedCount, totalEstimatedCount: -1, - lastKey: encodeSnapshotProgressCursor({ + lastKey: BinaryConvexSnapshotProgressCursor.encode({ cursor: pageCursor, finished: !page.hasMore }) @@ -661,67 +663,3 @@ function readTableName(change: ConvexRawDocument): string | null { } return table; } - -interface ConvexSnapshotProgressCursor { - cursor: string | null; - finished: boolean; -} - -const SNAPSHOT_PROGRESS_PREFIX = 'convex-snapshot-progress:'; - -function encodeSnapshotProgressCursor(progress: ConvexSnapshotProgressCursor): Uint8Array | null { - if (!progress.finished && progress.cursor == null) { - return null; - } - - if (!progress.finished) { - return Buffer.from(progress.cursor!, 'utf8'); - } - - return Buffer.from(`${SNAPSHOT_PROGRESS_PREFIX}${JSON.stringify(progress)}`, 'utf8'); -} - -function decodeSnapshotProgressCursor(value: Uint8Array | null | undefined): ConvexSnapshotProgressCursor { - if (value == null) { - return { - cursor: null, - finished: false - }; - } - - const serialized = Buffer.from(value).toString('utf8'); - if (!serialized.startsWith(SNAPSHOT_PROGRESS_PREFIX)) { - return { - cursor: serialized, - finished: false - }; - } - - let parsed: unknown; - try { - parsed = JSON.parse(serialized.slice(SNAPSHOT_PROGRESS_PREFIX.length)); - } catch (error) { - throw new ReplicationAssertionError( - `Convex snapshot progress cursor is not valid JSON: ${error instanceof Error ? error.message : `${error}`}` - ); - } - - if (typeof parsed != 'object' || parsed == null || Array.isArray(parsed)) { - throw new ReplicationAssertionError('Convex snapshot progress cursor must decode to an object'); - } - - const parsedProgress = parsed as { cursor?: unknown; finished?: unknown }; - const cursor = parsedProgress.cursor; - const finished = parsedProgress.finished; - if (cursor != null && typeof cursor != 'string') { - throw new ReplicationAssertionError('Convex snapshot progress cursor must contain a string cursor or null'); - } - if (typeof finished != 'boolean') { - throw new ReplicationAssertionError('Convex snapshot progress cursor must contain a boolean finished flag'); - } - - return { - cursor: cursor ?? null, - finished - }; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 12b702ae7..57d1e69be 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -207,6 +207,9 @@ importers: '@powersync/service-types': specifier: workspace:* version: link:../../packages/types + bson: + specifier: ^6.10.4 + version: 6.10.4 ts-codec: specifier: ^1.3.0 version: 1.3.0 @@ -732,7 +735,7 @@ importers: version: link:../sync-rules vitest: specifier: 'catalog:' - version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.1(@types/node@25.5.0)(yaml@2.8.2)) + version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)) devDependencies: '@opentelemetry/sdk-metrics': specifier: ^1.30.1 @@ -1521,9 +1524,6 @@ packages: peerDependencies: '@opentelemetry/api': ^1.1.0 - '@oxc-project/types@0.120.0': - resolution: {integrity: sha512-k1YNu55DuvAip/MGE1FTsIuU3FUCn6v/ujG9V7Nq5Df/kX2CWb13hhwD0lmJGMGqE+bE1MXvv9SZVnMzEXlWcg==} - '@oxc-project/types@0.126.0': resolution: {integrity: sha512-oGfVtjAgwQVVpfBrbtk4e1XDyWHRFta6BS3GWVzrF8xYBT2VGQAk39yJS/wFSMrZqoiCU4oghT3Ch0HaHGIHcQ==} @@ -1652,73 +1652,36 @@ packages: '@protobufjs/utf8@1.1.0': resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} - '@rolldown/binding-android-arm64@1.0.0-rc.10': - resolution: {integrity: sha512-jOHxwXhxmFKuXztiu1ORieJeTbx5vrTkcOkkkn2d35726+iwhrY1w/+nYY/AGgF12thg33qC3R1LMBF5tHTZHg==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [arm64] - os: [android] - '@rolldown/binding-android-arm64@1.0.0-rc.16': resolution: {integrity: sha512-rhY3k7Bsae9qQfOtph2Pm2jZEA+s8Gmjoz4hhmx70K9iMQ/ddeae+xhRQcM5IuVx5ry1+bGfkvMn7D6MJggVSA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [android] - '@rolldown/binding-darwin-arm64@1.0.0-rc.10': - resolution: {integrity: sha512-gED05Teg/vtTZbIJBc4VNMAxAFDUPkuO/rAIyyxZjTj1a1/s6z5TII/5yMGZ0uLRCifEtwUQn8OlYzuYc0m70w==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [arm64] - os: [darwin] - '@rolldown/binding-darwin-arm64@1.0.0-rc.16': resolution: {integrity: sha512-rNz0yK078yrNn3DrdgN+PKiMOW8HfQ92jQiXxwX8yW899ayV00MLVdaCNeVBhG/TbH3ouYVObo8/yrkiectkcQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [darwin] - '@rolldown/binding-darwin-x64@1.0.0-rc.10': - resolution: {integrity: sha512-rI15NcM1mA48lqrIxVkHfAqcyFLcQwyXWThy+BQ5+mkKKPvSO26ir+ZDp36AgYoYVkqvMcdS8zOE6SeBsR9e8A==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [x64] - os: [darwin] - '@rolldown/binding-darwin-x64@1.0.0-rc.16': resolution: {integrity: sha512-r/OmdR00HmD4i79Z//xO06uEPOq5hRXdhw7nzkxQxwSavs3PSHa1ijntdpOiZ2mzOQ3fVVu8C1M19FoNM+dMUQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [darwin] - '@rolldown/binding-freebsd-x64@1.0.0-rc.10': - resolution: {integrity: sha512-XZRXHdTa+4ME1MuDVp021+doQ+z6Ei4CCFmNc5/sKbqb8YmkiJdj8QKlV3rCI0AJtAeSB5n0WGPuJWNL9p/L2w==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [x64] - os: [freebsd] - '@rolldown/binding-freebsd-x64@1.0.0-rc.16': resolution: {integrity: sha512-KcRE5w8h0OnjUatG8pldyD14/CQ5Phs1oxfR+3pKDjboHRo9+MkqQaiIZlZRpsxC15paeXme/I127tUa9TXJ6g==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [freebsd] - '@rolldown/binding-linux-arm-gnueabihf@1.0.0-rc.10': - resolution: {integrity: sha512-R0SQMRluISSLzFE20sPWYHVmJdDQnRyc/FzSCN72BqQmh2SOZUFG+N3/vBZpR4C6WpEUVYJLrYUXaj43sJsNLA==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [arm] - os: [linux] - '@rolldown/binding-linux-arm-gnueabihf@1.0.0-rc.16': resolution: {integrity: sha512-bT0guA1bpxEJ/ZhTRniQf7rNF8ybvXOuWbNIeLABaV5NGjx4EtOWBTSRGWFU9ZWVkPOZ+HNFP8RMcBokBiZ0Kg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [linux] - '@rolldown/binding-linux-arm64-gnu@1.0.0-rc.10': - resolution: {integrity: sha512-Y1reMrV/o+cwpduYhJuOE3OMKx32RMYCidf14y+HssARRmhDuWXJ4yVguDg2R/8SyyGNo+auzz64LnPK9Hq6jg==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [arm64] - os: [linux] - libc: [glibc] - '@rolldown/binding-linux-arm64-gnu@1.0.0-rc.16': resolution: {integrity: sha512-+tHktCHWV8BDQSjemUqm/Jl/TPk3QObCTIjmdDy/nlupcujZghmKK2962LYrqFpWu+ai01AN/REOH3NEpqvYQg==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1726,13 +1689,6 @@ packages: os: [linux] libc: [glibc] - '@rolldown/binding-linux-arm64-musl@1.0.0-rc.10': - resolution: {integrity: sha512-vELN+HNb2IzuzSBUOD4NHmP9yrGwl1DVM29wlQvx1OLSclL0NgVWnVDKl/8tEks79EFek/kebQKnNJkIAA4W2g==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [arm64] - os: [linux] - libc: [musl] - '@rolldown/binding-linux-arm64-musl@1.0.0-rc.16': resolution: {integrity: sha512-3fPzdREH806oRLxpTWW1Gt4tQHs0TitZFOECB2xzCFLPKnSOy90gwA7P29cksYilFO6XVRY1kzga0cL2nRjKPg==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1740,13 +1696,6 @@ packages: os: [linux] libc: [musl] - '@rolldown/binding-linux-ppc64-gnu@1.0.0-rc.10': - resolution: {integrity: sha512-ZqrufYTgzxbHwpqOjzSsb0UV/aV2TFIY5rP8HdsiPTv/CuAgCRjM6s9cYFwQ4CNH+hf9Y4erHW1GjZuZ7WoI7w==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [ppc64] - os: [linux] - libc: [glibc] - '@rolldown/binding-linux-ppc64-gnu@1.0.0-rc.16': resolution: {integrity: sha512-EKwI1tSrLs7YVw+JPJT/G2dJQ1jl9qlTTTEG0V2Ok/RdOenRfBw2PQdLPyjhIu58ocdBfP7vIRN/pvMsPxs/AQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1754,13 +1703,6 @@ packages: os: [linux] libc: [glibc] - '@rolldown/binding-linux-s390x-gnu@1.0.0-rc.10': - resolution: {integrity: sha512-gSlmVS1FZJSRicA6IyjoRoKAFK7IIHBs7xJuHRSmjImqk3mPPWbR7RhbnfH2G6bcmMEllCt2vQ/7u9e6bBnByg==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [s390x] - os: [linux] - libc: [glibc] - '@rolldown/binding-linux-s390x-gnu@1.0.0-rc.16': resolution: {integrity: sha512-Uknladnb3Sxqu6SEcqBldQyJUpk8NleooZEc0MbRBJ4inEhRYWZX0NJu12vNf2mqAq7gsofAxHrGghiUYjhaLQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1768,13 +1710,6 @@ packages: os: [linux] libc: [glibc] - '@rolldown/binding-linux-x64-gnu@1.0.0-rc.10': - resolution: {integrity: sha512-eOCKUpluKgfObT2pHjztnaWEIbUabWzk3qPZ5PuacuPmr4+JtQG4k2vGTY0H15edaTnicgU428XW/IH6AimcQw==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [x64] - os: [linux] - libc: [glibc] - '@rolldown/binding-linux-x64-gnu@1.0.0-rc.16': resolution: {integrity: sha512-FIb8+uG49sZBtLTn+zt1AJ20TqVcqWeSIyoVt0or7uAWesgKaHbiBh6OpA/k9v0LTt+PTrb1Lao133kP4uVxkg==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1782,13 +1717,6 @@ packages: os: [linux] libc: [glibc] - '@rolldown/binding-linux-x64-musl@1.0.0-rc.10': - resolution: {integrity: sha512-Xdf2jQbfQowJnLcgYfD/m0Uu0Qj5OdxKallD78/IPPfzaiaI4KRAwZzHcKQ4ig1gtg1SuzC7jovNiM2TzQsBXA==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [x64] - os: [linux] - libc: [musl] - '@rolldown/binding-linux-x64-musl@1.0.0-rc.16': resolution: {integrity: sha512-RuERhF9/EgWxZEXYWCOaViUWHIboceK4/ivdtQ3R0T44NjLkIIlGIAVAuCddFxsZ7vnRHtNQUrt2vR2n2slB2w==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1796,55 +1724,29 @@ packages: os: [linux] libc: [musl] - '@rolldown/binding-openharmony-arm64@1.0.0-rc.10': - resolution: {integrity: sha512-o1hYe8hLi1EY6jgPFyxQgQ1wcycX+qz8eEbVmot2hFkgUzPxy9+kF0u0NIQBeDq+Mko47AkaFFaChcvZa9UX9Q==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [arm64] - os: [openharmony] - '@rolldown/binding-openharmony-arm64@1.0.0-rc.16': resolution: {integrity: sha512-mXcXnvd9GpazCxeUCCnZ2+YF7nut+ZOEbE4GtaiPtyY6AkhZWbK70y1KK3j+RDhjVq5+U8FySkKRb/+w0EeUwA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [openharmony] - '@rolldown/binding-wasm32-wasi@1.0.0-rc.10': - resolution: {integrity: sha512-Ugv9o7qYJudqQO5Y5y2N2SOo6S4WiqiNOpuQyoPInnhVzCY+wi/GHltcLHypG9DEUYMB0iTB/huJrpadiAcNcA==} - engines: {node: '>=14.0.0'} - cpu: [wasm32] - '@rolldown/binding-wasm32-wasi@1.0.0-rc.16': resolution: {integrity: sha512-3Q2KQxnC8IJOLqXmUMoYwyIPZU9hzRbnHaoV3Euz+VVnjZKcY8ktnNP8T9R4/GGQtb27C/UYKABxesKWb8lsvQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [wasm32] - '@rolldown/binding-win32-arm64-msvc@1.0.0-rc.10': - resolution: {integrity: sha512-7UODQb4fQUNT/vmgDZBl3XOBAIOutP5R3O/rkxg0aLfEGQ4opbCgU5vOw/scPe4xOqBwL9fw7/RP1vAMZ6QlAQ==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [arm64] - os: [win32] - '@rolldown/binding-win32-arm64-msvc@1.0.0-rc.16': resolution: {integrity: sha512-tj7XRemQcOcFwv7qhpUxMTBbI5mWMlE4c1Omhg5+h8GuLXzyj8HviYgR+bB2DMDgRqUE+jiDleqSCRjx4aYk/Q==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [win32] - '@rolldown/binding-win32-x64-msvc@1.0.0-rc.10': - resolution: {integrity: sha512-PYxKHMVHOb5NJuDL53vBUl1VwUjymDcYI6rzpIni0C9+9mTiJedvUxSk7/RPp7OOAm3v+EjgMu9bIy3N6b408w==} - engines: {node: ^20.19.0 || >=22.12.0} - cpu: [x64] - os: [win32] - '@rolldown/binding-win32-x64-msvc@1.0.0-rc.16': resolution: {integrity: sha512-PH5DRZT+F4f2PTXRXR8uJxnBq2po/xFtddyabTJVJs/ZYVHqXPEgNIr35IHTEa6bpa0Q8Awg+ymkTaGnKITw4g==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [win32] - '@rolldown/pluginutils@1.0.0-rc.10': - resolution: {integrity: sha512-UkVDEFk1w3mveXeKgaTuYfKWtPbvgck1dT8TUG3bnccrH0XtLTuAyfCoks4Q/M5ZGToSVJTIQYCzy2g/atAOeg==} - '@rolldown/pluginutils@1.0.0-rc.16': resolution: {integrity: sha512-45+YtqxLYKDWQouLKCrpIZhke+nXxhsw+qAHVzHDVwttyBlHNBVs2K25rDXrZzhpTp9w1FlAlvweV1H++fdZoA==} @@ -3361,10 +3263,6 @@ packages: resolution: {integrity: sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==} engines: {node: '>=8.6'} - picomatch@4.0.3: - resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==} - engines: {node: '>=12'} - picomatch@4.0.4: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} @@ -3387,10 +3285,6 @@ packages: resolution: {integrity: sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==} engines: {node: ^10 || ^12 || >=14} - postcss@8.5.8: - resolution: {integrity: sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==} - engines: {node: ^10 || ^12 || >=14} - postgres-array@2.0.0: resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} engines: {node: '>=4'} @@ -3561,11 +3455,6 @@ packages: rfdc@1.4.1: resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} - rolldown@1.0.0-rc.10: - resolution: {integrity: sha512-q7j6vvarRFmKpgJUT8HCAUljkgzEp4LAhPlJUvQhA5LA1SUL36s5QCysMutErzL3EbNOZOkoziSx9iZC4FddKA==} - engines: {node: ^20.19.0 || >=22.12.0} - hasBin: true - rolldown@1.0.0-rc.16: resolution: {integrity: sha512-rzi5WqKzEZw3SooTt7cgm4eqIoujPIyGcJNGFL7iPEuajQw7vxMHUkXylu4/vhCkJGXsgRmxqMKXUpT6FEgl0g==} engines: {node: ^20.19.0 || >=22.12.0} @@ -3964,49 +3853,6 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} - vite@8.0.1: - resolution: {integrity: sha512-wt+Z2qIhfFt85uiyRt5LPU4oVEJBXj8hZNWKeqFG4gRG/0RaRGJ7njQCwzFVjO+v4+Ipmf5CY7VdmZRAYYBPHw==} - engines: {node: ^20.19.0 || >=22.12.0} - hasBin: true - peerDependencies: - '@types/node': ^20.19.0 || >=22.12.0 - '@vitejs/devtools': ^0.1.0 - esbuild: ^0.27.0 - jiti: '>=1.21.0' - less: ^4.0.0 - sass: ^1.70.0 - sass-embedded: ^1.70.0 - stylus: '>=0.54.8' - sugarss: ^5.0.0 - terser: ^5.16.0 - tsx: ^4.8.1 - yaml: ^2.4.2 - peerDependenciesMeta: - '@types/node': - optional: true - '@vitejs/devtools': - optional: true - esbuild: - optional: true - jiti: - optional: true - less: - optional: true - sass: - optional: true - sass-embedded: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - tsx: - optional: true - yaml: - optional: true - vite@8.0.9: resolution: {integrity: sha512-t7g7GVRpMXjNpa67HaVWI/8BWtdVIQPCL2WoozXXA7LBGEFK4AkkKkHx2hAQf5x1GZSlcmEDPkVLSGahxnEEZw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4168,11 +4014,6 @@ packages: yallist@2.1.2: resolution: {integrity: sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==} - yaml@2.8.2: - resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==} - engines: {node: '>= 14.6'} - hasBin: true - yaml@2.8.3: resolution: {integrity: sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==} engines: {node: '>= 14.6'} @@ -5084,8 +4925,6 @@ snapshots: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 2.0.1(@opentelemetry/api@1.9.0) - '@oxc-project/types@0.120.0': {} - '@oxc-project/types@0.126.0': {} '@pinojs/redact@0.4.0': {} @@ -5241,83 +5080,42 @@ snapshots: '@protobufjs/utf8@1.1.0': {} - '@rolldown/binding-android-arm64@1.0.0-rc.10': - optional: true - '@rolldown/binding-android-arm64@1.0.0-rc.16': optional: true - '@rolldown/binding-darwin-arm64@1.0.0-rc.10': - optional: true - '@rolldown/binding-darwin-arm64@1.0.0-rc.16': optional: true - '@rolldown/binding-darwin-x64@1.0.0-rc.10': - optional: true - '@rolldown/binding-darwin-x64@1.0.0-rc.16': optional: true - '@rolldown/binding-freebsd-x64@1.0.0-rc.10': - optional: true - '@rolldown/binding-freebsd-x64@1.0.0-rc.16': optional: true - '@rolldown/binding-linux-arm-gnueabihf@1.0.0-rc.10': - optional: true - '@rolldown/binding-linux-arm-gnueabihf@1.0.0-rc.16': optional: true - '@rolldown/binding-linux-arm64-gnu@1.0.0-rc.10': - optional: true - '@rolldown/binding-linux-arm64-gnu@1.0.0-rc.16': optional: true - '@rolldown/binding-linux-arm64-musl@1.0.0-rc.10': - optional: true - '@rolldown/binding-linux-arm64-musl@1.0.0-rc.16': optional: true - '@rolldown/binding-linux-ppc64-gnu@1.0.0-rc.10': - optional: true - '@rolldown/binding-linux-ppc64-gnu@1.0.0-rc.16': optional: true - '@rolldown/binding-linux-s390x-gnu@1.0.0-rc.10': - optional: true - '@rolldown/binding-linux-s390x-gnu@1.0.0-rc.16': optional: true - '@rolldown/binding-linux-x64-gnu@1.0.0-rc.10': - optional: true - '@rolldown/binding-linux-x64-gnu@1.0.0-rc.16': optional: true - '@rolldown/binding-linux-x64-musl@1.0.0-rc.10': - optional: true - '@rolldown/binding-linux-x64-musl@1.0.0-rc.16': optional: true - '@rolldown/binding-openharmony-arm64@1.0.0-rc.10': - optional: true - '@rolldown/binding-openharmony-arm64@1.0.0-rc.16': optional: true - '@rolldown/binding-wasm32-wasi@1.0.0-rc.10': - dependencies: - '@napi-rs/wasm-runtime': 1.1.1 - optional: true - '@rolldown/binding-wasm32-wasi@1.0.0-rc.16': dependencies: '@emnapi/core': 1.9.2 @@ -5325,20 +5123,12 @@ snapshots: '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.9.2)(@emnapi/runtime@1.9.2) optional: true - '@rolldown/binding-win32-arm64-msvc@1.0.0-rc.10': - optional: true - '@rolldown/binding-win32-arm64-msvc@1.0.0-rc.16': optional: true - '@rolldown/binding-win32-x64-msvc@1.0.0-rc.10': - optional: true - '@rolldown/binding-win32-x64-msvc@1.0.0-rc.16': optional: true - '@rolldown/pluginutils@1.0.0-rc.10': {} - '@rolldown/pluginutils@1.0.0-rc.16': {} '@sec-ant/readable-stream@0.4.1': {} @@ -5590,14 +5380,6 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.0.3 - '@vitest/mocker@4.1.0(vite@8.0.1(@types/node@25.5.0)(yaml@2.8.2))': - dependencies: - '@vitest/spy': 4.1.0 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 8.0.1(@types/node@25.5.0)(yaml@2.8.2) - '@vitest/mocker@4.1.0(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.0 @@ -6817,8 +6599,6 @@ snapshots: picomatch@2.3.2: {} - picomatch@4.0.3: {} - picomatch@4.0.4: {} pify@4.0.1: {} @@ -6849,12 +6629,6 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - postcss@8.5.8: - dependencies: - nanoid: 3.3.11 - picocolors: 1.1.1 - source-map-js: 1.2.1 - postgres-array@2.0.0: {} postgres-bytea@1.0.0: {} @@ -7041,27 +6815,6 @@ snapshots: rfdc@1.4.1: {} - rolldown@1.0.0-rc.10: - dependencies: - '@oxc-project/types': 0.120.0 - '@rolldown/pluginutils': 1.0.0-rc.10 - optionalDependencies: - '@rolldown/binding-android-arm64': 1.0.0-rc.10 - '@rolldown/binding-darwin-arm64': 1.0.0-rc.10 - '@rolldown/binding-darwin-x64': 1.0.0-rc.10 - '@rolldown/binding-freebsd-x64': 1.0.0-rc.10 - '@rolldown/binding-linux-arm-gnueabihf': 1.0.0-rc.10 - '@rolldown/binding-linux-arm64-gnu': 1.0.0-rc.10 - '@rolldown/binding-linux-arm64-musl': 1.0.0-rc.10 - '@rolldown/binding-linux-ppc64-gnu': 1.0.0-rc.10 - '@rolldown/binding-linux-s390x-gnu': 1.0.0-rc.10 - '@rolldown/binding-linux-x64-gnu': 1.0.0-rc.10 - '@rolldown/binding-linux-x64-musl': 1.0.0-rc.10 - '@rolldown/binding-openharmony-arm64': 1.0.0-rc.10 - '@rolldown/binding-wasm32-wasi': 1.0.0-rc.10 - '@rolldown/binding-win32-arm64-msvc': 1.0.0-rc.10 - '@rolldown/binding-win32-x64-msvc': 1.0.0-rc.10 - rolldown@1.0.0-rc.16: dependencies: '@oxc-project/types': 0.126.0 @@ -7432,18 +7185,6 @@ snapshots: vary@1.1.2: {} - vite@8.0.1(@types/node@25.5.0)(yaml@2.8.2): - dependencies: - lightningcss: 1.32.0 - picomatch: 4.0.3 - postcss: 8.5.8 - rolldown: 1.0.0-rc.10 - tinyglobby: 0.2.15 - optionalDependencies: - '@types/node': 25.5.0 - fsevents: 2.3.3 - yaml: 2.8.2 - vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3): dependencies: lightningcss: 1.32.0 @@ -7497,35 +7238,6 @@ snapshots: transitivePeerDependencies: - msw - vitest@4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.1(@types/node@25.5.0)(yaml@2.8.2)): - dependencies: - '@vitest/expect': 4.1.0 - '@vitest/mocker': 4.1.0(vite@8.0.1(@types/node@25.5.0)(yaml@2.8.2)) - '@vitest/pretty-format': 4.1.0 - '@vitest/runner': 4.1.0 - '@vitest/snapshot': 4.1.0 - '@vitest/spy': 4.1.0 - '@vitest/utils': 4.1.0 - es-module-lexer: 2.0.0 - expect-type: 1.3.0 - magic-string: 0.30.21 - obug: 2.1.1 - pathe: 2.0.3 - picomatch: 4.0.3 - std-env: 4.0.0 - tinybench: 2.9.0 - tinyexec: 1.0.2 - tinyglobby: 0.2.15 - tinyrainbow: 3.0.3 - vite: 8.0.1(@types/node@25.5.0)(yaml@2.8.2) - why-is-node-running: 2.3.0 - optionalDependencies: - '@opentelemetry/api': 1.9.0 - '@types/node': 25.5.0 - '@vitest/ui': 4.1.0(vitest@4.1.0) - transitivePeerDependencies: - - msw - vitest@4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.0 @@ -7636,9 +7348,6 @@ snapshots: yallist@2.1.2: {} - yaml@2.8.2: - optional: true - yaml@2.8.3: {} yargs-parser@21.1.1: {} From 42ccc8cf1514c94f09f7e340bba803d8d7aeb040 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 5 May 2026 18:30:23 +0200 Subject: [PATCH 48/86] wip integration tests and resumable replication tests --- modules/module-convex/.gitignore | 2 + modules/module-convex/README.md | 13 + modules/module-convex/convex/.gitignore | 1 + modules/module-convex/convex/lists.ts | 46 ++ .../convex/powersync_checkpoints.ts | 14 + modules/module-convex/convex/schema.ts | 100 ++++ modules/module-convex/convex/todos.ts | 58 +++ .../module-convex/convex/utils/collections.ts | 17 + .../scripts/convex-sanity-check.ts | 12 + modules/module-convex/package.json | 8 +- .../src/client/ConvexApiClient.ts | 10 +- .../module-convex/src/module/ConvexModule.ts | 4 +- .../src/replication/ConvexStream.ts | 21 +- modules/module-convex/src/types/types.ts | 20 +- modules/module-convex/test/src/env.ts | 24 +- .../integration/resuming_snapshots.test.ts | 165 ++++++ .../src/test-utils/ConvexStreamTestContext.ts | 114 ++++ .../module-convex/test/src/test-utils/util.ts | 120 +++++ modules/module-convex/test/tsconfig.json | 8 +- modules/module-convex/vitest.config.ts | 11 +- modules/module-postgres/vitest.config.ts | 9 +- packages/service-core-tests/package.json | 1 + .../test-utils/AbstractStreamTestContext.ts | 180 +++++++ .../src/test-utils/test-utils-index.ts | 1 + pnpm-lock.yaml | 489 +++++++++++++++++- 25 files changed, 1397 insertions(+), 51 deletions(-) create mode 100644 modules/module-convex/.gitignore create mode 100644 modules/module-convex/convex/.gitignore create mode 100644 modules/module-convex/convex/lists.ts create mode 100644 modules/module-convex/convex/powersync_checkpoints.ts create mode 100644 modules/module-convex/convex/schema.ts create mode 100644 modules/module-convex/convex/todos.ts create mode 100644 modules/module-convex/convex/utils/collections.ts create mode 100644 modules/module-convex/development/scripts/convex-sanity-check.ts create mode 100644 modules/module-convex/test/src/integration/resuming_snapshots.test.ts create mode 100644 modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts create mode 100644 modules/module-convex/test/src/test-utils/util.ts create mode 100644 packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts diff --git a/modules/module-convex/.gitignore b/modules/module-convex/.gitignore new file mode 100644 index 000000000..8c5fbb9ce --- /dev/null +++ b/modules/module-convex/.gitignore @@ -0,0 +1,2 @@ + +.env.local diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 98ef66271..235e42678 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -18,6 +18,19 @@ replication: 1. Simplest is to run the convex demo in the self-host-demo [repo](https://github.com/powersync-ja/self-host-demo) +# Development + +Run the `dev:convex` script to start a local Convex development instance. + +```bash +# In the modules/module-convex folder +pnpm run dev:convex + +# OR +# From the repo root +pnpm run -C modules/modules-convex dev:convex +``` + # Technical notes The content below is written in an agents.md style describing the behavior of `module-convex`. diff --git a/modules/module-convex/convex/.gitignore b/modules/module-convex/convex/.gitignore new file mode 100644 index 000000000..94f0f0d9a --- /dev/null +++ b/modules/module-convex/convex/.gitignore @@ -0,0 +1 @@ +_generated \ No newline at end of file diff --git a/modules/module-convex/convex/lists.ts b/modules/module-convex/convex/lists.ts new file mode 100644 index 000000000..6e605210a --- /dev/null +++ b/modules/module-convex/convex/lists.ts @@ -0,0 +1,46 @@ +import { v } from 'convex/values'; +import { mutation, query } from './_generated/server.js'; +import schema from './schema.js'; + +const BATCH_SIZE = 4000; + +export const get = query({ + args: {}, + handler: async (ctx) => { + return await ctx.db.query('lists').collect(); + } +}); + +/** + * Deletes a batch of items. + * Convex limits the number of ops in a mutation/transaction to 16_000. + * This will return the number of items deleted. + * Rerun this till completion if all items should be deleted. + */ +export const deleteBatch = mutation({ + args: { + batch_size: v.optional(v.number()) + }, + handler: async (ctx, args) => { + const batchSize = args.batch_size ?? BATCH_SIZE; + const lists = await ctx.db.query('lists').take(batchSize); + for (const list of lists) { + await ctx.db.delete(list._id); + } + return lists.length; + } +}); + +export const createBatch = mutation({ + args: { + lists: v.array(schema.tables.lists.validator) + }, + handler: async (ctx, args) => { + const ids = []; + for (const list of args.lists) { + const id = await ctx.db.insert('lists', list); + ids.push(id); + } + return ids; + } +}); diff --git a/modules/module-convex/convex/powersync_checkpoints.ts b/modules/module-convex/convex/powersync_checkpoints.ts new file mode 100644 index 000000000..7ad660756 --- /dev/null +++ b/modules/module-convex/convex/powersync_checkpoints.ts @@ -0,0 +1,14 @@ +import { mutation } from './_generated/server.js'; + +export const createCheckpoint = mutation({ + args: {}, + handler: async (ctx) => { + const existing = await ctx.db.query('powersync_checkpoints').first(); + + if (existing) { + await ctx.db.patch(existing._id, { last_updated: Date.now() }); + } else { + await ctx.db.insert('powersync_checkpoints', { last_updated: Date.now() }); + } + } +}); diff --git a/modules/module-convex/convex/schema.ts b/modules/module-convex/convex/schema.ts new file mode 100644 index 000000000..d0346b376 --- /dev/null +++ b/modules/module-convex/convex/schema.ts @@ -0,0 +1,100 @@ +import { authTables } from '@convex-dev/auth/server'; +import { defineSchema, defineTable } from 'convex/server'; +import { v } from 'convex/values'; + +export default defineSchema({ + ...authTables, + lists: defineTable({ + created_at: v.optional(v.string()), + name: v.string(), + owner_id: v.optional(v.string()), + tags: v.optional(v.array(v.string())), + attributes: v.optional(v.record(v.string(), v.string())), + settings: v.optional( + v.object({ + theme: v.string(), + color: v.string(), + is_public: v.boolean() + }) + ), + + // External stable key from PowerSync writes. + uuid: v.string(), + // Legacy fields kept for backwards compatibility with existing data. + owner: v.optional(v.string()), + archived: v.optional(v.number()) + }).index('by_uuid', ['uuid']), + + todos: defineTable({ + // Basic fields + /** + * Due to ID mapping, we cant require a strict Convex `id` ID field - which + * correlates to a matching todo record here. + * This value will be the local-first UUID. + */ + uuid: v.string(), + created_at: v.optional(v.string()), + completed_at: v.optional(v.union(v.null(), v.string())), + description: v.string(), + list_id: v.id('lists'), + /** + * Local-first version of list_id + */ + list_uuid: v.string(), + + // All Convex datatypes for stress testing + // String types + title: v.optional(v.string()), + notes: v.optional(v.string()), + category: v.optional(v.string()), + + // Number types + priority: v.optional(v.number()), + estimated_hours: v.optional(v.float64()), + progress_percentage: v.optional(v.float64()), + + // Boolean types + is_urgent: v.optional(v.boolean()), + is_private: v.optional(v.boolean()), + has_attachments: v.optional(v.boolean()), + + // Array types + tags: v.optional(v.array(v.string())), + dependencies: v.optional(v.array(v.id('todos'))), + assigned_users: v.optional(v.array(v.string())), + + // Object types + metadata: v.optional(v.record(v.string(), v.any())), + custom_fields: v.optional(v.record(v.string(), v.union(v.string(), v.number(), v.boolean()))), + + // ID references + parent_task_id: v.optional(v.id('todos')), + project_id: v.optional(v.id('lists')), + + // Union types + status: v.optional( + v.union(v.literal('pending'), v.literal('in_progress'), v.literal('completed'), v.literal('cancelled')) + ), + difficulty: v.optional(v.union(v.literal('easy'), v.literal('medium'), v.literal('hard'))), + + // Null handling + archived_at: v.optional(v.union(v.null(), v.string())), + deleted_by: v.optional(v.union(v.null(), v.string())), + + // Legacy fields kept for backwards compatibility + completed: v.optional(v.number()), + created_by: v.optional(v.union(v.null(), v.string())), + completed_by: v.optional(v.union(v.null(), v.string())), + photo_id: v.optional(v.union(v.null(), v.string())), + owner_id: v.optional(v.string()) + }) + .index('by_uuid', ['uuid']) + .index('by_list_id', ['list_id']) + .index('by_status', ['status']) + .index('by_priority', ['priority']) + .index('by_project', ['project_id']), + + powersync_checkpoints: defineTable({ + last_updated: v.float64() + }) +}); diff --git a/modules/module-convex/convex/todos.ts b/modules/module-convex/convex/todos.ts new file mode 100644 index 000000000..ba7988806 --- /dev/null +++ b/modules/module-convex/convex/todos.ts @@ -0,0 +1,58 @@ +import { v } from 'convex/values'; +import { mutation, query } from './_generated/server.js'; +import schema from './schema.js'; +import { findListByUuid } from './utils/collections.js'; + +const BATCH_SIZE = 4000; + +export const get = query({ + args: {}, + handler: async (ctx) => { + return await ctx.db.query('todos').collect(); + } +}); + +/** + * Deletes a batch of items. + * Convex limits the number of ops in a mutation/transaction to 16_000. + * This will return the number of items deleted. + * Rerun this till completion if all items should be deleted. + */ +export const deleteBatch = mutation({ + args: { + batch_size: v.optional(v.number()) + }, + handler: async (ctx, args) => { + const batchSize = args.batch_size ?? BATCH_SIZE; + const todos = await ctx.db.query('todos').take(batchSize); + for (const todo of todos) { + await ctx.db.delete(todo._id); + } + return todos.length; + } +}); + +export const createBatch = mutation({ + args: { + // We don't require clients usually to track the Convex list_id, they work with list_uuid + todos: v.array(schema.tables.todos.validator.omit('list_id')) + }, + handler: async (ctx, args) => { + const { db } = ctx; + const ids = []; + for (const todo of args.todos) { + const list = await findListByUuid({ db, uuid: todo.list_uuid }); + if (!list) { + // Continue for simple testing purposes + continue; + } + const id = await ctx.db.insert('todos', { + ...todo, + list_id: list._id + }); + + ids.push(id); + } + return ids; + } +}); diff --git a/modules/module-convex/convex/utils/collections.ts b/modules/module-convex/convex/utils/collections.ts new file mode 100644 index 000000000..fc7bbc8d9 --- /dev/null +++ b/modules/module-convex/convex/utils/collections.ts @@ -0,0 +1,17 @@ +import { DatabaseReader } from '../_generated/server.js'; + +export const findListByUuid = async (params: { db: DatabaseReader; uuid: string }) => { + const { db, uuid } = params; + return db + .query('lists') + .withIndex('by_uuid', (q) => q.eq('uuid', uuid)) + .first(); +}; + +export const findTodoByUuid = async (params: { db: DatabaseReader; uuid: string }) => { + const { db, uuid } = params; + return db + .query('todos') + .withIndex('by_uuid', (q) => q.eq('uuid', uuid)) + .first(); +}; diff --git a/modules/module-convex/development/scripts/convex-sanity-check.ts b/modules/module-convex/development/scripts/convex-sanity-check.ts new file mode 100644 index 000000000..c0fb52a4c --- /dev/null +++ b/modules/module-convex/development/scripts/convex-sanity-check.ts @@ -0,0 +1,12 @@ +import { ConvexHttpClient } from 'convex/browser'; +import * as dotenv from 'dotenv'; +import { api } from '../../convex/_generated/api.js'; + +dotenv.config({ path: '.env.local' }); + +const client = new ConvexHttpClient(process.env['CONVEX_URL']!); + +// List all lists +const lists = await client.query(api.lists.get); +console.log('Fetched the following lists from Convex'); +console.log(JSON.stringify(lists, null, '\t')); diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json index 92ba46ed4..a0eeddbdf 100644 --- a/modules/module-convex/package.json +++ b/modules/module-convex/package.json @@ -13,7 +13,8 @@ "build": "tsc -b", "build:tests": "tsc -b test/tsconfig.json", "clean": "rm -rf ./dist && tsc -b --clean", - "test": "vitest" + "test": "vitest", + "dev:convex": "pnpm exec convex dev" }, "exports": { ".": { @@ -39,6 +40,9 @@ "devDependencies": { "@powersync/service-core-tests": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", - "@powersync/service-module-postgres-storage": "workspace:*" + "@powersync/service-module-postgres-storage": "workspace:*", + "@convex-dev/auth": "^0.0.92", + "convex": "^1.37.0", + "dotenv": "^16.4.5" } } diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index d3e52e03e..5a1474bae 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -1,7 +1,7 @@ -import { setTimeout as delay } from 'timers/promises'; import { JSONBig } from '@powersync/service-jsonbig'; -import { NormalizedConvexConnectionConfig } from '../types/types.js'; +import { setTimeout as delay } from 'timers/promises'; import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; +import { NormalizedConvexConnectionConfig } from '../types/types.js'; const CONVEX_REQUEST_TIMEOUT_MS = 60_000; @@ -192,7 +192,7 @@ export class ConvexApiClient { extraHeaders?: Record; includeJsonFormat?: boolean; }): Promise> { - const url = new URL(options.path, this.config.deploymentUrl); + const url = new URL(options.path, this.config.deployment_url); if (options.includeJsonFormat ?? true) { url.searchParams.set('format', 'json'); } @@ -222,7 +222,7 @@ export class ConvexApiClient { const response = await fetch(url, { method: options.method ?? 'GET', headers: { - Authorization: `Convex ${this.config.deployKey}`, + Authorization: `Convex ${this.config.deploy_key}`, Accept: 'application/json', ...(options.extraHeaders ?? {}) }, @@ -286,7 +286,7 @@ export class ConvexApiClient { return; } - const hostname = new URL(this.config.deploymentUrl).hostname; + const hostname = new URL(this.config.deployment_url).hostname; await new Promise((resolve, reject) => { this.config.lookup!(hostname, {}, (error) => { diff --git a/modules/module-convex/src/module/ConvexModule.ts b/modules/module-convex/src/module/ConvexModule.ts index 7cb5d3381..cae5daffd 100644 --- a/modules/module-convex/src/module/ConvexModule.ts +++ b/modules/module-convex/src/module/ConvexModule.ts @@ -34,7 +34,7 @@ export class ConvexModule extends replication.ReplicationModule | null = null; + constructor(private readonly options: ConvexStreamOptions) { this.storage = options.storage; this.metrics = options.metrics; @@ -84,7 +86,7 @@ export class ConvexStream { } private get pollingIntervalMs() { - return this.connections.config.pollingIntervalMs; + return this.connections.config.polling_interval_ms; } get stopped() { @@ -93,7 +95,10 @@ export class ConvexStream { async replicate() { try { - await this.initReplication(); + this.initialSnapshotPromise = this.initReplication(); + // This pattern/member is used for tests + await this.initialSnapshotPromise; + await this.streamChanges(); } catch (error) { await this.storage.reportError(error); @@ -101,6 +106,18 @@ export class ConvexStream { } } + /** + * After calling replicate(), call this to wait for the initial snapshot to complete. + * + * For tests only. + */ + async waitForInitialSnapshot() { + if (this.initialSnapshotPromise == null) { + throw new ReplicationAssertionError(`Initial snapshot not started yet`); + } + return this.initialSnapshotPromise; + } + async initReplication() { const status = await this.initSlot(); if (!status.needsInitialSync) { diff --git a/modules/module-convex/src/types/types.ts b/modules/module-convex/src/types/types.ts index f3648df51..bbd4f7eb2 100644 --- a/modules/module-convex/src/types/types.ts +++ b/modules/module-convex/src/types/types.ts @@ -8,12 +8,13 @@ export const CONVEX_CONNECTION_TYPE = 'convex' as const; export interface NormalizedConvexConnectionConfig { id: string; tag: string; + type: typeof CONVEX_CONNECTION_TYPE; - deploymentUrl: string; - deployKey: string; + deployment_url: string; + deploy_key: string; - debugApi: boolean; - pollingIntervalMs: number; + debug_api: boolean; + polling_interval_ms: number; lookup?: LookupFunction; } @@ -65,17 +66,18 @@ export function normalizeConnectionConfig(options: ConvexConnectionConfig): Norm return { id: options.id ?? 'default', tag: options.tag ?? 'default', + type: 'convex', - deploymentUrl: deploymentURL.toString().replace(/\/$/, ''), - deployKey: options.deploy_key, + deployment_url: deploymentURL.toString().replace(/\/$/, ''), + deploy_key: options.deploy_key, - debugApi: options.debug_api ?? false, - pollingIntervalMs: options.polling_interval_ms ?? 1_000, + debug_api: options.debug_api ?? false, + polling_interval_ms: options.polling_interval_ms ?? 1_000, lookup }; } export function baseUri(config: ResolvedConvexConnectionConfig) { - return config.deploymentUrl; + return config.deployment_url; } diff --git a/modules/module-convex/test/src/env.ts b/modules/module-convex/test/src/env.ts index 7451deb2d..f519b2e06 100644 --- a/modules/module-convex/test/src/env.ts +++ b/modules/module-convex/test/src/env.ts @@ -1,9 +1,29 @@ import { utils } from '@powersync/lib-services-framework'; +import fs from 'fs'; +import path from 'node:path'; +function obtainDefaultLocalConvexDeployKey(): string | null { + const localConfigPath = path.join(import.meta.dirname, '../../.convex/local/default/config.json'); + try { + if (!fs.existsSync(localConfigPath)) { + return null; + } + + const content = JSON.parse(fs.readFileSync(localConfigPath, 'utf8')); + return content.adminKey; + } catch (ex) { + console.warn(`Could not find local convex config in .convex`); + return null; + } +} export const env = utils.collectEnvironmentVariables({ CONVEX_URL: utils.type.string.default('http://127.0.0.1:3210'), - CONVEX_DEPLOY_KEY: utils.type.string.default(''), + CONVEX_DEPLOY_KEY: utils.type.string.default(obtainDefaultLocalConvexDeployKey() ?? ''), + CONVEX_DEPLOYMENT: utils.type.string.default('anonymous:anonymous-module-convex'), MONGO_TEST_URL: utils.type.string.default('mongodb://127.0.0.1:27017/powersync_test?directConnection=true'), + PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_storage_test'), CI: utils.type.boolean.default('false'), - SLOW_TESTS: utils.type.boolean.default('false') + SLOW_TESTS: utils.type.boolean.default('false'), + TEST_MONGO_STORAGE: utils.type.boolean.default('true'), + TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') }); diff --git a/modules/module-convex/test/src/integration/resuming_snapshots.test.ts b/modules/module-convex/test/src/integration/resuming_snapshots.test.ts new file mode 100644 index 000000000..685da4e71 --- /dev/null +++ b/modules/module-convex/test/src/integration/resuming_snapshots.test.ts @@ -0,0 +1,165 @@ +import { METRICS_HELPER } from '@powersync/service-core-tests'; +import { ReplicationMetric } from '@powersync/service-types'; +import { randomUUID } from 'node:crypto'; +import * as timers from 'node:timers/promises'; +import { describe, expect, test } from 'vitest'; +import { env } from '../env.js'; +import { ConvexStreamTestContext } from '../test-utils/ConvexStreamTestContext.js'; +import { describeWithStorage, StorageVersionTestContext } from '../test-utils/util.js'; + +describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () { + describeWithStorage({ timeout: 240_000 }, function ({ factory, storageVersion }) { + test('resuming initial replication (1)', async () => { + // Stop early - likely to not include deleted row in first replication attempt. + await testResumingReplication(factory, storageVersion, 2000); + }); + test('resuming initial replication (2)', async () => { + // Stop late - likely to include deleted row in first replication attempt. + await testResumingReplication(factory, storageVersion, 8000); + }); + }); +}); + +async function testResumingReplication( + factory: StorageVersionTestContext['factory'], + storageVersion: number, + stopAfter: number +) { + // This tests interrupting and then resuming initial replication. + // We interrupt replication after lists has fully replicated, and + // todos has partially replicated. + // This test relies on interval behavior that is not 100% deterministic: + // 1. We attempt to abort initial replication once a certain number of + // rows have been replicated, but this is not exact. Our only requirement + // is that we have not fully replicated todos yet. + // 2. Order of replication is not deterministic, so which specific rows + // have been / have not been replicated at that point is not deterministic. + // We do allow for some variation in the test results to account for this. + + await using context = await ConvexStreamTestContext.open(factory, { + storageVersion + }); + + await context.updateSyncRules(/* yaml */ `bucket_definitions: + global: + data: + - SELECT uuid as id, * FROM lists + - SELECT uuid as id, * FROM todos`); + + const { backend } = context; + + // Seed the database + // Max number of mutations is batch size supported is 8192 + // Maximum number of reads is 4096 in a single mutation + await backend.client.mutation(backend.api.lists.createBatch, { + lists: Array.from({ length: 2_000 }).map((_, index) => ({ + uuid: randomUUID(), + name: `list-${index}` + })) + }); + await backend.client.mutation(backend.api.todos.createBatch, { + todos: Array.from({ length: 2_000 }).map((_, index) => ({ + uuid: randomUUID(), + list_uuid: randomUUID(), + description: `list-${index}` + })) + }); + + const p = context.replicateSnapshot(); + + let done = false; + + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + try { + (async () => { + while (!done) { + const count = + ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount; + + if (count >= stopAfter) { + break; + } + await timers.setTimeout(1); + } + // This interrupts initial replication + await context.dispose(); + })(); + // This confirms that initial replication was interrupted + await expect(p).rejects.toThrowError(); + done = true; + } finally { + done = true; + } + + // // Bypass the usual "clear db on factory open" step. + // await using context2 = await ConvexStreamTestContext.open(factory, { + // doNotClear: true, + // storageVersion + // }); + + // // This delete should be using one of the ids already replicated + // const { + // rows: [delete1] + // } = await context2.pool.query(`DELETE FROM test_data2 WHERE id = (SELECT id FROM test_data2 LIMIT 1) RETURNING id`); + // // This update should also be using one of the ids already replicated + // const { + // rows: [delete2] + // } = await context2.pool.query( + // `UPDATE test_data2 SET description = 'update1' WHERE id = (SELECT id FROM test_data2 LIMIT 1) RETURNING id` + // ); + // const { + // rows: [delete3] + // } = await context2.pool.query(`INSERT INTO test_data2(description) SELECT 'insert1' RETURNING id`); + + // await context2.loadNextSyncRules(); + // await context2.replicateSnapshot(); + + // const data = await context2.getBucketData('global[]', undefined, {}); + + // const deletedRowOps = data.filter( + // (row) => row.object_type == 'test_data2' && row.object_id === String(delete1.decodeWithoutCustomTypes(0)) + // ); + // const updatedRowOps = data.filter( + // (row) => row.object_type == 'test_data2' && row.object_id === String(delete2.decodeWithoutCustomTypes(0)) + // ); + // const insertedRowOps = data.filter( + // (row) => row.object_type == 'test_data2' && row.object_id === String(delete3.decodeWithoutCustomTypes(0)) + // ); + + // if (deletedRowOps.length != 0) { + // // The deleted row was part of the first replication batch, + // // so it is removed by streaming replication. + // expect(deletedRowOps.length).toEqual(2); + // expect(deletedRowOps[1].op).toEqual('REMOVE'); + // } else { + // // The deleted row was not part of the first replication batch, + // // so it's not in the resulting ops at all. + // } + + // expect(updatedRowOps.length).toBeGreaterThanOrEqual(2); + // // description for the first op could be 'foo' or 'update1'. + // // We only test the final version. + // expect(JSON.parse(updatedRowOps[updatedRowOps.length - 1].data as string).description).toEqual('update1'); + + // expect(insertedRowOps.length).toBeGreaterThanOrEqual(1); + // expect(JSON.parse(insertedRowOps[0].data as string).description).toEqual('insert1'); + // expect(JSON.parse(insertedRowOps[insertedRowOps.length - 1].data as string).description).toEqual('insert1'); + + // // 1000 of test_data1 during first replication attempt. + // // N >= 1000 of test_data2 during first replication attempt. + // // 10000 - N - 1 + 1 of test_data2 during second replication attempt. + // // An additional update during streaming replication (2x total for this row). + // // An additional insert during streaming replication (2x total for this row). + // // If the deleted row was part of the first replication batch, it's removed by streaming replication. + // // This adds 2 ops. + // // We expect this to be 11002 for stopAfter: 2000, and 11004 for stopAfter: 8000. + // // However, this is not deterministic. + // const expectedCount = 11000 - 2 + insertedRowOps.length + updatedRowOps.length + deletedRowOps.length; + // expect(data.length).toEqual(expectedCount); + + // const replicatedCount = + // ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount; + + // // With resumable replication, there should be no need to re-replicate anything. + // expect(replicatedCount).toBeGreaterThanOrEqual(expectedCount); +} diff --git a/modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts b/modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts new file mode 100644 index 000000000..2ff90c6a5 --- /dev/null +++ b/modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts @@ -0,0 +1,114 @@ +import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; +import { ConvexConnectionManager } from '@module/replication/ConvexConnectionManager.js'; +import { ConvexStream, ConvexStreamOptions } from '@module/replication/ConvexStream.js'; +import { logger } from '@powersync/lib-services-framework'; +import { + BucketStorageFactory, + createCoreReplicationMetrics, + initializeCoreReplicationMetrics, + LEGACY_STORAGE_VERSION, + storage +} from '@powersync/service-core'; +import { AbstractStreamTestContext, METRICS_HELPER } from '@powersync/service-core-tests'; +import { clearTestDb, connectConvex, TEST_CONNECTION_OPTIONS, TestConvexConnection } from './util.js'; + +export class ConvexStreamTestContext extends AbstractStreamTestContext { + protected _stream?: ConvexStream; + + /** + * Tests operating on the stream need to configure the stream and manage asynchronous + * replication, which gets a little tricky. + * + * This configures all the context, and tears it down afterwards. + */ + static async open( + factory: (options: storage.TestStorageOptions) => Promise, + options?: { doNotClear?: boolean; storageVersion?: number; streamOptions?: Partial } + ) { + const f = await factory({ doNotClear: options?.doNotClear }); + const connectionManager = new ConvexConnectionManager(TEST_CONNECTION_OPTIONS); + + const convexBackend = connectConvex(); + + if (!options?.doNotClear) { + await clearTestDb(convexBackend); + } + + const storageVersion = options?.storageVersion ?? LEGACY_STORAGE_VERSION; + + return new ConvexStreamTestContext(f, connectionManager, convexBackend, options?.streamOptions, storageVersion); + } + + constructor( + public factory: BucketStorageFactory, + public connectionManager: ConvexConnectionManager, + public backend: TestConvexConnection, + protected streamOptions?: Partial, + protected storageVersion: number = LEGACY_STORAGE_VERSION + ) { + super(); + createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + } + + get connectionTag() { + return this.connectionManager.connectionTag; + } + + get stream() { + if (this.storage == null) { + throw new Error('updateSyncRules() first'); + } + if (this._stream) { + return this._stream; + } + const options: ConvexStreamOptions = { + storage: this.storage, + metrics: METRICS_HELPER.metricsEngine, + connections: this.connectionManager, + abortSignal: this.abortController.signal, + ...this.streamOptions + }; + this._stream = new ConvexStream(options); + return this._stream!; + } + + protected async _dispose(): Promise { + await this.connectionManager.end(); + } + + protected triggerReplication(): Promise { + return this.stream.replicate(); + } + + protected waitForInitialSnapshot(): Promise { + return this.stream.waitForInitialSnapshot(); + } + + async getClientCheckpoint(options?: { timeout?: number }): Promise { + const start = Date.now(); + + const api = new ConvexRouteAPIAdapter(this.connectionManager.config); + const lsn = await api.createReplicationHead(async (lsn) => lsn); + + // This old API needs a persisted checkpoint id. + // Since we don't use LSNs anymore, the only way to get that is to wait. + + const timeout = options?.timeout ?? 50_000; + + logger.info(`Waiting for LSN checkpoint: ${lsn}`); + while (Date.now() - start < timeout) { + const storage = await this.factory.getActiveStorage(); + const cp = await storage?.getCheckpoint(); + + if (cp?.lsn != null && cp.lsn >= lsn) { + logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`); + return cp.checkpoint; + } + + await new Promise((resolve) => setTimeout(resolve, 5)); + } + + throw new Error('Timeout while waiting for checkpoint'); + } +} diff --git a/modules/module-convex/test/src/test-utils/util.ts b/modules/module-convex/test/src/test-utils/util.ts new file mode 100644 index 000000000..47123e03d --- /dev/null +++ b/modules/module-convex/test/src/test-utils/util.ts @@ -0,0 +1,120 @@ +import * as types from '@module/types/types.js'; +import { api } from '@testing-convex/_generated/api.js'; +import { ConvexHttpClient } from 'convex/browser'; + +import { SUPPORTED_STORAGE_VERSIONS, TestStorageConfig, TestStorageFactory } from '@powersync/service-core'; +import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import * as postgres_storage from '@powersync/service-module-postgres-storage'; +import { describe, TestOptions } from 'vitest'; +import { env } from '../env.js'; + +export type TestConvexConnection = { + client: ConvexHttpClient; + api: typeof api; +}; + +export const TEST_URI = env.CONVEX_URL; + +export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ + url: env.MONGO_TEST_URL, + isCI: env.CI +}); + +export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.postgresTestSetup({ + url: env.PG_STORAGE_TEST_URL +}); + +const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; + +export interface StorageVersionTestContext { + factory: TestStorageFactory; + storageVersion: number; +} + +export function describeWithStorage( + options: TestOptions & { storageVersions?: number[] }, + fn: (context: StorageVersionTestContext) => void +) { + const storageVersions = options.storageVersions ?? TEST_STORAGE_VERSIONS; + const describeFactory = (storageName: string, config: TestStorageConfig) => { + describe(`${storageName} storage`, options, function () { + for (const storageVersion of storageVersions) { + describe(`storage v${storageVersion}`, function () { + fn({ + factory: config.factory, + storageVersion + }); + }); + } + }); + }; + + if (env.TEST_MONGO_STORAGE) { + describeFactory('mongodb', INITIALIZED_MONGO_STORAGE_FACTORY); + } + + if (env.TEST_POSTGRES_STORAGE) { + describeFactory('postgres', INITIALIZED_POSTGRES_STORAGE_FACTORY); + } +} + +export const RAW_TEST_CONNECTION_OPTIONS: types.ConvexConnectionConfig = { + type: 'convex', + deploy_key: env.CONVEX_DEPLOY_KEY, + deployment_url: env.CONVEX_URL +} as const; + +export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig(RAW_TEST_CONNECTION_OPTIONS); + +export function connectConvex(): TestConvexConnection { + return { + client: new ConvexHttpClient(env.CONVEX_URL), + api + }; +} + +export async function clearTestDb(connection: TestConvexConnection) { + const { api, client } = connection; + + // Delete all lists + let deletedCount = 0; + do { + deletedCount = await client.mutation(api.lists.deleteBatch, {}); + } while (deletedCount > 0); + + deletedCount = 0; + do { + deletedCount = await client.mutation(api.todos.deleteBatch, {}); + } while (deletedCount > 0); +} + +// export async function getClientCheckpoint( +// db: pgwire.PgClient, +// storageFactory: BucketStorageFactory, +// options?: { timeout?: number } +// ): Promise { +// const start = Date.now(); + +// const api = new PostgresRouteAPIAdapter(db); +// const lsn = await api.createReplicationHead(async (lsn) => lsn); + +// // This old API needs a persisted checkpoint id. +// // Since we don't use LSNs anymore, the only way to get that is to wait. + +// const timeout = options?.timeout ?? 50_000; + +// logger.info(`Waiting for LSN checkpoint: ${lsn}`); +// while (Date.now() - start < timeout) { +// const storage = await storageFactory.getActiveStorage(); +// const cp = await storage?.getCheckpoint(); + +// if (cp?.lsn != null && cp.lsn >= lsn) { +// logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`); +// return cp.checkpoint; +// } + +// await new Promise((resolve) => setTimeout(resolve, 5)); +// } + +// throw new Error('Timeout while waiting for checkpoint'); +// } diff --git a/modules/module-convex/test/tsconfig.json b/modules/module-convex/test/tsconfig.json index e89f5c03b..1e518f4b0 100644 --- a/modules/module-convex/test/tsconfig.json +++ b/modules/module-convex/test/tsconfig.json @@ -1,8 +1,7 @@ { "extends": "../../../tsconfig.tests.json", "compilerOptions": { - "rootDir": "src", - "baseUrl": "./", + "rootDir": "..", "noEmit": true, "esModuleInterop": true, "skipLibCheck": true, @@ -10,10 +9,11 @@ "paths": { "@/*": ["../../../packages/service-core/src/*"], "@module/*": ["../src/*"], - "@core-tests/*": ["../../../packages/service-core/test/src/*"] + "@core-tests/*": ["../../../packages/service-core/test/src/*"], + "@testing-convex/*": ["../convex/*"] } }, - "include": ["src"], + "include": ["src", "../convex/**/*.ts"], "references": [ { "path": "../" diff --git a/modules/module-convex/vitest.config.ts b/modules/module-convex/vitest.config.ts index 7cc2993f9..9f1670135 100644 --- a/modules/module-convex/vitest.config.ts +++ b/modules/module-convex/vitest.config.ts @@ -1,3 +1,12 @@ +import path from 'node:path'; import { serviceIntegrationTestConfig } from '../test_config'; -export default serviceIntegrationTestConfig(__dirname); +const baseConfig = serviceIntegrationTestConfig(__dirname); +baseConfig.resolve = { + ...baseConfig.resolve, + alias: { + ...baseConfig.resolve?.alias, + '@testing-convex': path.resolve(__dirname, 'convex') + } +}; +export default baseConfig; diff --git a/modules/module-postgres/vitest.config.ts b/modules/module-postgres/vitest.config.ts index 7cc2993f9..e3d77ae5a 100644 --- a/modules/module-postgres/vitest.config.ts +++ b/modules/module-postgres/vitest.config.ts @@ -1,3 +1,10 @@ import { serviceIntegrationTestConfig } from '../test_config'; -export default serviceIntegrationTestConfig(__dirname); +const baseConfig = serviceIntegrationTestConfig(__dirname); +baseConfig.resolve = { + ...baseConfig.resolve, + alias: { + ...baseConfig.resolve?.alias + } +}; +export default baseConfig; diff --git a/packages/service-core-tests/package.json b/packages/service-core-tests/package.json index 43132adb2..755b03da6 100644 --- a/packages/service-core-tests/package.json +++ b/packages/service-core-tests/package.json @@ -14,6 +14,7 @@ "clean": "rm -rf ./dist && tsc -b --clean" }, "dependencies": { + "@powersync/lib-services-framework": "workspace:^", "@powersync/service-core": "workspace:^", "@powersync/service-jsonbig": "workspace:^", "@powersync/service-sync-rules": "workspace:^" diff --git a/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts b/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts new file mode 100644 index 000000000..477ed89a9 --- /dev/null +++ b/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts @@ -0,0 +1,180 @@ +import { ReplicationAbortedError } from '@powersync/lib-services-framework'; +import { + BucketStorageFactory, + InternalOpId, + settledPromise, + storage, + SyncRulesBucketStorage, + unsettledPromise, + updateSyncRulesFromYaml +} from '@powersync/service-core'; +import { StorageDataHelpers } from './StorageDataHelpers.js'; +import { bucketRequest } from './general-utils.js'; +import { fromAsync } from './stream_utils.js'; + +// TODO: Might want to share this with other replication module tests +export abstract class AbstractStreamTestContext implements AsyncDisposable { + protected abortController = new AbortController(); + protected syncRulesContent?: storage.PersistedSyncRulesContent; + public storage?: SyncRulesBucketStorage; + protected settledReplicationPromise?: Promise>; + + abstract get factory(): BucketStorageFactory; + protected abstract get storageVersion(): number; + + async [Symbol.asyncDispose]() { + await this.dispose(); + } + + protected abstract _dispose(): Promise; + + async dispose() { + this.abortController.abort(); + try { + await this.settledReplicationPromise; + await this._dispose(); + await this.factory?.[Symbol.asyncDispose](); + } catch (e) { + // Throwing here may result in SuppressedError. The underlying errors often don't show up + // in the test output, so we log it here. + // If we could get vitest to log SuppressedError.error and SuppressedError.suppressed, we + // could remove this. + console.error('Error during ConvexStreamTestContext dispose', e); + throw e; + } + } + + async updateSyncRules(content: string) { + const syncRules = await this.factory.updateSyncRules( + updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }) + ); + this.syncRulesContent = syncRules; + this.storage = this.factory.getInstance(syncRules); + return this.storage!; + } + + async loadNextSyncRules() { + const syncRules = await this.factory.getNextSyncRulesContent(); + if (syncRules == null) { + throw new Error(`Next sync rules not available`); + } + + this.syncRulesContent = syncRules; + this.storage = this.factory.getInstance(syncRules); + return this.storage!; + } + + async loadActiveSyncRules() { + const syncRules = await this.factory.getActiveSyncRulesContent(); + if (syncRules == null) { + throw new Error(`Active sync rules not available`); + } + + this.syncRulesContent = syncRules; + this.storage = this.factory.getInstance(syncRules); + return this.storage!; + } + + private getSyncRulesContent(): storage.PersistedSyncRulesContent { + if (this.syncRulesContent == null) { + throw new Error('Sync rules not configured - call updateSyncRules() first'); + } + return this.syncRulesContent; + } + + /** + * Replicate a snapshot, start streaming, and wait for a consistent checkpoint. + */ + async initializeReplication() { + await this.replicateSnapshot(); + // Make sure we're up to date + await this.getCheckpoint(); + } + + protected abstract triggerReplication(): Promise; + protected abstract waitForInitialSnapshot(): Promise; + + /** + * Replicate the initial snapshot, and start streaming. + */ + async replicateSnapshot() { + // Use a settledPromise to avoid unhandled rejections + this.settledReplicationPromise = settledPromise(this.triggerReplication()); + try { + await Promise.race([unsettledPromise(this.settledReplicationPromise), this.waitForInitialSnapshot()]); + } catch (e) { + if (e instanceof ReplicationAbortedError && e.cause != null) { + // Edge case for tests: replicate() can throw an error, but we'd receive the ReplicationAbortedError from + // waitForInitialSnapshot() first. In that case, prioritize the cause, e.g. MissingReplicationSlotError. + // This is not a concern for production use, since we only use waitForInitialSnapshot() in tests. + throw e.cause; + } + throw e; + } + } + + abstract getClientCheckpoint(options?: { timeout?: number }): Promise; + + async getCheckpoint(options?: { timeout?: number }) { + let checkpoint = await Promise.race([ + this.getClientCheckpoint(options), + unsettledPromise(this.settledReplicationPromise!) + ]); + if (checkpoint == null) { + // This indicates an issue with the test setup - replicationPromise completed instead + // of getClientCheckpoint() + throw new Error('Test failure - replicationPromise completed'); + } + return checkpoint; + } + + async getBucketsDataBatch(buckets: Record, options?: { timeout?: number }) { + const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent()); + const checkpoint = await this.getCheckpoint(options); + return helpers.getBucketsDataBatch(buckets, checkpoint); + } + + /** + * This waits for a client checkpoint. + */ + async getBucketData(bucket: string, start?: InternalOpId | string | undefined, options?: { timeout?: number }) { + const helpers = new StorageDataHelpers(this.storage!, this.getSyncRulesContent()); + const checkpoint = await this.getCheckpoint(options); + return helpers.getBucketData(bucket, checkpoint, start); + } + + async getChecksums(buckets: string[], options?: { timeout?: number }) { + const checkpoint = await this.getCheckpoint(options); + const syncRules = this.getSyncRulesContent(); + const versionedBuckets = buckets.map((bucket) => bucketRequest(syncRules, bucket, 0n)); + const checksums = await this.storage!.getChecksums(checkpoint, versionedBuckets); + + const unversioned = new Map(); + for (let i = 0; i < buckets.length; i++) { + unversioned.set(buckets[i], checksums.get(versionedBuckets[i].bucket)!); + } + + return unversioned; + } + + async getChecksum(bucket: string, options?: { timeout?: number }) { + const checksums = await this.getChecksums([bucket], options); + return checksums.get(bucket); + } + + /** + * This does not wait for a client checkpoint. + */ + async getCurrentBucketData(bucket: string, start?: InternalOpId | string | undefined) { + start ??= 0n; + if (typeof start == 'string') { + start = BigInt(start); + } + const syncRules = this.getSyncRulesContent(); + const { checkpoint } = await this.storage!.getCheckpoint(); + const map = [bucketRequest(syncRules, bucket, start)]; + const batch = this.storage!.getBucketDataBatch(checkpoint, map); + const batches = await fromAsync(batch); + return batches[0]?.chunkData.data ?? []; + } +} diff --git a/packages/service-core-tests/src/test-utils/test-utils-index.ts b/packages/service-core-tests/src/test-utils/test-utils-index.ts index a79c44098..d356a9392 100644 --- a/packages/service-core-tests/src/test-utils/test-utils-index.ts +++ b/packages/service-core-tests/src/test-utils/test-utils-index.ts @@ -1,3 +1,4 @@ +export * from './AbstractStreamTestContext.js'; export * from './bucket-validation.js'; export * from './general-utils.js'; export * from './MetricsHelper.js'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 57d1e69be..6a1e148b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -88,10 +88,10 @@ importers: version: 5.9.3 vite: specifier: ^8.0.9 - version: 8.0.9(@types/node@22.16.2)(yaml@2.8.3) + version: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) vitest: specifier: 'catalog:' - version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)) + version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) ws: specifier: ^8.2.3 version: 8.18.0 @@ -188,7 +188,7 @@ importers: version: 4.17.6 vitest: specifier: 'catalog:' - version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)) + version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) modules/module-convex: dependencies: @@ -214,6 +214,9 @@ importers: specifier: ^1.3.0 version: 1.3.0 devDependencies: + '@convex-dev/auth': + specifier: ^0.0.92 + version: 0.0.92(@auth/core@0.37.4)(convex@1.37.0) '@powersync/service-core-tests': specifier: workspace:* version: link:../../packages/service-core-tests @@ -223,6 +226,12 @@ importers: '@powersync/service-module-postgres-storage': specifier: workspace:* version: link:../module-postgres-storage + convex: + specifier: ^1.37.0 + version: 1.37.0 + dotenv: + specifier: ^16.4.5 + version: 16.4.5 modules/module-core: dependencies: @@ -544,7 +553,7 @@ importers: devDependencies: vitest: specifier: 'catalog:' - version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)) + version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) packages/jsonbig: dependencies: @@ -724,6 +733,9 @@ importers: packages/service-core-tests: dependencies: + '@powersync/lib-services-framework': + specifier: workspace:^ + version: link:../../libs/lib-services '@powersync/service-core': specifier: workspace:^ version: link:../service-core @@ -735,7 +747,7 @@ importers: version: link:../sync-rules vitest: specifier: 'catalog:' - version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)) + version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) devDependencies: '@opentelemetry/sdk-metrics': specifier: ^1.30.1 @@ -778,7 +790,7 @@ importers: version: 1.0.0 vitest: specifier: 'catalog:' - version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)) + version: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) packages/types: dependencies: @@ -880,6 +892,20 @@ packages: peerDependencies: '@types/json-schema': ^7.0.15 + '@auth/core@0.37.4': + resolution: {integrity: sha512-HOXJwXWXQRhbBDHlMU0K/6FT1v+wjtzdKhsNg0ZN7/gne6XPsIrjZ4daMcFnbq0Z/vsAbYBinQhhua0d77v7qw==} + peerDependencies: + '@simplewebauthn/browser': ^9.0.1 + '@simplewebauthn/server': ^9.0.2 + nodemailer: ^6.8.0 + peerDependenciesMeta: + '@simplewebauthn/browser': + optional: true + '@simplewebauthn/server': + optional: true + nodemailer: + optional: true + '@azure-rest/core-client@2.5.1': resolution: {integrity: sha512-EHaOXW0RYDKS5CFffnixdyRPak5ytiCtU7uXDcP/uiY+A6jFRwNGzzJBiznkCzvi5EYpY+YWinieqHb0oY916A==} engines: {node: '>=20.0.0'} @@ -1036,6 +1062,17 @@ packages: resolution: {integrity: sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==} engines: {node: '>=0.1.90'} + '@convex-dev/auth@0.0.92': + resolution: {integrity: sha512-tNRIMTDxi2vrbT+3vz1FgNR1321IfIBDDBy59zul7E1DyzWQKoU0OzgFqWbiVm3o8gn0eQsYTU3UHNRX9kp3wQ==} + hasBin: true + peerDependencies: + '@auth/core': ^0.37.0 + convex: ^1.17.0 + react: ^18.2.0 || ^19.0.0-0 + peerDependenciesMeta: + react: + optional: true + '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -1061,6 +1098,162 @@ packages: '@emnapi/wasi-threads@1.2.1': resolution: {integrity: sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==} + '@esbuild/aix-ppc64@0.27.0': + resolution: {integrity: sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.27.0': + resolution: {integrity: sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.27.0': + resolution: {integrity: sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.27.0': + resolution: {integrity: sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.27.0': + resolution: {integrity: sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.0': + resolution: {integrity: sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.27.0': + resolution: {integrity: sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.0': + resolution: {integrity: sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.27.0': + resolution: {integrity: sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.27.0': + resolution: {integrity: sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.27.0': + resolution: {integrity: sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.27.0': + resolution: {integrity: sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.27.0': + resolution: {integrity: sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.27.0': + resolution: {integrity: sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.0': + resolution: {integrity: sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.27.0': + resolution: {integrity: sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.27.0': + resolution: {integrity: sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.27.0': + resolution: {integrity: sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.0': + resolution: {integrity: sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.27.0': + resolution: {integrity: sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.0': + resolution: {integrity: sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.27.0': + resolution: {integrity: sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.27.0': + resolution: {integrity: sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.27.0': + resolution: {integrity: sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.27.0': + resolution: {integrity: sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.27.0': + resolution: {integrity: sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@fastify/ajv-compiler@4.0.5': resolution: {integrity: sha512-KoWKW+MhvfTRWL4qrhUwAAZoaChluo0m0vbiJlGMt2GXvL4LVPQEjt8kSpHI3IBq5Rez8fg+XeH3cneztq+C7A==} @@ -1524,9 +1717,24 @@ packages: peerDependencies: '@opentelemetry/api': ^1.1.0 + '@oslojs/asn1@1.0.0': + resolution: {integrity: sha512-zw/wn0sj0j0QKbIXfIlnEcTviaCzYOY3V5rAyjR6YtOByFtJiT574+8p9Wlach0lZH9fddD4yb9laEAIl4vXQA==} + + '@oslojs/binary@1.0.0': + resolution: {integrity: sha512-9RCU6OwXU6p67H4NODbuxv2S3eenuQ4/WFLrsq+K/k682xrznH5EVWA7N4VFk9VYVcbFtKqur5YQQZc0ySGhsQ==} + + '@oslojs/crypto@1.0.1': + resolution: {integrity: sha512-7n08G8nWjAr/Yu3vu9zzrd0L9XnrJfpMioQcvCMxBIiF5orECHe5/3J0jmXRVvgfqMm/+4oxlQ+Sq39COYLcNQ==} + + '@oslojs/encoding@1.1.0': + resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==} + '@oxc-project/types@0.126.0': resolution: {integrity: sha512-oGfVtjAgwQVVpfBrbtk4e1XDyWHRFta6BS3GWVzrF8xYBT2VGQAk39yJS/wFSMrZqoiCU4oghT3Ch0HaHGIHcQ==} + '@panva/hkdf@1.2.1': + resolution: {integrity: sha512-6oclG6Y3PiDFcoyk8srjLfVKyMfVCKJ27JwNPViuXziFpmdz+MZnZN/aKY0JGXgYuO/VghU0jcOAZgWXZ1Dmrw==} + '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} @@ -2244,6 +2452,25 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + convex@1.37.0: + resolution: {integrity: sha512-xGSx5edIsXCEex3OU2U2N0oyB/cOa9qGwKiImF9yOWqjqZgOkx39idtpdlwNBTBSt4S30oAvs4yeXY5xxPIX3A==} + engines: {node: '>=18.0.0', npm: '>=7.0.0'} + hasBin: true + peerDependencies: + '@auth0/auth0-react': ^2.0.1 + '@clerk/clerk-react': ^4.12.8 || ^5.0.0 + '@clerk/react': ^6.4.3 + react: ^18.0.0 || ^19.0.0-0 || ^19.0.0 + peerDependenciesMeta: + '@auth0/auth0-react': + optional: true + '@clerk/clerk-react': + optional: true + '@clerk/react': + optional: true + react: + optional: true + cookie@1.0.2: resolution: {integrity: sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==} engines: {node: '>=18'} @@ -2382,6 +2609,11 @@ packages: es-module-lexer@2.0.0: resolution: {integrity: sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==} + esbuild@0.27.0: + resolution: {integrity: sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==} + engines: {node: '>=18'} + hasBin: true + escalade@3.1.2: resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} engines: {node: '>=6'} @@ -2680,6 +2912,10 @@ packages: resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==} engines: {node: '>=8'} + is-network-error@1.3.1: + resolution: {integrity: sha512-6QCxa49rQbmUWLfk0nuGqzql9U8uaV2H6279bRErPBHe/109hCzsLUBUHfbEtvLIHBd6hyXbgedBSHevm43Edw==} + engines: {node: '>=16'} + is-number@7.0.0: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} @@ -2743,6 +2979,9 @@ packages: jose@4.15.9: resolution: {integrity: sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==} + jose@5.10.0: + resolution: {integrity: sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==} + js-md4@0.3.2: resolution: {integrity: sha512-/GDnfQYsltsjRswQhN9fhv3EMw2sCpUdrdxyWDOUK7eyD++r3gRhzgiQgc/x4MAv2i1iuQ4lxO5mvqM3vj4bwA==} @@ -2795,6 +3034,10 @@ packages: jws@4.0.1: resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + jwt-decode@4.0.0: + resolution: {integrity: sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==} + engines: {node: '>=18'} + kuler@2.0.0: resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==} @@ -2945,6 +3188,10 @@ packages: resolution: {integrity: sha512-FbAj6lXil6t8z4z3j0E5mfRlPzxkySotzUHwRXjlpRh10vc6AI6WN62ehZj82VG7M20rqogJ0GLwar2Xa05a8Q==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} + lucia@3.2.2: + resolution: {integrity: sha512-P1FlFBGCMPMXu+EGdVD9W4Mjm0DqsusmKgO7Xc33mI5X1bklmsQb0hfzPhXomQr9waWIBDsiOjvr1e6BTaUqpA==} + deprecated: This package has been deprecated. Please see https://lucia-auth.com/lucia-v3/migrate. + magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} @@ -3131,6 +3378,9 @@ packages: resolution: {integrity: sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==} engines: {node: '>=18'} + oauth4webapi@3.8.6: + resolution: {integrity: sha512-iwemM91xz8nryHti2yTmg5fhyEMVOkOXwHNqbvcATjyajb5oQxCQzrNOA6uElRHuMhQQTKUyFKV9y/CNyg25BQ==} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -3228,6 +3478,9 @@ packages: path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + path-to-regexp@6.3.0: + resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} + path-type@4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} @@ -3301,6 +3554,14 @@ packages: resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} engines: {node: '>=0.10.0'} + preact-render-to-string@6.5.11: + resolution: {integrity: sha512-ubnauqoGczeGISiOh6RjX0/cdaF8v/oDXIjO85XALCQjwQP+SB4RDXXtvZ6yTYSjG+PC1QRP2AhPgCEsM2EvUw==} + peerDependencies: + preact: '>=10' + + preact@10.24.3: + resolution: {integrity: sha512-Z2dPnBnMUfyQfSQ+GBdsGa16hz35YmLmtTLhM169uW944hYL6xzTYkJjC07j+Wosz733pMWx0fgON3JNw1jJQA==} + prebuild-install@7.1.3: resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} engines: {node: '>=10'} @@ -3521,6 +3782,9 @@ packages: seq-queue@0.0.5: resolution: {integrity: sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==} + server-only@0.0.1: + resolution: {integrity: sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==} + set-cookie-parser@2.6.0: resolution: {integrity: sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==} @@ -4052,6 +4316,14 @@ snapshots: '@types/json-schema': 7.0.15 js-yaml: 4.1.1 + '@auth/core@0.37.4': + dependencies: + '@panva/hkdf': 1.2.1 + jose: 5.10.0 + oauth4webapi: 3.8.6 + preact: 10.24.3 + preact-render-to-string: 6.5.11(preact@10.24.3) + '@azure-rest/core-client@2.5.1': dependencies: '@azure/abort-controller': 2.1.2 @@ -4368,6 +4640,21 @@ snapshots: '@colors/colors@1.6.0': {} + '@convex-dev/auth@0.0.92(@auth/core@0.37.4)(convex@1.37.0)': + dependencies: + '@auth/core': 0.37.4 + '@oslojs/crypto': 1.0.1 + '@oslojs/encoding': 1.1.0 + convex: 1.37.0 + cookie: 1.0.2 + is-network-error: 1.3.1 + jose: 5.10.0 + jwt-decode: 4.0.0 + lucia: 3.2.2 + oauth4webapi: 3.8.6 + path-to-regexp: 6.3.0 + server-only: 0.0.1 + '@cspotcode/source-map-support@0.8.1': dependencies: '@jridgewell/trace-mapping': 0.3.9 @@ -4410,6 +4697,84 @@ snapshots: tslib: 2.8.1 optional: true + '@esbuild/aix-ppc64@0.27.0': + optional: true + + '@esbuild/android-arm64@0.27.0': + optional: true + + '@esbuild/android-arm@0.27.0': + optional: true + + '@esbuild/android-x64@0.27.0': + optional: true + + '@esbuild/darwin-arm64@0.27.0': + optional: true + + '@esbuild/darwin-x64@0.27.0': + optional: true + + '@esbuild/freebsd-arm64@0.27.0': + optional: true + + '@esbuild/freebsd-x64@0.27.0': + optional: true + + '@esbuild/linux-arm64@0.27.0': + optional: true + + '@esbuild/linux-arm@0.27.0': + optional: true + + '@esbuild/linux-ia32@0.27.0': + optional: true + + '@esbuild/linux-loong64@0.27.0': + optional: true + + '@esbuild/linux-mips64el@0.27.0': + optional: true + + '@esbuild/linux-ppc64@0.27.0': + optional: true + + '@esbuild/linux-riscv64@0.27.0': + optional: true + + '@esbuild/linux-s390x@0.27.0': + optional: true + + '@esbuild/linux-x64@0.27.0': + optional: true + + '@esbuild/netbsd-arm64@0.27.0': + optional: true + + '@esbuild/netbsd-x64@0.27.0': + optional: true + + '@esbuild/openbsd-arm64@0.27.0': + optional: true + + '@esbuild/openbsd-x64@0.27.0': + optional: true + + '@esbuild/openharmony-arm64@0.27.0': + optional: true + + '@esbuild/sunos-x64@0.27.0': + optional: true + + '@esbuild/win32-arm64@0.27.0': + optional: true + + '@esbuild/win32-ia32@0.27.0': + optional: true + + '@esbuild/win32-x64@0.27.0': + optional: true + '@fastify/ajv-compiler@4.0.5': dependencies: ajv: 8.18.0 @@ -4925,8 +5290,23 @@ snapshots: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 2.0.1(@opentelemetry/api@1.9.0) + '@oslojs/asn1@1.0.0': + dependencies: + '@oslojs/binary': 1.0.0 + + '@oslojs/binary@1.0.0': {} + + '@oslojs/crypto@1.0.1': + dependencies: + '@oslojs/asn1': 1.0.0 + '@oslojs/binary': 1.0.0 + + '@oslojs/encoding@1.1.0': {} + '@oxc-project/types@0.126.0': {} + '@panva/hkdf@1.2.1': {} + '@pinojs/redact@0.4.0': {} '@pnpm/cli.meta@1000.0.11': @@ -5369,7 +5749,7 @@ snapshots: obug: 2.1.1 std-env: 4.0.0 tinyrainbow: 3.0.3 - vitest: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)) + vitest: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) '@vitest/expect@4.1.0': dependencies: @@ -5380,21 +5760,21 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.0.3 - '@vitest/mocker@4.1.0(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3))': + '@vitest/mocker@4.1.0(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.0 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.9(@types/node@22.16.2)(yaml@2.8.3) + vite: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) - '@vitest/mocker@4.1.0(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3))': + '@vitest/mocker@4.1.0(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3))': dependencies: '@vitest/spy': 4.1.0 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.9(@types/node@25.5.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3) '@vitest/pretty-format@4.1.0': dependencies: @@ -5421,9 +5801,9 @@ snapshots: flatted: 3.4.0 pathe: 2.0.3 sirv: 3.0.2 - tinyglobby: 0.2.15 + tinyglobby: 0.2.16 tinyrainbow: 3.0.3 - vitest: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)) + vitest: 4.1.0(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) '@vitest/utils@4.1.0': dependencies: @@ -5692,6 +6072,15 @@ snapshots: convert-source-map@2.0.0: {} + convex@1.37.0: + dependencies: + esbuild: 0.27.0 + prettier: 3.4.2 + ws: 8.18.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + cookie@1.0.2: {} core-util-is@1.0.3: {} @@ -5793,6 +6182,35 @@ snapshots: es-module-lexer@2.0.0: {} + esbuild@0.27.0: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.0 + '@esbuild/android-arm': 0.27.0 + '@esbuild/android-arm64': 0.27.0 + '@esbuild/android-x64': 0.27.0 + '@esbuild/darwin-arm64': 0.27.0 + '@esbuild/darwin-x64': 0.27.0 + '@esbuild/freebsd-arm64': 0.27.0 + '@esbuild/freebsd-x64': 0.27.0 + '@esbuild/linux-arm': 0.27.0 + '@esbuild/linux-arm64': 0.27.0 + '@esbuild/linux-ia32': 0.27.0 + '@esbuild/linux-loong64': 0.27.0 + '@esbuild/linux-mips64el': 0.27.0 + '@esbuild/linux-ppc64': 0.27.0 + '@esbuild/linux-riscv64': 0.27.0 + '@esbuild/linux-s390x': 0.27.0 + '@esbuild/linux-x64': 0.27.0 + '@esbuild/netbsd-arm64': 0.27.0 + '@esbuild/netbsd-x64': 0.27.0 + '@esbuild/openbsd-arm64': 0.27.0 + '@esbuild/openbsd-x64': 0.27.0 + '@esbuild/openharmony-arm64': 0.27.0 + '@esbuild/sunos-x64': 0.27.0 + '@esbuild/win32-arm64': 0.27.0 + '@esbuild/win32-ia32': 0.27.0 + '@esbuild/win32-x64': 0.27.0 + escalade@3.1.2: {} esprima@4.0.1: {} @@ -6096,6 +6514,8 @@ snapshots: is-interactive@1.0.0: {} + is-network-error@1.3.1: {} + is-number@7.0.0: {} is-plain-obj@4.1.0: {} @@ -6144,6 +6564,8 @@ snapshots: jose@4.15.9: {} + jose@5.10.0: {} + js-md4@0.3.2: {} js-tokens@10.0.0: {} @@ -6202,6 +6624,8 @@ snapshots: jwa: 2.0.1 safe-buffer: 5.2.1 + jwt-decode@4.0.0: {} + kuler@2.0.0: {} leven@3.1.0: {} @@ -6316,6 +6740,11 @@ snapshots: lru.min@1.1.1: {} + lucia@3.2.2: + dependencies: + '@oslojs/crypto': 1.0.1 + '@oslojs/encoding': 1.1.0 + magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -6483,6 +6912,8 @@ snapshots: path-key: 4.0.0 unicorn-magic: 0.3.0 + oauth4webapi@3.8.6: {} + object-assign@4.1.1: {} obug@2.1.1: {} @@ -6570,6 +7001,8 @@ snapshots: path-parse@1.0.7: {} + path-to-regexp@6.3.0: {} + path-type@4.0.0: {} pathe@2.0.3: {} @@ -6639,6 +7072,12 @@ snapshots: dependencies: xtend: 4.0.2 + preact-render-to-string@6.5.11(preact@10.24.3): + dependencies: + preact: 10.24.3 + + preact@10.24.3: {} + prebuild-install@7.1.3: dependencies: detect-libc: 2.1.2 @@ -6882,6 +7321,8 @@ snapshots: seq-queue@0.0.5: {} + server-only@0.0.1: {} + set-cookie-parser@2.6.0: {} shebang-command@1.2.0: @@ -7185,7 +7626,7 @@ snapshots: vary@1.1.2: {} - vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3): + vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -7194,10 +7635,11 @@ snapshots: tinyglobby: 0.2.16 optionalDependencies: '@types/node': 22.16.2 + esbuild: 0.27.0 fsevents: 2.3.3 yaml: 2.8.3 - vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3): + vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -7206,13 +7648,14 @@ snapshots: tinyglobby: 0.2.16 optionalDependencies: '@types/node': 25.5.0 + esbuild: 0.27.0 fsevents: 2.3.3 yaml: 2.8.3 - vitest@4.1.0(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)): + vitest@4.1.0(@opentelemetry/api@1.9.0)(@types/node@22.16.2)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.0 - '@vitest/mocker': 4.1.0(vite@8.0.9(@types/node@22.16.2)(yaml@2.8.3)) + '@vitest/mocker': 4.1.0(vite@8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3)) '@vitest/pretty-format': 4.1.0 '@vitest/runner': 4.1.0 '@vitest/snapshot': 4.1.0 @@ -7227,9 +7670,9 @@ snapshots: std-env: 4.0.0 tinybench: 2.9.0 tinyexec: 1.0.2 - tinyglobby: 0.2.15 + tinyglobby: 0.2.16 tinyrainbow: 3.0.3 - vite: 8.0.9(@types/node@22.16.2)(yaml@2.8.3) + vite: 8.0.9(@types/node@22.16.2)(esbuild@0.27.0)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.0 @@ -7238,10 +7681,10 @@ snapshots: transitivePeerDependencies: - msw - vitest@4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)): + vitest@4.1.0(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/ui@4.1.0)(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)): dependencies: '@vitest/expect': 4.1.0 - '@vitest/mocker': 4.1.0(vite@8.0.9(@types/node@25.5.0)(yaml@2.8.3)) + '@vitest/mocker': 4.1.0(vite@8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3)) '@vitest/pretty-format': 4.1.0 '@vitest/runner': 4.1.0 '@vitest/snapshot': 4.1.0 @@ -7256,9 +7699,9 @@ snapshots: std-env: 4.0.0 tinybench: 2.9.0 tinyexec: 1.0.2 - tinyglobby: 0.2.15 + tinyglobby: 0.2.16 tinyrainbow: 3.0.3 - vite: 8.0.9(@types/node@25.5.0)(yaml@2.8.3) + vite: 8.0.9(@types/node@25.5.0)(esbuild@0.27.0)(yaml@2.8.3) why-is-node-running: 2.3.0 optionalDependencies: '@opentelemetry/api': 1.9.0 From cd64ef2df191328a871607edfa066faad67ed1ae Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 6 May 2026 08:43:26 +0200 Subject: [PATCH 49/86] test for resumable replication --- modules/module-convex/convex/lists.ts | 16 +- modules/module-convex/convex/todos.ts | 2 +- .../ConvexSnapshotProgresCursor.ts | 4 +- .../test/src/ConvexStream.test.ts | 69 +----- .../integration/resuming_snapshots.test.ts | 201 ++++++++---------- modules/module-convex/test/src/types.test.ts | 6 +- 6 files changed, 121 insertions(+), 177 deletions(-) diff --git a/modules/module-convex/convex/lists.ts b/modules/module-convex/convex/lists.ts index 6e605210a..1ebdbb048 100644 --- a/modules/module-convex/convex/lists.ts +++ b/modules/module-convex/convex/lists.ts @@ -1,8 +1,9 @@ import { v } from 'convex/values'; import { mutation, query } from './_generated/server.js'; import schema from './schema.js'; +import { findListByUuid } from './utils/collections.js'; -const BATCH_SIZE = 4000; +const BATCH_SIZE = 2000; export const get = query({ args: {}, @@ -11,6 +12,19 @@ export const get = query({ } }); +export const deleteItem = mutation({ + args: { + uuid: v.string() + }, + handler: async ({ db }, { uuid }) => { + const found = await findListByUuid({ db: db, uuid }); + if (!found) { + throw new Error('Not found'); + } + await db.delete('lists', found._id); + } +}); + /** * Deletes a batch of items. * Convex limits the number of ops in a mutation/transaction to 16_000. diff --git a/modules/module-convex/convex/todos.ts b/modules/module-convex/convex/todos.ts index ba7988806..2474ef409 100644 --- a/modules/module-convex/convex/todos.ts +++ b/modules/module-convex/convex/todos.ts @@ -3,7 +3,7 @@ import { mutation, query } from './_generated/server.js'; import schema from './schema.js'; import { findListByUuid } from './utils/collections.js'; -const BATCH_SIZE = 4000; +const BATCH_SIZE = 2000; export const get = query({ args: {}, diff --git a/modules/module-convex/src/replication/ConvexSnapshotProgresCursor.ts b/modules/module-convex/src/replication/ConvexSnapshotProgresCursor.ts index 074165ba2..9f6483667 100644 --- a/modules/module-convex/src/replication/ConvexSnapshotProgresCursor.ts +++ b/modules/module-convex/src/replication/ConvexSnapshotProgresCursor.ts @@ -1,5 +1,5 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework'; -import bson from 'bson'; +import * as bson from 'bson'; import * as t from 'ts-codec'; export const ConvexSnapshotProgressCursor = t.object({ @@ -34,7 +34,7 @@ export function decodeSnapshotProgressCursor(value: Uint8Array | null | undefine return BinaryConvexSnapshotProgressCursor.decode(value); } catch (error) { throw new ReplicationAssertionError( - `Convex snapshot progress cursor is not valid JSON: ${error instanceof Error ? error.message : `${error}`}` + `Convex snapshot progress cursor is not valid BSON: ${error instanceof Error ? error.message : `${error}`}` ); } } diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index b1eb28808..f424a032b 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -1,8 +1,9 @@ +import { toConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; +import { BinaryConvexSnapshotProgressCursor } from '@module/replication/ConvexSnapshotProgresCursor.js'; +import { ConvexStream } from '@module/replication/ConvexStream.js'; import { SaveOperationTag, SourceTable } from '@powersync/service-core'; import { TablePattern } from '@powersync/service-sync-rules'; import { describe, expect, it, vi } from 'vitest'; -import { toConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; -import { ConvexStream } from '@module/replication/ConvexStream.js'; const CURSOR_100 = '1772817606884944100'; const CURSOR_101 = '1772817606884944101'; @@ -287,70 +288,16 @@ describe('ConvexStream', () => { expect(context.saves[0]?.after.avatar).toEqual(Uint8Array.of(1, 2, 3)); }); - it('resumes table snapshots from the persisted page cursor', async () => { - const context = createFakeStorage({ - snapshotLsn: toConvexLsn(CURSOR_200), - tableSnapshotStatus: { - replicatedCount: 1, - totalEstimatedCount: -1, - lastKey: Buffer.from('page-2', 'utf8') - } - }); - const abortController = new AbortController(); - - const snapshotCalls: any[] = []; - const listSnapshot = vi.fn(async (options: any) => { - snapshotCalls.push(options ?? {}); - return { - snapshot: CURSOR_200, - cursor: null, - hasMore: false, - values: [{ _table: 'users', _id: 'u2', name: 'Bob' }] - }; - }); - - const getGlobalSnapshotCursor = vi.fn(async () => 'should-not-be-called'); - - const stream = new ConvexStream({ - abortSignal: abortController.signal, - storage: context.storage as any, - metrics: { - getCounter: () => ({ add: () => {} }) - } as any, - connections: { - schema: 'convex', - connectionTag: 'default', - connectionId: '1', - config: { pollingIntervalMs: 1 }, - client: { - getJsonSchemas: async () => ({ - tables: [{ tableName: 'users', schema: {} }], - raw: {} - }), - listSnapshot, - getGlobalSnapshotCursor - } - } as any - }); - - await stream.initReplication(); - - expect(getGlobalSnapshotCursor).not.toHaveBeenCalled(); - expect(snapshotCalls.length).toBe(1); - expect(snapshotCalls[0]?.snapshot).toBe(CURSOR_200); - expect(snapshotCalls[0]?.cursor).toBe('page-2'); - expect(context.saves.length).toBe(1); - expect(context.tableProgressUpdates).toHaveLength(1); - expect(context.tableProgressUpdates[0]?.replicatedCount).toBe(2); - }); - it('marks snapshot done without re-reading rows when the final page was already flushed', async () => { const context = createFakeStorage({ snapshotLsn: toConvexLsn(CURSOR_200), tableSnapshotStatus: { replicatedCount: 2, totalEstimatedCount: -1, - lastKey: Buffer.from('convex-snapshot-progress:{"cursor":null,"finished":true}', 'utf8') + lastKey: BinaryConvexSnapshotProgressCursor.encode({ + cursor: null, + finished: true + }) } }); const abortController = new AbortController(); @@ -371,7 +318,7 @@ describe('ConvexStream', () => { schema: 'convex', connectionTag: 'default', connectionId: '1', - config: { pollingIntervalMs: 1 }, + config: { polling_interval_ms: 1 }, client: { getJsonSchemas: async () => ({ tables: [{ tableName: 'users', schema: {} }], diff --git a/modules/module-convex/test/src/integration/resuming_snapshots.test.ts b/modules/module-convex/test/src/integration/resuming_snapshots.test.ts index 685da4e71..6340ef7cb 100644 --- a/modules/module-convex/test/src/integration/resuming_snapshots.test.ts +++ b/modules/module-convex/test/src/integration/resuming_snapshots.test.ts @@ -1,21 +1,18 @@ -import { METRICS_HELPER } from '@powersync/service-core-tests'; -import { ReplicationMetric } from '@powersync/service-types'; import { randomUUID } from 'node:crypto'; -import * as timers from 'node:timers/promises'; -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; import { env } from '../env.js'; import { ConvexStreamTestContext } from '../test-utils/ConvexStreamTestContext.js'; import { describeWithStorage, StorageVersionTestContext } from '../test-utils/util.js'; describe.skipIf(!(env.CI || env.SLOW_TESTS))('batch replication', function () { describeWithStorage({ timeout: 240_000 }, function ({ factory, storageVersion }) { - test('resuming initial replication (1)', async () => { - // Stop early - likely to not include deleted row in first replication attempt. - await testResumingReplication(factory, storageVersion, 2000); - }); - test('resuming initial replication (2)', async () => { - // Stop late - likely to include deleted row in first replication attempt. - await testResumingReplication(factory, storageVersion, 8000); + test('resuming initial replication', async () => { + // The initial replication will be split into + // 1 - The first 1001 list records + // 2 - A batch of 1000 todo records + // 3 - Another 1000 batch of todo records + // We interupt the todos after batch 2 (in the middle of 3) + await testResumingReplication(factory, storageVersion, 2500); }); }); }); @@ -44,7 +41,7 @@ async function testResumingReplication( global: data: - SELECT uuid as id, * FROM lists - - SELECT uuid as id, * FROM todos`); + - SELECT uuid as id, * FROM todos `); const { backend } = context; @@ -52,114 +49,100 @@ async function testResumingReplication( // Max number of mutations is batch size supported is 8192 // Maximum number of reads is 4096 in a single mutation await backend.client.mutation(backend.api.lists.createBatch, { - lists: Array.from({ length: 2_000 }).map((_, index) => ({ + lists: Array.from({ length: 1_000 }).map((_, index) => ({ uuid: randomUUID(), name: `list-${index}` })) }); + + // create a single row to track deleted items + const deletableListId = randomUUID(); + await backend.client.mutation(backend.api.lists.createBatch, { + lists: [ + { + uuid: deletableListId, + name: 'parent' + } + ] + }); + // create a single one with a tracked uuid for relationships + const relationalListId = randomUUID(); + await backend.client.mutation(backend.api.lists.createBatch, { + lists: [ + { + uuid: relationalListId, + name: 'parent' + } + ] + }); + await backend.client.mutation(backend.api.todos.createBatch, { + todos: Array.from({ length: 2_000 }).map((_, index) => ({ + uuid: randomUUID(), + list_uuid: relationalListId, + description: `todo-${index}` + })) + }); + // twice in order to get many todos (see limits above) await backend.client.mutation(backend.api.todos.createBatch, { todos: Array.from({ length: 2_000 }).map((_, index) => ({ uuid: randomUUID(), - list_uuid: randomUUID(), - description: `list-${index}` + list_uuid: relationalListId, + description: `todo-${index}` })) }); - const p = context.replicateSnapshot(); + let stopped = new Promise((resolve) => { + context.storage!.registerListener({ + batchStarted: (batch) => { + //register a pre-emptive spy in order to halt writes + const original = batch.save; + let savedCount = 0; + vi.spyOn(batch, 'save').mockImplementation(async (param) => { + if (savedCount >= stopAfter) { + // This interrupts initial replication + // don't await this since awaiting will cause a deadlock + context.dispose(); + resolve(); + throw new Error('Stopping now'); + } + savedCount++; + return original.call(batch, param); + }); + } + }); + }); - let done = false; + const replicationError = context.replicateSnapshot().catch((error) => error); - const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; - try { - (async () => { - while (!done) { - const count = - ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount; + await stopped; + await replicationError; - if (count >= stopAfter) { - break; - } - await timers.setTimeout(1); - } - // This interrupts initial replication - await context.dispose(); - })(); - // This confirms that initial replication was interrupted - await expect(p).rejects.toThrowError(); - done = true; - } finally { - done = true; - } - - // // Bypass the usual "clear db on factory open" step. - // await using context2 = await ConvexStreamTestContext.open(factory, { - // doNotClear: true, - // storageVersion - // }); - - // // This delete should be using one of the ids already replicated - // const { - // rows: [delete1] - // } = await context2.pool.query(`DELETE FROM test_data2 WHERE id = (SELECT id FROM test_data2 LIMIT 1) RETURNING id`); - // // This update should also be using one of the ids already replicated - // const { - // rows: [delete2] - // } = await context2.pool.query( - // `UPDATE test_data2 SET description = 'update1' WHERE id = (SELECT id FROM test_data2 LIMIT 1) RETURNING id` - // ); - // const { - // rows: [delete3] - // } = await context2.pool.query(`INSERT INTO test_data2(description) SELECT 'insert1' RETURNING id`); - - // await context2.loadNextSyncRules(); - // await context2.replicateSnapshot(); - - // const data = await context2.getBucketData('global[]', undefined, {}); - - // const deletedRowOps = data.filter( - // (row) => row.object_type == 'test_data2' && row.object_id === String(delete1.decodeWithoutCustomTypes(0)) - // ); - // const updatedRowOps = data.filter( - // (row) => row.object_type == 'test_data2' && row.object_id === String(delete2.decodeWithoutCustomTypes(0)) - // ); - // const insertedRowOps = data.filter( - // (row) => row.object_type == 'test_data2' && row.object_id === String(delete3.decodeWithoutCustomTypes(0)) - // ); - - // if (deletedRowOps.length != 0) { - // // The deleted row was part of the first replication batch, - // // so it is removed by streaming replication. - // expect(deletedRowOps.length).toEqual(2); - // expect(deletedRowOps[1].op).toEqual('REMOVE'); - // } else { - // // The deleted row was not part of the first replication batch, - // // so it's not in the resulting ops at all. - // } - - // expect(updatedRowOps.length).toBeGreaterThanOrEqual(2); - // // description for the first op could be 'foo' or 'update1'. - // // We only test the final version. - // expect(JSON.parse(updatedRowOps[updatedRowOps.length - 1].data as string).description).toEqual('update1'); - - // expect(insertedRowOps.length).toBeGreaterThanOrEqual(1); - // expect(JSON.parse(insertedRowOps[0].data as string).description).toEqual('insert1'); - // expect(JSON.parse(insertedRowOps[insertedRowOps.length - 1].data as string).description).toEqual('insert1'); - - // // 1000 of test_data1 during first replication attempt. - // // N >= 1000 of test_data2 during first replication attempt. - // // 10000 - N - 1 + 1 of test_data2 during second replication attempt. - // // An additional update during streaming replication (2x total for this row). - // // An additional insert during streaming replication (2x total for this row). - // // If the deleted row was part of the first replication batch, it's removed by streaming replication. - // // This adds 2 ops. - // // We expect this to be 11002 for stopAfter: 2000, and 11004 for stopAfter: 8000. - // // However, this is not deterministic. - // const expectedCount = 11000 - 2 + insertedRowOps.length + updatedRowOps.length + deletedRowOps.length; - // expect(data.length).toEqual(expectedCount); - - // const replicatedCount = - // ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount; - - // // With resumable replication, there should be no need to re-replicate anything. - // expect(replicatedCount).toBeGreaterThanOrEqual(expectedCount); + // Add delete a row which has already been replicated + await backend.client.mutation(backend.api.lists.deleteItem, { + uuid: deletableListId + }); + + await using context2 = await ConvexStreamTestContext.open(factory, { + doNotClear: true, + storageVersion + }); + + // Spy on the list-snapshot endpoint + const snapshotSpy = vi.spyOn(context2.connectionManager.client, 'listSnapshot'); + + await context2.loadNextSyncRules(); + await context2.replicateSnapshot(); + + // The second replication should have called list snapshot for the todos table, at a specific cursor value (resuming) + expect(snapshotSpy).called; + const firstCall = snapshotSpy.mock.calls[0]; + expect(firstCall).toBeDefined(); + expect(firstCall[0].tableName).eq('todos'); + expect(firstCall[0].cursor).toBeDefined(); //it resumed within the table + + // Check the final bucket data + const data = await context2.getBucketData('global[]', undefined, {}); + expect(data.length).eq(5003); // Puts: 1000 + 1 + 1 + 2000 + 2000, REMOVE: 1 + expect(data.filter((item) => item.op == 'PUT').length).eq(5002); + expect(data.filter((item) => item.op == 'REMOVE').length).eq(1); } diff --git a/modules/module-convex/test/src/types.test.ts b/modules/module-convex/test/src/types.test.ts index 7a4c9866f..ef2087604 100644 --- a/modules/module-convex/test/src/types.test.ts +++ b/modules/module-convex/test/src/types.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from 'vitest'; import { CONVEX_CONNECTION_TYPE, normalizeConnectionConfig } from '@module/types/types.js'; +import { describe, expect, it } from 'vitest'; describe('Convex connection config', () => { it('normalizes defaults', () => { @@ -11,8 +11,8 @@ describe('Convex connection config', () => { expect(config.id).toBe('default'); expect(config.tag).toBe('default'); - expect(config.pollingIntervalMs).toBe(1000); - expect(config.deploymentUrl).toBe('https://example.convex.cloud'); + expect(config.polling_interval_ms).toBe(1000); + expect(config.deployment_url).toBe('https://example.convex.cloud'); }); it('throws for invalid URL', () => { From c9d5e6d8fbc2deac6b448caaf17bd702b573643d Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 6 May 2026 09:07:25 +0200 Subject: [PATCH 50/86] Add integration tests to CI tests workflow --- .github/workflows/test.yml | 57 +++++++++++++++++++++++++++++++++ README.md | 4 +++ modules/module-convex/README.md | 28 +++++++++++++--- 3 files changed, 85 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c4e697f5..14725a73f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -258,6 +258,63 @@ jobs: - name: Test Replication run: pnpm --filter='./modules/module-mongodb' test + run-convex-replication-tests: + name: Convex Replication Test + runs-on: ubuntu-latest + needs: run-core-tests + env: + CONVEX_DEPLOYMENT: anonymous:anonymous-module-convex + CONVEX_URL: http://127.0.0.1:3210 + CONVEX_SITE_URL: http://127.0.0.1:3211 + PG_STORAGE_TEST_URL: postgres://postgres:postgres@localhost:5431/powersync_storage_test + + steps: + - uses: actions/checkout@v5 + + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Start MongoDB + uses: ./.github/actions/start-mongodb + + - name: Start PostgreSQL (Storage) + uses: ./.github/actions/start-postgres-storage + + - name: Setup Node and build + uses: ./.github/actions/setup-node-build + + - name: Start Convex + working-directory: modules/module-convex + run: | + # Start the local Convex backend in the background so the next test step can use it. + pnpm run dev:convex > convex-dev.log 2>&1 & + + # Wait until Convex has generated its local admin key and the schema API is responding. + timeout 120 bash -c ' + until node --input-type=module -e '"'"' + import fs from "node:fs"; + const config = JSON.parse(fs.readFileSync(".convex/local/default/config.json", "utf8")); + const response = await fetch("http://127.0.0.1:3210/api/json_schemas?format=json", { + headers: { Authorization: `Convex ${config.adminKey}` } + }); + if (!response.ok) process.exit(1); + '"'"'; do + sleep 2 + done + ' + + - name: Test Replication + run: pnpm --filter='./modules/module-convex' test + + - name: Print Convex logs + if: failure() + working-directory: modules/module-convex + run: cat convex-dev.log + run-mongodb-storage-tests: name: MongoDB Storage Test runs-on: ubuntu-latest diff --git a/README.md b/README.md index db80e75e1..0473f06b1 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ The service can be started using the public Docker image. See the image [notes]( - MySQL replication module. +- [modules/module-convex](./modules/module-convex/README.md) + + - Convex replication module. See the module README for local integration testing with the `dev:convex` backend. + - [modules/module-postgres](./modules/module-postgres/README.md) - Postgres replication module. diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 235e42678..925ae49ff 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -18,9 +18,9 @@ replication: 1. Simplest is to run the convex demo in the self-host-demo [repo](https://github.com/powersync-ja/self-host-demo) -# Development +## Development -Run the `dev:convex` script to start a local Convex development instance. +Run the `dev:convex` script to start the local Convex development backend used by the module tests. ```bash # In the modules/module-convex folder @@ -28,10 +28,30 @@ pnpm run dev:convex # OR # From the repo root -pnpm run -C modules/modules-convex dev:convex +pnpm run -C modules/module-convex dev:convex ``` -# Technical notes +The local backend listens on `http://127.0.0.1:3210` by default. The integration tests read the local deploy key from +`modules/module-convex/.convex/local/default/config.json`, which is created by `dev:convex`. + +To run the Convex module tests locally: + +```bash +# Terminal 1 +pnpm run -C modules/module-convex dev:convex + +# Terminal 2, from the repo root +pnpm --filter='./modules/module-convex' test +``` + +Some integration tests are gated behind `CI=true` or `SLOW_TESTS=true`. To run them locally, keep `dev:convex` running +and start the required storage backends (MongoDB and Postgres storage), then run: + +```bash +CI=true pnpm --filter='./modules/module-convex' test +``` + +## Technical notes The content below is written in an agents.md style describing the behavior of `module-convex`. From 0ced940b613034ba117fc8f72efc28ffd7299849 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 6 May 2026 11:40:15 +0200 Subject: [PATCH 51/86] Add stream integration tests --- modules/module-convex/README.md | 3 + modules/module-convex/convex/lists.ts | 62 ++++ .../src/client/ConvexApiClient.ts | 8 +- .../src/replication/ConvexStream.ts | 22 +- .../test/src/ConvexStream.test.ts | 64 +++- .../ConvexStream.integration.test.ts | 323 ++++++++++++++++++ ...=> resuming_snapshots.integration.test.ts} | 0 7 files changed, 473 insertions(+), 9 deletions(-) create mode 100644 modules/module-convex/test/src/integration/ConvexStream.integration.test.ts rename modules/module-convex/test/src/integration/{resuming_snapshots.test.ts => resuming_snapshots.integration.test.ts} (100%) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 925ae49ff..9e1bea0a3 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -156,3 +156,6 @@ The content below is written in an agents.md style describing the behavior of `m - Example: a mutation that deletes 5 documents and updates 3 produces 8 entries in `document_deltas`, all with identical `_ts`. - The Convex backend enforces this by never splitting a page mid-timestamp: when the row limit is reached mid-transaction, the page extends until all rows at that `_ts` are included before stopping. - Consequence for replication: all writes from a single mutation always appear in the same `document_deltas` page and are committed to bucket storage atomically as one batch. + - `TRANSACTIONS_REPLICATED` is counted from distinct `_ts` values among replicated changes, not from `document_deltas` pages. A single page can contain multiple Convex mutations, so a committed page may increase the transaction metric by more than one. + - The stream relies on Convex returning `document_deltas` in mutation order by `_ts`. Row order within the same `_ts` is not significant: those rows belong to the same Convex mutation and are committed atomically. + - The stream asserts that observed `_ts` values are non-decreasing across pages. Equal `_ts` values are allowed because they represent rows from the same Convex mutation. diff --git a/modules/module-convex/convex/lists.ts b/modules/module-convex/convex/lists.ts index 1ebdbb048..520b841c0 100644 --- a/modules/module-convex/convex/lists.ts +++ b/modules/module-convex/convex/lists.ts @@ -25,6 +25,20 @@ export const deleteItem = mutation({ } }); +export const updateName = mutation({ + args: { + uuid: v.string(), + name: v.string() + }, + handler: async ({ db }, { uuid, name }) => { + const found = await findListByUuid({ db, uuid }); + if (!found) { + throw new Error('Not found'); + } + await db.patch(found._id, { name }); + } +}); + /** * Deletes a batch of items. * Convex limits the number of ops in a mutation/transaction to 16_000. @@ -58,3 +72,51 @@ export const createBatch = mutation({ return ids; } }); + +/** + * A test mutation which creates and updates multiple lists multiple times. + * This is used to ensure the order of ops is safely replicated. + */ +export const testUpdateMultipleTimes = mutation({ + args: {}, + handler: async (ctx) => { + const listCount = 5; + const createdListIds = []; + for (let i = 0; i < listCount; i++) { + const id = await ctx.db.insert('lists', { + name: `list-${i}`, + uuid: 'fake-uuid' + }); + createdListIds.push(id); + } + + // Update all the lists + for (let i = 0; i < listCount; i++) { + await ctx.db.patch('lists', createdListIds[i]!, { + name: `list-${i}-a`, + uuid: createdListIds[i]! // keep this for later + }); + } + + // Do a second update, but operate on the items in reverse order + for (let i = listCount - 1; i >= 0; i--) { + await ctx.db.patch('lists', createdListIds[i]!, { + name: `list-${i}-a-b` + }); + } + + for (let i = 0; i < listCount; i++) { + await ctx.db.patch('lists', createdListIds[i]!, { + name: `list-${i}-a-b-c` + }); + } + + // delete a random list + const [deleteId] = createdListIds.splice(Math.floor(Math.random() * createdListIds.length), 1); + await ctx.db.delete('lists', deleteId); + + return { + listIds: createdListIds + }; + } +}); diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index 5a1474bae..701aad5d9 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -12,6 +12,10 @@ export interface ConvexRawDocument { [key: string]: any; } +export interface ConvexDocumentDelta extends ConvexRawDocument { + _ts: bigint; +} + export interface ConvexTableSchema { tableName: string; schema: Record; @@ -44,7 +48,7 @@ export interface ConvexDocumentDeltasOptions { export interface ConvexDocumentDeltasResult { cursor: string; hasMore: boolean; - values: ConvexRawDocument[]; + values: ConvexDocumentDelta[]; } export class ConvexApiError extends Error { @@ -115,7 +119,7 @@ export class ConvexApiClient { return { cursor: stringifyCursor(payload.cursor), hasMore: parseHasMore(payload), - values: parseValues(payload) + values: parseValues(payload) as ConvexDocumentDelta[] }; } diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index a814b0a25..a33f88c9d 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -156,6 +156,7 @@ export class ConvexStream { await this.resolveAllSourceTables(batch); let cursor = parseConvexLsn(resumeFromLsn); + let lastTransactionTimestamp: bigint | null = null; while (!this.abortSignal.aborted) { const page = await this.connections.client @@ -174,11 +175,18 @@ export class ConvexStream { const pageLsn = toConvexLsn(nextCursor); let changesInPage = 0; + const transactionTimestampsInPage = new Set(); let sawCheckpointMarker = false; const snapshottedTablesInPage = new Set(); let didMarkOldestUncommitedChange = false; + /** + * Convex returns document_deltas in mutation order by _ts (corresponding to mutation/transaction). + * The row order inside each transaction is out-of-order. + * It looks like Convex squashes multiple mutations on rows before storing deltas. + * We currently don't sort values by their `_creationTime` value. + */ for (const change of page.values) { if (this.abortSignal.aborted) { throw new ReplicationAbortedError('Replication interrupted'); @@ -194,6 +202,14 @@ export class ConvexStream { continue; } + const transactionTimestamp = change._ts; + if (lastTransactionTimestamp != null && transactionTimestamp < lastTransactionTimestamp) { + throw new ReplicationAssertionError( + `Convex document_deltas returned out-of-order _ts values: ${transactionTimestamp} after ${lastTransactionTimestamp}` + ); + } + lastTransactionTimestamp = transactionTimestamp; + const table = await this.getOrResolveTable(batch, tableName, nextCursor, snapshottedTablesInPage); if (table == null || !table.syncAny) { continue; @@ -220,7 +236,11 @@ export class ConvexStream { continue; } + // Convex assigns one _ts commit timestamp to every write in a mutation. + // document_deltas may return multiple mutations in one page, so transaction + // metrics are counted by distinct _ts values, not by delta pages. changesInPage += 1; + transactionTimestampsInPage.add(transactionTimestamp.toString()); } if (changesInPage > 0) { @@ -234,8 +254,8 @@ export class ConvexStream { oldestUncommittedChange: this.replicationLag.oldestUncommittedChange }); + this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(transactionTimestampsInPage.size); if (!checkpointBlocked) { - this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); this.replicationLag.markCommitted(); } } else if (sawCheckpointMarker) { diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index f424a032b..c4ab26841 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -3,6 +3,7 @@ import { BinaryConvexSnapshotProgressCursor } from '@module/replication/ConvexSn import { ConvexStream } from '@module/replication/ConvexStream.js'; import { SaveOperationTag, SourceTable } from '@powersync/service-core'; import { TablePattern } from '@powersync/service-sync-rules'; +import { ReplicationMetric } from '@powersync/service-types'; import { describe, expect, it, vi } from 'vitest'; const CURSOR_100 = '1772817606884944100'; @@ -382,12 +383,19 @@ describe('ConvexStream', () => { let calls = 0; const deltaCalls: any[] = []; + const transactionCounts: number[] = []; const stream = new ConvexStream({ abortSignal: abortController.signal, storage: context.storage as any, metrics: { - getCounter: () => ({ add: () => {} }) + getCounter: (metric: string) => ({ + add: (value: number) => { + if (metric == ReplicationMetric.TRANSACTIONS_REPLICATED) { + transactionCounts.push(value); + } + } + }) } as any, connections: { schema: 'convex', @@ -404,11 +412,12 @@ describe('ConvexStream', () => { deltaCalls.push(options ?? {}); setTimeout(() => abortController.abort(), 0); return { - cursor: CURSOR_101, + cursor: CURSOR_102, hasMore: false, values: [ - { _table: 'users', _id: 'u1', name: 'Updated' }, - { _table: 'users', _id: 'u2', _deleted: true } + { _table: 'users', _id: 'u1', _ts: BigInt(CURSOR_101), name: 'Updated' }, + { _table: 'users', _id: 'u2', _ts: BigInt(CURSOR_101), _deleted: true }, + { _table: 'users', _id: 'u3', _ts: BigInt(CURSOR_102), name: 'Second transaction' } ] }; } @@ -419,11 +428,54 @@ describe('ConvexStream', () => { await stream.streamChanges(); expect(calls).toBeGreaterThan(0); - expect(context.saves.length).toBe(2); + expect(context.saves.length).toBe(3); expect(context.saves[0]?.tag).toBe(SaveOperationTag.UPDATE); expect(context.saves[1]?.tag).toBe(SaveOperationTag.DELETE); - expect(context.commits.at(-1)).toBe(toConvexLsn(CURSOR_101)); + expect(context.saves[2]?.tag).toBe(SaveOperationTag.UPDATE); + expect(context.commits.at(-1)).toBe(toConvexLsn(CURSOR_102)); expect(deltaCalls[0]?.tableName).toBeUndefined(); + expect(transactionCounts).toEqual([2]); + }); + + it('fails when document_deltas returns decreasing transaction timestamps', async () => { + // This should never happen in practice. + // We assert that _ts is increasing in ConvexStream + // This test just verifies the assertion would catch an issue if it ever happened for some reason. + const context = createFakeStorage({ + snapshotDone: true, + resumeFromLsn: toConvexLsn(CURSOR_100) + }); + const abortController = new AbortController(); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: {} }], + raw: {} + }), + documentDeltas: async () => ({ + cursor: CURSOR_102, + hasMore: false, + values: [ + { _table: 'users', _id: 'u1', _ts: BigInt(CURSOR_102), name: 'Later' }, + { _table: 'users', _id: 'u2', _ts: BigInt(CURSOR_101), name: 'Earlier' } + ] + }) + } + } as any + }); + + await expect(stream.streamChanges()).rejects.toThrow(/out-of-order _ts values/); }); it('refreshes metadata before snapshotting a newly discovered wildcard-matched table inline', async () => { diff --git a/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts b/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts new file mode 100644 index 000000000..454a4c69c --- /dev/null +++ b/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts @@ -0,0 +1,323 @@ +import { randomUUID } from 'node:crypto'; + +import { METRICS_HELPER, removeOp } from '@powersync/service-core-tests'; +import { JSONBig } from '@powersync/service-jsonbig'; +import { ReplicationMetric } from '@powersync/service-types'; +import { describe, expect, test, vi } from 'vitest'; +import { env } from '../env.js'; +import { ConvexStreamTestContext } from '../test-utils/ConvexStreamTestContext.js'; +import { describeWithStorage, StorageVersionTestContext } from '../test-utils/util.js'; + +const BASIC_SYNC_RULES = ` +bucket_definitions: + global: + data: + - SELECT uuid as id, name FROM "lists" +`; + +describe.skipIf(!env.CONVEX_DEPLOY_KEY)('ConvexStream integration tests', function () { + describeWithStorage({ timeout: 120_000 }, function ({ factory, storageVersion }) { + defineConvexStreamTests(factory, storageVersion); + }); +}); + +function defineConvexStreamTests( + factory: StorageVersionTestContext['factory'], + storageVersion: StorageVersionTestContext['storageVersion'] +) { + test('Initial snapshot sync', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const testData = await createList(context, { name: 'snapshot-list' }); + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + + await context.replicateSnapshot(); + + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([convexPutOp('lists', syncedId(testData), listBucketRow(testData))]); + expect(endRowCount - startRowCount).toEqual(1); + }); + + test('Replicate basic values', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + await context.replicateSnapshot(); + + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + + const testData = await createList(context, { name: 'streamed-list' }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([convexPutOp('lists', syncedId(testData), listBucketRow(testData))]); + + await vi.waitFor(async () => { + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + expect(endRowCount - startRowCount).toEqual(1); + expect(endTxCount - startTxCount).toEqual(1); + }); + }); + + test('Counts Convex batch mutations as single replicated transactions', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + await context.replicateSnapshot(); + + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + + const batchOne = await createLists(context, ['batch-one-a', 'batch-one-b']); + const single = await createList(context, { name: 'single-mutation' }); + const batchTwo = await createLists(context, ['batch-two-a', 'batch-two-b']); + + const data = await context.getBucketData('global[]'); + expect(data).toHaveLength(5); + expect(data.slice(0, 2)).toEqual(expectListOps(batchOne)); + expect(data[2]).toMatchObject(convexPutOp('lists', syncedId(single), listBucketRow(single))); + expect(data.slice(3, 5)).toEqual(expectListOps(batchTwo)); + + await vi.waitFor(async () => { + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + expect(endRowCount - startRowCount).toEqual(5); + expect(endTxCount - startTxCount).toEqual(3); + }); + }); + + test('Replicated rows in transactions are correctly ordered', async () => { + /** + * out-of-ordered operations on different rows are not an issue. + * out-of-ordered operations on the same row could be large issues. + */ + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + await context.replicateSnapshot(); + + const backendResult = await context.backend.client.mutation(context.backend.api.lists.testUpdateMultipleTimes, {}); + + const data = await context.getBucketData('global[]'); + // Convex seems to squash the deltas, so we don't get a delta for each update which happened in the backend + // The deleted row is reported as _deleted:true, which causes us not to replicate it + expect(data.length).eq(backendResult.listIds.length); + + // All the put ops should contain a value for name which ends with a-b-c + expect( + data.every((item) => { + const parsed = JSON.parse(item.data!); + return parsed.name.endsWith('a-b-c'); + }) + ).true; + }); + + test('Replicate row updates', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const testData = await createList(context, { name: 'initial-list' }); + await context.replicateSnapshot(); + + const updatedData = { + ...testData, + name: 'updated-list' + }; + await context.backend.client.mutation(context.backend.api.lists.updateName, { + uuid: testData.uuid, + name: updatedData.name + }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([ + convexPutOp('lists', syncedId(testData), listBucketRow(testData)), + convexPutOp('lists', syncedId(updatedData), listBucketRow(updatedData)) + ]); + }); + + test('Replicate row deletions', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const testData = await createList(context, { name: 'deleted-list' }); + await context.replicateSnapshot(); + + await context.backend.client.mutation(context.backend.api.lists.deleteItem, { + uuid: testData.uuid + }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([ + convexPutOp('lists', syncedId(testData), listBucketRow(testData)), + removeOp('lists', syncedId(testData)) + ]); + }); + + test('Replicate matched wildcard tables in sync rules', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT uuid as id, * FROM "%" +`); + + const list = await createList(context, { name: 'parent-list' }); + const todo = await createTodo(context, { + listUuid: list.uuid, + description: 'snapshot-todo' + }); + + await context.replicateSnapshot(); + + const streamedList = await createList(context, { name: 'streamed-list' }); + const streamedTodo = await createTodo(context, { + listUuid: streamedList.uuid, + description: 'streamed-todo' + }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([ + convexOp('PUT', 'lists', syncedId(list)), + convexOp('PUT', 'todos', syncedId(todo)), + convexOp('PUT', 'lists', syncedId(streamedList)), + convexOp('PUT', 'todos', syncedId(streamedTodo)) + ]); + }); + + test('Replication for tables not in the sync rules are ignored', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const list = await createList(context, { name: 'synced-parent' }); + await context.replicateSnapshot(); + + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + + // Basic sync streams here don't replicate todo rows + await createTodo(context, { + listUuid: list.uuid, + description: 'ignored-todo' + }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([convexPutOp('lists', syncedId(list), listBucketRow(list))]); + + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + expect(endRowCount - startRowCount).toEqual(0); + expect(endTxCount - startTxCount).toEqual(0); + }); + + test('Table matching is case sensitive', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT uuid as id, name FROM "Lists" +`); + + await context.replicateSnapshot(); + + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + + await createList(context, { name: 'case-sensitive-list' }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([]); + + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + expect(endRowCount - startRowCount).toEqual(0); + expect(endTxCount - startTxCount).toEqual(0); + }); +} + +async function createList(context: ConvexStreamTestContext, options: { name: string }) { + const [list] = await createLists(context, [options.name]); + return list; +} + +async function createLists(context: ConvexStreamTestContext, names: string[]) { + const lists = names.map((name) => ({ + uuid: randomUUID(), + name + })); + + const ids = await context.backend.client.mutation(context.backend.api.lists.createBatch, { + lists + }); + + return lists.map((list, index) => ({ + id: ids[index]!, + uuid: list.uuid, + name: list.name + })); +} + +async function createTodo( + context: ConvexStreamTestContext, + options: { + listUuid: string; + description: string; + } +) { + const uuid = randomUUID(); + const [id] = await context.backend.client.mutation(context.backend.api.todos.createBatch, { + todos: [ + { + uuid, + list_uuid: options.listUuid, + description: options.description + } + ] + }); + + return { + id, + uuid, + listUuid: options.listUuid, + description: options.description + }; +} + +function syncedId(document: { uuid: string }) { + return document.uuid; +} + +function listBucketRow(document: { uuid: string; name: string }) { + return { + id: document.uuid, + name: document.name + }; +} + +/** + * The order of ops do not match that done in a single mutation, this is due to out-of-order (internal) updates. + */ +function expectListOps(lists: Array<{ uuid: string; name: string }>) { + return expect.arrayContaining( + lists.map((list) => expect.objectContaining(convexPutOp('lists', syncedId(list), listBucketRow(list)))) + ); +} + +function convexPutOp(table: string, id: string, data: Record) { + return { + ...convexOp('PUT', table, id), + data: JSONBig.stringify(data) + }; +} + +function convexOp(op: 'PUT' | 'REMOVE', table: string, id: string) { + return { + op, + object_type: table, + object_id: id + }; +} diff --git a/modules/module-convex/test/src/integration/resuming_snapshots.test.ts b/modules/module-convex/test/src/integration/resuming_snapshots.integration.test.ts similarity index 100% rename from modules/module-convex/test/src/integration/resuming_snapshots.test.ts rename to modules/module-convex/test/src/integration/resuming_snapshots.integration.test.ts From e4ec169638741fe8d58135c081dc69f0c3885b19 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 6 May 2026 11:53:02 +0200 Subject: [PATCH 52/86] add notes for metrics --- modules/module-convex/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 9e1bea0a3..296881b5f 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -150,6 +150,7 @@ The content below is written in an agents.md style describing the behavior of `m - On an idle system, multiple successive calls to `/api/document_deltas` will return the same cursor value i.e. the cursor is not wall clock based. - **Mutation Transaction Atomicity in** `document_deltas` + - The `cursor` in `/api/document_deltas` is a Convex commit **timestamp** (`i64`), not a per-operation counter. - Every Convex mutation is an ACID transaction that commits with a single timestamp; all writes within that mutation share the same `_ts` value in the delta stream. - Therefore, the cursor advances **once per mutation**, not once per individual CRUD operation inside it. @@ -159,3 +160,12 @@ The content below is written in an agents.md style describing the behavior of `m - `TRANSACTIONS_REPLICATED` is counted from distinct `_ts` values among replicated changes, not from `document_deltas` pages. A single page can contain multiple Convex mutations, so a committed page may increase the transaction metric by more than one. - The stream relies on Convex returning `document_deltas` in mutation order by `_ts`. Row order within the same `_ts` is not significant: those rows belong to the same Convex mutation and are committed atomically. - The stream asserts that observed `_ts` values are non-decreasing across pages. Equal `_ts` values are allowed because they represent rows from the same Convex mutation. + +- **Replication metrics** + - Implemented: + - `ROWS_REPLICATED`: incremented for each row written from snapshots and deltas. + - `TRANSACTIONS_REPLICATED`: incremented by the number of distinct Convex `_ts` mutation timestamps replicated from `document_deltas`. + - Not implemented yet: + - `DATA_REPLICATED_BYTES`: Convex does not currently report source bytes replicated into PowerSync. This would need explicit accounting in the Convex replication/client path. + - `CHUNKS_REPLICATED`: Convex does not currently report replication chunks. + - Bucket storage size gauges (`REPLICATION_SIZE_BYTES`, `OPERATION_SIZE_BYTES`, `PARAMETER_SIZE_BYTES`) are reported by the configured bucket storage backend, not by this replication module. From 34ee0672f51cf0ccea29e2d1cd36c2154e5fbb66 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 6 May 2026 13:12:42 +0200 Subject: [PATCH 53/86] update tsconfig --- packages/service-core-tests/tsconfig.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/service-core-tests/tsconfig.json b/packages/service-core-tests/tsconfig.json index 64548dbfc..31a397bf7 100644 --- a/packages/service-core-tests/tsconfig.json +++ b/packages/service-core-tests/tsconfig.json @@ -17,6 +17,9 @@ }, { "path": "../service-core" + }, + { + "path": "../../libs/lib-services" } ] } From 0f26b12863cf382fd3c7061762f46a55fa3e20db Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 6 May 2026 13:34:45 +0200 Subject: [PATCH 54/86] fix formating and convex generated code --- modules/module-convex/package.json | 5 +++-- modules/module-convex/src/common/convex-to-sqlite.ts | 4 ++-- .../module-convex/src/replication/replication-index.ts | 2 +- modules/module-convex/test/src/ConvexApiClient.test.ts | 2 +- .../module-convex/test/src/ConvexCheckpoints.test.ts | 2 +- modules/module-convex/test/src/ConvexLSN.test.ts | 2 +- .../test/src/ConvexRouteAPIAdapter.test.ts | 4 ++-- .../module-convex/test/src/convex-to-sqlite.test.ts | 10 +++++----- modules/module-convex/test/src/slow_tests.test.ts | 2 +- modules/module-convex/test/src/util.ts | 4 ++-- packages/schema/src/scripts/compile-json-schema.ts | 2 +- 11 files changed, 20 insertions(+), 19 deletions(-) diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json index a0eeddbdf..941fd9fe3 100644 --- a/modules/module-convex/package.json +++ b/modules/module-convex/package.json @@ -11,10 +11,11 @@ }, "scripts": { "build": "tsc -b", - "build:tests": "tsc -b test/tsconfig.json", + "build:tests": "pnpm build:convex && tsc -b test/tsconfig.json", + "build:convex": "convex codegen", "clean": "rm -rf ./dist && tsc -b --clean", "test": "vitest", - "dev:convex": "pnpm exec convex dev" + "dev:convex": "convex dev" }, "exports": { ".": { diff --git a/modules/module-convex/src/common/convex-to-sqlite.ts b/modules/module-convex/src/common/convex-to-sqlite.ts index 8113be849..2acfddd3e 100644 --- a/modules/module-convex/src/common/convex-to-sqlite.ts +++ b/modules/module-convex/src/common/convex-to-sqlite.ts @@ -1,11 +1,11 @@ +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; import { - DatabaseInputValue, DatabaseInputRow, + DatabaseInputValue, ExpressionType, SqliteInputRow, toSyncRulesRow } from '@powersync/service-sync-rules'; -import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; import { ConvexRawDocument } from '../client/ConvexApiClient.js'; /** diff --git a/modules/module-convex/src/replication/replication-index.ts b/modules/module-convex/src/replication/replication-index.ts index 1a3436644..427c400fd 100644 --- a/modules/module-convex/src/replication/replication-index.ts +++ b/modules/module-convex/src/replication/replication-index.ts @@ -1,6 +1,6 @@ export * from './ConvexConnectionManager.js'; export * from './ConvexConnectionManagerFactory.js'; export * from './ConvexErrorRateLimiter.js'; -export * from './ConvexReplicator.js'; export * from './ConvexReplicationJob.js'; +export * from './ConvexReplicator.js'; export * from './ConvexStream.js'; diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index 54f49138b..b68dc89c1 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -1,7 +1,7 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; import { ConvexApiClient } from '@module/client/ConvexApiClient.js'; import { CONVEX_CHECKPOINT_TABLE } from '@module/common/ConvexCheckpoints.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; +import { afterEach, describe, expect, it, vi } from 'vitest'; const baseConfig = normalizeConnectionConfig({ type: 'convex', diff --git a/modules/module-convex/test/src/ConvexCheckpoints.test.ts b/modules/module-convex/test/src/ConvexCheckpoints.test.ts index fa07f7a5a..7e2f64051 100644 --- a/modules/module-convex/test/src/ConvexCheckpoints.test.ts +++ b/modules/module-convex/test/src/ConvexCheckpoints.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from 'vitest'; import { isConvexCheckpointTable } from '@module/common/ConvexCheckpoints.js'; +import { describe, expect, it } from 'vitest'; describe('ConvexCheckpoints', () => { it('recognizes the checkpoint table name', () => { diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts index d3591902c..8bb553698 100644 --- a/modules/module-convex/test/src/ConvexLSN.test.ts +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from 'vitest'; import { parseConvexLsn, toConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; +import { describe, expect, it } from 'vitest'; describe('Convex cursor LSN helpers', () => { it('validates and round-trips the numeric cursor', () => { diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index 3a79d0edc..894a60bbf 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -1,8 +1,8 @@ -import { ExpressionType, SqlSyncRules } from '@powersync/service-sync-rules'; -import { describe, expect, it, vi } from 'vitest'; import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; import { toConvexLsn } from '@module/common/ConvexLSN.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; +import { ExpressionType, SqlSyncRules } from '@powersync/service-sync-rules'; +import { describe, expect, it, vi } from 'vitest'; const HEAD_CURSOR = '1772817606884944123'; diff --git a/modules/module-convex/test/src/convex-to-sqlite.test.ts b/modules/module-convex/test/src/convex-to-sqlite.test.ts index 383d329a4..fd920da3f 100644 --- a/modules/module-convex/test/src/convex-to-sqlite.test.ts +++ b/modules/module-convex/test/src/convex-to-sqlite.test.ts @@ -1,3 +1,8 @@ +import { + readConvexFieldType, + toExpressionTypeFromConvexType, + toSqliteInputRow +} from '@module/common/convex-to-sqlite.js'; import { applyRowContext, CompatibilityContext, @@ -6,11 +11,6 @@ import { isJsonValue } from '@powersync/service-sync-rules'; import { describe, expect, it } from 'vitest'; -import { - readConvexFieldType, - toExpressionTypeFromConvexType, - toSqliteInputRow -} from '@module/common/convex-to-sqlite.js'; const context = new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }); diff --git a/modules/module-convex/test/src/slow_tests.test.ts b/modules/module-convex/test/src/slow_tests.test.ts index 42f809488..87a13dcf4 100644 --- a/modules/module-convex/test/src/slow_tests.test.ts +++ b/modules/module-convex/test/src/slow_tests.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; -import { ConvexStreamTestContext, INITIALIZED_MONGO_STORAGE_FACTORY, makeConvexConnectionManager } from './util.js'; import { env } from './env.js'; +import { ConvexStreamTestContext, INITIALIZED_MONGO_STORAGE_FACTORY, makeConvexConnectionManager } from './util.js'; describe.runIf(env.SLOW_TESTS && !!env.CONVEX_DEPLOY_KEY)('convex slow tests', { timeout: 120_000 }, function () { test('connects to Convex and lists table schemas', async () => { diff --git a/modules/module-convex/test/src/util.ts b/modules/module-convex/test/src/util.ts index 6a4e2bf91..9870e5988 100644 --- a/modules/module-convex/test/src/util.ts +++ b/modules/module-convex/test/src/util.ts @@ -1,4 +1,3 @@ -import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import { BucketStorageFactory, createCoreReplicationMetrics, @@ -10,11 +9,12 @@ import { updateSyncRulesFromYaml } from '@powersync/service-core'; import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; +import * as mongo_storage from '@powersync/service-module-mongodb-storage'; +import { ZERO_LSN } from '@module/common/ConvexLSN.js'; import { ConvexConnectionManager } from '@module/replication/ConvexConnectionManager.js'; import { ConvexStream, ConvexStreamOptions } from '@module/replication/ConvexStream.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; -import { ZERO_LSN } from '@module/common/ConvexLSN.js'; import { env } from './env.js'; export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ diff --git a/packages/schema/src/scripts/compile-json-schema.ts b/packages/schema/src/scripts/compile-json-schema.ts index b76307abe..9d68b053b 100644 --- a/packages/schema/src/scripts/compile-json-schema.ts +++ b/packages/schema/src/scripts/compile-json-schema.ts @@ -1,6 +1,6 @@ +import { ConvexConnectionConfig } from '@powersync/service-module-convex/types'; import { MongoStorageConfig } from '@powersync/service-module-mongodb-storage/types'; import { MongoConnectionConfig } from '@powersync/service-module-mongodb/types'; -import { ConvexConnectionConfig } from '@powersync/service-module-convex/types'; import { MSSQLConnectionConfig } from '@powersync/service-module-mssql/types'; import { MySQLConnectionConfig } from '@powersync/service-module-mysql/types'; import { PostgresStorageConfig } from '@powersync/service-module-postgres-storage/types'; From 1c9b72be611c937fe0e3375b38af0cc3b01278f2 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 6 May 2026 13:39:12 +0200 Subject: [PATCH 55/86] track convex generated code --- .prettierignore | 3 +- modules/module-convex/convex/.gitignore | 1 - modules/module-convex/convex/README.md | 1 + .../module-convex/convex/_generated/api.d.ts | 55 +++++++ .../module-convex/convex/_generated/api.js | 23 +++ .../convex/_generated/dataModel.d.ts | 60 ++++++++ .../convex/_generated/server.d.ts | 143 ++++++++++++++++++ .../module-convex/convex/_generated/server.js | 93 ++++++++++++ 8 files changed, 377 insertions(+), 2 deletions(-) delete mode 100644 modules/module-convex/convex/.gitignore create mode 100644 modules/module-convex/convex/README.md create mode 100644 modules/module-convex/convex/_generated/api.d.ts create mode 100644 modules/module-convex/convex/_generated/api.js create mode 100644 modules/module-convex/convex/_generated/dataModel.d.ts create mode 100644 modules/module-convex/convex/_generated/server.d.ts create mode 100644 modules/module-convex/convex/_generated/server.js diff --git a/.prettierignore b/.prettierignore index aeeb3baa6..4cbdb8700 100644 --- a/.prettierignore +++ b/.prettierignore @@ -8,4 +8,5 @@ pnpm-lock.yaml **/*.sql # Generated files packages/schema/json-schema -packages/sync-rules/schema \ No newline at end of file +packages/sync-rules/schema +modules/module-convex/convex/_generated \ No newline at end of file diff --git a/modules/module-convex/convex/.gitignore b/modules/module-convex/convex/.gitignore deleted file mode 100644 index 94f0f0d9a..000000000 --- a/modules/module-convex/convex/.gitignore +++ /dev/null @@ -1 +0,0 @@ -_generated \ No newline at end of file diff --git a/modules/module-convex/convex/README.md b/modules/module-convex/convex/README.md new file mode 100644 index 000000000..373f9f488 --- /dev/null +++ b/modules/module-convex/convex/README.md @@ -0,0 +1 @@ +Note that the \_generated/ folder is intentionally tracked with Git. See https://docs.convex.dev/understanding/best-practices/other-recommendations#check-generated-code-into-version-control. diff --git a/modules/module-convex/convex/_generated/api.d.ts b/modules/module-convex/convex/_generated/api.d.ts new file mode 100644 index 000000000..cd335ca88 --- /dev/null +++ b/modules/module-convex/convex/_generated/api.d.ts @@ -0,0 +1,55 @@ +/* eslint-disable */ +/** + * Generated `api` utility. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import type * as lists from "../lists.js"; +import type * as powersync_checkpoints from "../powersync_checkpoints.js"; +import type * as todos from "../todos.js"; +import type * as utils_collections from "../utils/collections.js"; + +import type { + ApiFromModules, + FilterApi, + FunctionReference, +} from "convex/server"; + +declare const fullApi: ApiFromModules<{ + lists: typeof lists; + powersync_checkpoints: typeof powersync_checkpoints; + todos: typeof todos; + "utils/collections": typeof utils_collections; +}>; + +/** + * A utility for referencing Convex functions in your app's public API. + * + * Usage: + * ```js + * const myFunctionReference = api.myModule.myFunction; + * ``` + */ +export declare const api: FilterApi< + typeof fullApi, + FunctionReference +>; + +/** + * A utility for referencing Convex functions in your app's internal API. + * + * Usage: + * ```js + * const myFunctionReference = internal.myModule.myFunction; + * ``` + */ +export declare const internal: FilterApi< + typeof fullApi, + FunctionReference +>; + +export declare const components: {}; diff --git a/modules/module-convex/convex/_generated/api.js b/modules/module-convex/convex/_generated/api.js new file mode 100644 index 000000000..44bf98583 --- /dev/null +++ b/modules/module-convex/convex/_generated/api.js @@ -0,0 +1,23 @@ +/* eslint-disable */ +/** + * Generated `api` utility. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import { anyApi, componentsGeneric } from "convex/server"; + +/** + * A utility for referencing Convex functions in your app's API. + * + * Usage: + * ```js + * const myFunctionReference = api.myModule.myFunction; + * ``` + */ +export const api = anyApi; +export const internal = anyApi; +export const components = componentsGeneric(); diff --git a/modules/module-convex/convex/_generated/dataModel.d.ts b/modules/module-convex/convex/_generated/dataModel.d.ts new file mode 100644 index 000000000..f97fd1942 --- /dev/null +++ b/modules/module-convex/convex/_generated/dataModel.d.ts @@ -0,0 +1,60 @@ +/* eslint-disable */ +/** + * Generated data model types. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import type { + DataModelFromSchemaDefinition, + DocumentByName, + TableNamesInDataModel, + SystemTableNames, +} from "convex/server"; +import type { GenericId } from "convex/values"; +import schema from "../schema.js"; + +/** + * The names of all of your Convex tables. + */ +export type TableNames = TableNamesInDataModel; + +/** + * The type of a document stored in Convex. + * + * @typeParam TableName - A string literal type of the table name (like "users"). + */ +export type Doc = DocumentByName< + DataModel, + TableName +>; + +/** + * An identifier for a document in Convex. + * + * Convex documents are uniquely identified by their `Id`, which is accessible + * on the `_id` field. To learn more, see [Document IDs](https://docs.convex.dev/using/document-ids). + * + * Documents can be loaded using `db.get(tableName, id)` in query and mutation functions. + * + * IDs are just strings at runtime, but this type can be used to distinguish them from other + * strings when type checking. + * + * @typeParam TableName - A string literal type of the table name (like "users"). + */ +export type Id = + GenericId; + +/** + * A type describing your Convex data model. + * + * This type includes information about what tables you have, the type of + * documents stored in those tables, and the indexes defined on them. + * + * This type is used to parameterize methods like `queryGeneric` and + * `mutationGeneric` to make them type-safe. + */ +export type DataModel = DataModelFromSchemaDefinition; diff --git a/modules/module-convex/convex/_generated/server.d.ts b/modules/module-convex/convex/_generated/server.d.ts new file mode 100644 index 000000000..bec05e681 --- /dev/null +++ b/modules/module-convex/convex/_generated/server.d.ts @@ -0,0 +1,143 @@ +/* eslint-disable */ +/** + * Generated utilities for implementing server-side Convex query and mutation functions. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import { + ActionBuilder, + HttpActionBuilder, + MutationBuilder, + QueryBuilder, + GenericActionCtx, + GenericMutationCtx, + GenericQueryCtx, + GenericDatabaseReader, + GenericDatabaseWriter, +} from "convex/server"; +import type { DataModel } from "./dataModel.js"; + +/** + * Define a query in this Convex app's public API. + * + * This function will be allowed to read your Convex database and will be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export declare const query: QueryBuilder; + +/** + * Define a query that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to read from your Convex database. It will not be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export declare const internalQuery: QueryBuilder; + +/** + * Define a mutation in this Convex app's public API. + * + * This function will be allowed to modify your Convex database and will be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export declare const mutation: MutationBuilder; + +/** + * Define a mutation that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to modify your Convex database. It will not be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export declare const internalMutation: MutationBuilder; + +/** + * Define an action in this Convex app's public API. + * + * An action is a function which can execute any JavaScript code, including non-deterministic + * code and code with side-effects, like calling third-party services. + * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. + * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. + * + * @param func - The action. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped action. Include this as an `export` to name it and make it accessible. + */ +export declare const action: ActionBuilder; + +/** + * Define an action that is only accessible from other Convex functions (but not from the client). + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped function. Include this as an `export` to name it and make it accessible. + */ +export declare const internalAction: ActionBuilder; + +/** + * Define an HTTP action. + * + * The wrapped function will be used to respond to HTTP requests received + * by a Convex deployment if the requests matches the path and method where + * this action is routed. Be sure to route your httpAction in `convex/http.js`. + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument + * and a Fetch API `Request` object as its second. + * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. + */ +export declare const httpAction: HttpActionBuilder; + +/** + * A set of services for use within Convex query functions. + * + * The query context is passed as the first argument to any Convex query + * function run on the server. + * + * This differs from the {@link MutationCtx} because all of the services are + * read-only. + */ +export type QueryCtx = GenericQueryCtx; + +/** + * A set of services for use within Convex mutation functions. + * + * The mutation context is passed as the first argument to any Convex mutation + * function run on the server. + */ +export type MutationCtx = GenericMutationCtx; + +/** + * A set of services for use within Convex action functions. + * + * The action context is passed as the first argument to any Convex action + * function run on the server. + */ +export type ActionCtx = GenericActionCtx; + +/** + * An interface to read from the database within Convex query functions. + * + * The two entry points are {@link DatabaseReader.get}, which fetches a single + * document by its {@link Id}, or {@link DatabaseReader.query}, which starts + * building a query. + */ +export type DatabaseReader = GenericDatabaseReader; + +/** + * An interface to read from and write to the database within Convex mutation + * functions. + * + * Convex guarantees that all writes within a single mutation are + * executed atomically, so you never have to worry about partial writes leaving + * your data in an inconsistent state. See [the Convex Guide](https://docs.convex.dev/understanding/convex-fundamentals/functions#atomicity-and-optimistic-concurrency-control) + * for the guarantees Convex provides your functions. + */ +export type DatabaseWriter = GenericDatabaseWriter; diff --git a/modules/module-convex/convex/_generated/server.js b/modules/module-convex/convex/_generated/server.js new file mode 100644 index 000000000..bf3d25ad3 --- /dev/null +++ b/modules/module-convex/convex/_generated/server.js @@ -0,0 +1,93 @@ +/* eslint-disable */ +/** + * Generated utilities for implementing server-side Convex query and mutation functions. + * + * THIS CODE IS AUTOMATICALLY GENERATED. + * + * To regenerate, run `npx convex dev`. + * @module + */ + +import { + actionGeneric, + httpActionGeneric, + queryGeneric, + mutationGeneric, + internalActionGeneric, + internalMutationGeneric, + internalQueryGeneric, +} from "convex/server"; + +/** + * Define a query in this Convex app's public API. + * + * This function will be allowed to read your Convex database and will be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export const query = queryGeneric; + +/** + * Define a query that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to read from your Convex database. It will not be accessible from the client. + * + * @param func - The query function. It receives a {@link QueryCtx} as its first argument. + * @returns The wrapped query. Include this as an `export` to name it and make it accessible. + */ +export const internalQuery = internalQueryGeneric; + +/** + * Define a mutation in this Convex app's public API. + * + * This function will be allowed to modify your Convex database and will be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export const mutation = mutationGeneric; + +/** + * Define a mutation that is only accessible from other Convex functions (but not from the client). + * + * This function will be allowed to modify your Convex database. It will not be accessible from the client. + * + * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument. + * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible. + */ +export const internalMutation = internalMutationGeneric; + +/** + * Define an action in this Convex app's public API. + * + * An action is a function which can execute any JavaScript code, including non-deterministic + * code and code with side-effects, like calling third-party services. + * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive. + * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}. + * + * @param func - The action. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped action. Include this as an `export` to name it and make it accessible. + */ +export const action = actionGeneric; + +/** + * Define an action that is only accessible from other Convex functions (but not from the client). + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument. + * @returns The wrapped function. Include this as an `export` to name it and make it accessible. + */ +export const internalAction = internalActionGeneric; + +/** + * Define an HTTP action. + * + * The wrapped function will be used to respond to HTTP requests received + * by a Convex deployment if the requests matches the path and method where + * this action is routed. Be sure to route your httpAction in `convex/http.js`. + * + * @param func - The function. It receives an {@link ActionCtx} as its first argument + * and a Fetch API `Request` object as its second. + * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up. + */ +export const httpAction = httpActionGeneric; From bd8764f37c1eb50887aaa74becd2fda46aeb1d05 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 6 May 2026 14:37:51 +0200 Subject: [PATCH 56/86] Add powersync checkpoint test to connection testing --- modules/module-convex/README.md | 2 +- .../module-convex/src/module/ConvexModule.ts | 65 ++++++++++++++++++- .../ConvexModule.integration.test.ts | 11 ++++ 3 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 modules/module-convex/test/src/integration/ConvexModule.integration.test.ts diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 296881b5f..b335506df 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -107,7 +107,7 @@ The content below is written in an agents.md style describing the behavior of `m - Convex `json_schemas` does not provide a schema change token or revision cursor that can be checkpointed. - Current behavior uses `json_schemas` for discovery/debug, but does not continuously diff source schema versions. -- Operational caveat: if Convex schema changes (tables or columns), developers must review and redeploy Sync Streams manually. +- Schema changes are not automatically supported at this point. If Convex schema changes (tables or columns), developers must review and redeploy Sync Streams rules manually so PowerSync re-resolves the schema and re-replicates affected streams. - Future improvement: cache a canonicalized `json_schemas` hash, poll periodically, and raise diagnostics when schema drift is detected. ## 7) Datatype Mapping diff --git a/modules/module-convex/src/module/ConvexModule.ts b/modules/module-convex/src/module/ConvexModule.ts index cae5daffd..69b1abe06 100644 --- a/modules/module-convex/src/module/ConvexModule.ts +++ b/modules/module-convex/src/module/ConvexModule.ts @@ -7,6 +7,7 @@ import { TearDownOptions } from '@powersync/service-core'; import { ConvexRouteAPIAdapter } from '../api/ConvexRouteAPIAdapter.js'; +import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; import { ConvexConnectionManagerFactory } from '../replication/ConvexConnectionManagerFactory.js'; import { ConvexErrorRateLimiter } from '../replication/ConvexErrorRateLimiter.js'; @@ -62,8 +63,70 @@ export class ConvexModule extends replication.ReplicationModule { const connectionManager = new ConvexConnectionManager(normalizedConfig); + const missingMutatorErrorFragment = ` +Define a mutator for the PowerSync service to use + +convex/powersync_checkpoints.ts +\`\`\`TypeScript +import { mutation } from './_generated/server.js'; + +export const createCheckpoint = mutation({ + args: {}, + handler: async (ctx) => { + const existing = await ctx.db.query('powersync_checkpoints').first(); + + if (existing) { + await ctx.db.patch(existing._id, { last_updated: Date.now() }); + } else { + await ctx.db.insert('powersync_checkpoints', { last_updated: Date.now() }); + } + } +}); +\`\`\` + `; try { - await connectionManager.client.getJsonSchemas(); + // Check if the database is reachable by fetching the schema. + const schema = await connectionManager.client.getJsonSchemas().catch((error) => { + throw new Error('Could not fetch Convex schema for provided connection configuration.', { + cause: error + }); + }); + + if (!schema.tables.find((table) => table.tableName == CONVEX_CHECKPOINT_TABLE)) { + throw new Error(` +Could not find the ${CONVEX_CHECKPOINT_TABLE} table in the schema. + +Define the ${CONVEX_CHECKPOINT_TABLE} table in the Convex schema: + +convex/schema.ts +\`\`\`TypeScript +//... + +export default defineSchema({ + // ... your other tables + + powersync_checkpoints: defineTable({ + last_updated: v.float64() + }) +}); +\`\`\` + +${missingMutatorErrorFragment} +`); + } + + // Check that the PowerSync checkpoint table and mutation are deployed. + // It should be safe to update this table at any point. We only use it for emiting a replication event. + await connectionManager.client.createWriteCheckpointMarker().catch((error) => { + throw new Error( + ` +Could not call the createCheckpoint mutator. + +${missingMutatorErrorFragment} + `, + { cause: error } + ); + }); } finally { await connectionManager.end(); } diff --git a/modules/module-convex/test/src/integration/ConvexModule.integration.test.ts b/modules/module-convex/test/src/integration/ConvexModule.integration.test.ts new file mode 100644 index 000000000..efd4cdc67 --- /dev/null +++ b/modules/module-convex/test/src/integration/ConvexModule.integration.test.ts @@ -0,0 +1,11 @@ +import { ConvexModule } from '@module/module/ConvexModule.js'; +import { describe, expect, test } from 'vitest'; +import { TEST_CONNECTION_OPTIONS } from '../test-utils/util.js'; + +describe('ConvexModule', () => { + test('Testing Connections should succeed for valid connections', async () => { + // It's not easy to test for in invalid Convex backend, since we only configure a valid backend. + const result = await ConvexModule.testConnection(TEST_CONNECTION_OPTIONS); + expect(result.connectionDescription).eq(TEST_CONNECTION_OPTIONS.deployment_url); + }); +}); From 5a56a0ed17483d461b4fb318a0739f2a0f079950 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 6 May 2026 16:39:57 +0200 Subject: [PATCH 57/86] add initial replication benchmark --- .../module-convex/convex/_generated/api.d.ts | 2 + modules/module-convex/convex/benchmark.ts | 45 ++++++++ .../src/bench/initial_replication.test.ts | 106 ++++++++++++++++++ modules/module-convex/test/src/env.ts | 3 +- .../module-convex/test/src/test-utils/util.ts | 3 + 5 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 modules/module-convex/convex/benchmark.ts create mode 100644 modules/module-convex/test/src/bench/initial_replication.test.ts diff --git a/modules/module-convex/convex/_generated/api.d.ts b/modules/module-convex/convex/_generated/api.d.ts index cd335ca88..8abbb03e8 100644 --- a/modules/module-convex/convex/_generated/api.d.ts +++ b/modules/module-convex/convex/_generated/api.d.ts @@ -8,6 +8,7 @@ * @module */ +import type * as benchmark from "../benchmark.js"; import type * as lists from "../lists.js"; import type * as powersync_checkpoints from "../powersync_checkpoints.js"; import type * as todos from "../todos.js"; @@ -20,6 +21,7 @@ import type { } from "convex/server"; declare const fullApi: ApiFromModules<{ + benchmark: typeof benchmark; lists: typeof lists; powersync_checkpoints: typeof powersync_checkpoints; todos: typeof todos; diff --git a/modules/module-convex/convex/benchmark.ts b/modules/module-convex/convex/benchmark.ts new file mode 100644 index 000000000..576d693e6 --- /dev/null +++ b/modules/module-convex/convex/benchmark.ts @@ -0,0 +1,45 @@ +import { v } from 'convex/values'; +import { mutation } from './_generated/server.js'; + +export const BENCHMARK_LIST_PAYLOAD = { + uuid: 'benchmark-list', + name: 'Benchmark list' +} as const; + +export const BENCHMARK_TODO_PAYLOAD = { + uuid: 'benchmark-todo', + description: 'Benchmark todo', + list_uuid: BENCHMARK_LIST_PAYLOAD.uuid +} as const; + +/** + * Creates seed data for sync replication benchmarking purposes. + * + * Logical payload estimate for the current row shape is calculated in the benchmark test from + * these payload constants. Each loop creates one list and one todo, so the estimate is: + * rowsPerTable * (encoded JSON list bytes + encoded JSON todo bytes + representative list_id bytes). + * Convex write-limit accounting can be higher because it also includes document metadata, + * indexes and storage internals. + */ +export const seedInitialReplicationBatch = mutation({ + args: { + rowsPerTable: v.number() + }, + handler: async (ctx, args) => { + for (let i = 0; i < args.rowsPerTable; i++) { + const listId = await ctx.db.insert('lists', { + ...BENCHMARK_LIST_PAYLOAD + }); + + await ctx.db.insert('todos', { + ...BENCHMARK_TODO_PAYLOAD, + list_id: listId + }); + } + + return { + lists: args.rowsPerTable, + todos: args.rowsPerTable + }; + } +}); diff --git a/modules/module-convex/test/src/bench/initial_replication.test.ts b/modules/module-convex/test/src/bench/initial_replication.test.ts new file mode 100644 index 000000000..8983715af --- /dev/null +++ b/modules/module-convex/test/src/bench/initial_replication.test.ts @@ -0,0 +1,106 @@ +import { afterAll, describe, test } from 'vitest'; +import { env } from '../env.js'; +import { ConvexStreamTestContext } from '../test-utils/ConvexStreamTestContext.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY } from '../test-utils/util.js'; + +/** + * Vitest has built in benchmarking functionality, but, it seems non-trivial to preseed the database. + * Run this by running: + * ```bash + * export SHOULD_RUN_BENCHMARK=true + * vitest initial_replication.test.ts + * ``` + */ +const SEED_BATCH_SIZE = 1_000; +const ESTIMATED_STORAGE_BYTES_PER_1K_ROWS = 350 * 1024; + +const BENCHMARK_SYNC_RULES = ` +bucket_definitions: + global: + data: + - SELECT uuid as id, name FROM "lists" + - SELECT uuid as id, description, list_uuid FROM "todos" +`; + +describe + .skipIf(!env.SHOULD_RUN_BENCHMARK) + .sequential('Convex initial replication benchmark', { timeout: Infinity }, () => { + let testResults = new Map(); + + afterAll(() => { + console.log('Test results'); + console.log('| Total Rows | Estimated Size (MiB) | Elapsed Time (ms) | Estimated Rate (MiB/s) |'); + for (const [totalRows, elapsedTime] of testResults.entries()) { + const estimatedSize = estimateStorageSize(totalRows); + const estimatedRate = estimateReplicationRate(estimatedSize.bytes, elapsedTime); + console.log( + `| ${totalRows} | ${estimatedSize.mib.toFixed(2)} | ${Math.round(elapsedTime)} | ${estimatedRate.toFixed( + 2 + )} |` + ); + } + }); + + function defineTest({ totalRows }: { totalRows: number }) { + return test(`starting initial replication for ${totalRows} total rows`, async () => { + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY.factory, { + doNotClear: false // Will clear the source (Convex) database + }); + // seed with data + await seedBenchmarkRows(context, totalRows); + + await context.updateSyncRules(BENCHMARK_SYNC_RULES); + + const startedAt = performance.now(); + await context.replicateSnapshot(); + const elapsedMs = performance.now() - startedAt; + + console.info(`Initial Convex replication benchmark replicated ${totalRows} rows in ${Math.round(elapsedMs)}ms`); + + testResults.set(totalRows, elapsedMs); + }); + } + + // Now register the tests for each dataset + // Storage estimate is based on current observed storage use: 1_000 rows ~= 350 KiB. + defineTest({ + totalRows: 10_000 + }); + + defineTest({ + totalRows: 50_000 + }); + + defineTest({ + totalRows: 100_000 + }); + }); + +function estimateStorageSize(totalRows: number) { + const bytes = (totalRows / 1_000) * ESTIMATED_STORAGE_BYTES_PER_1K_ROWS; + return { + bytes, + mib: bytes / 1024 / 1024 + }; +} + +function estimateReplicationRate(bytes: number, elapsedMs: number) { + return bytes / 1024 / 1024 / (elapsedMs / 1_000); +} + +async function seedBenchmarkRows(context: ConvexStreamTestContext, totalRows: number) { + const totalRowsPerTable = Math.floor(totalRows / 2); + for (let counter = 0; counter < totalRowsPerTable; counter += SEED_BATCH_SIZE) { + const count = Math.min(SEED_BATCH_SIZE, totalRowsPerTable - counter); + await context.backend.client.mutation(context.backend.api.benchmark.seedInitialReplicationBatch, { + rowsPerTable: count + }); + + console.info(`Seeded ${(counter + count) * 2}/${totalRows} rows.`); + /** + * Nasty workarround for default Convex write limits + * {"code":"TooManyWrites","message":"Too many writes per second. Your deployment is limited to 4 MiB bytes written per 1 second. Reduce your write rate or upgrade to a larger deployment."} + */ + await new Promise((r) => setTimeout(r, 1_000)); + } +} diff --git a/modules/module-convex/test/src/env.ts b/modules/module-convex/test/src/env.ts index f519b2e06..af7f7cbb6 100644 --- a/modules/module-convex/test/src/env.ts +++ b/modules/module-convex/test/src/env.ts @@ -25,5 +25,6 @@ export const env = utils.collectEnvironmentVariables({ CI: utils.type.boolean.default('false'), SLOW_TESTS: utils.type.boolean.default('false'), TEST_MONGO_STORAGE: utils.type.boolean.default('true'), - TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') + TEST_POSTGRES_STORAGE: utils.type.boolean.default('true'), + SHOULD_RUN_BENCHMARK: utils.type.boolean.default('false') }); diff --git a/modules/module-convex/test/src/test-utils/util.ts b/modules/module-convex/test/src/test-utils/util.ts index 47123e03d..fe51aa20c 100644 --- a/modules/module-convex/test/src/test-utils/util.ts +++ b/modules/module-convex/test/src/test-utils/util.ts @@ -78,13 +78,16 @@ export async function clearTestDb(connection: TestConvexConnection) { // Delete all lists let deletedCount = 0; + console.info(`Clearing Convex DB`); do { deletedCount = await client.mutation(api.lists.deleteBatch, {}); + console.info(`Cleared ${deletedCount} lists`); } while (deletedCount > 0); deletedCount = 0; do { deletedCount = await client.mutation(api.todos.deleteBatch, {}); + console.info(`Cleared ${deletedCount} todos`); } while (deletedCount > 0); } From f99c32bcf1c84fd90a018fb5dcb68435f12792fe Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 12 May 2026 10:12:08 +0200 Subject: [PATCH 58/86] cleanup --- modules/module-convex/README.md | 5 +++ .../module-convex/test/src/test-utils/util.ts | 31 ------------------- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index b335506df..5f001db26 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -2,6 +2,11 @@ Convex replication module for PowerSync. +> [!WARNING] +> The Convex replicator is currently released as an alpha feature. APIs, configuration, metrics, schema-change handling, +> and replication behavior may change before it is considered stable. Test carefully before using it with production +> workloads. + ## Configuration ```yaml diff --git a/modules/module-convex/test/src/test-utils/util.ts b/modules/module-convex/test/src/test-utils/util.ts index fe51aa20c..b99263df0 100644 --- a/modules/module-convex/test/src/test-utils/util.ts +++ b/modules/module-convex/test/src/test-utils/util.ts @@ -90,34 +90,3 @@ export async function clearTestDb(connection: TestConvexConnection) { console.info(`Cleared ${deletedCount} todos`); } while (deletedCount > 0); } - -// export async function getClientCheckpoint( -// db: pgwire.PgClient, -// storageFactory: BucketStorageFactory, -// options?: { timeout?: number } -// ): Promise { -// const start = Date.now(); - -// const api = new PostgresRouteAPIAdapter(db); -// const lsn = await api.createReplicationHead(async (lsn) => lsn); - -// // This old API needs a persisted checkpoint id. -// // Since we don't use LSNs anymore, the only way to get that is to wait. - -// const timeout = options?.timeout ?? 50_000; - -// logger.info(`Waiting for LSN checkpoint: ${lsn}`); -// while (Date.now() - start < timeout) { -// const storage = await storageFactory.getActiveStorage(); -// const cp = await storage?.getCheckpoint(); - -// if (cp?.lsn != null && cp.lsn >= lsn) { -// logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`); -// return cp.checkpoint; -// } - -// await new Promise((resolve) => setTimeout(resolve, 5)); -// } - -// throw new Error('Timeout while waiting for checkpoint'); -// } From 00996c5820cbaba41e6ab8efb859d570c1c2d6db Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 12 May 2026 10:12:40 +0200 Subject: [PATCH 59/86] Add option for clearingSource to test context. --- .../test/src/test-utils/ConvexStreamTestContext.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts b/modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts index 2ff90c6a5..120ecf9b3 100644 --- a/modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts +++ b/modules/module-convex/test/src/test-utils/ConvexStreamTestContext.ts @@ -23,14 +23,19 @@ export class ConvexStreamTestContext extends AbstractStreamTestContext { */ static async open( factory: (options: storage.TestStorageOptions) => Promise, - options?: { doNotClear?: boolean; storageVersion?: number; streamOptions?: Partial } + options?: { + doNotClear?: boolean; + storageVersion?: number; + streamOptions?: Partial; + clearSource?: boolean; + } ) { const f = await factory({ doNotClear: options?.doNotClear }); const connectionManager = new ConvexConnectionManager(TEST_CONNECTION_OPTIONS); const convexBackend = connectConvex(); - if (!options?.doNotClear) { + if (options?.clearSource ?? !options?.doNotClear) { await clearTestDb(convexBackend); } From f08123e09baf2f5e54b50cd33824a576ee1abec6 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 12 May 2026 10:27:12 +0200 Subject: [PATCH 60/86] add changeset --- .changeset/breezy-sites-mix.md | 5 +++++ .changeset/bright-facts-nail.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/breezy-sites-mix.md create mode 100644 .changeset/bright-facts-nail.md diff --git a/.changeset/breezy-sites-mix.md b/.changeset/breezy-sites-mix.md new file mode 100644 index 000000000..800f96a82 --- /dev/null +++ b/.changeset/breezy-sites-mix.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-image': minor +--- + +Added initial support for Convex replication. diff --git a/.changeset/bright-facts-nail.md b/.changeset/bright-facts-nail.md new file mode 100644 index 000000000..a4ef6eda5 --- /dev/null +++ b/.changeset/bright-facts-nail.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-module-convex': patch +--- + +Initial alpha release From 7421cbc5ad333adb9aaf8b935b0f5c5b8ac55bb9 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 12 May 2026 10:31:34 +0200 Subject: [PATCH 61/86] fix dev image release --- .github/workflows/development_image_release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/development_image_release.yaml b/.github/workflows/development_image_release.yaml index 92cc5ef7e..18013a142 100644 --- a/.github/workflows/development_image_release.yaml +++ b/.github/workflows/development_image_release.yaml @@ -49,7 +49,7 @@ jobs: # If no changesets are available the status check will fail # We should not continue if there are no changesets pnpm changeset status - pnpm changeset version --no-git-tag --snapshot dev + pnpm changeset version --snapshot dev # This uses the service's package.json version for the Docker Image tag # The changeset command above should change this to a dev package From 31bca10833a372bda4a78563ef8863dd3e525ebf Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 12 May 2026 17:22:48 +0200 Subject: [PATCH 62/86] cleanup ConvexAPIClient --- .changeset/twelve-poets-sort.md | 5 + .../src/schema/json-schema/keywords.ts | 5 +- modules/module-convex/package.json | 1 + .../src/client/ConvexAPITypes.ts | 112 +++++ .../src/client/ConvexApiClient.ts | 393 +++++------------- modules/module-convex/src/common/ConvexLSN.ts | 4 +- .../src/common/convex-to-sqlite.ts | 2 +- .../src/replication/ConvexStream.ts | 16 +- .../ConvexRouteAPIAdapter.integration.test.ts | 153 +++++++ pnpm-lock.yaml | 9 +- 10 files changed, 399 insertions(+), 301 deletions(-) create mode 100644 .changeset/twelve-poets-sort.md create mode 100644 modules/module-convex/src/client/ConvexAPITypes.ts create mode 100644 modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts diff --git a/.changeset/twelve-poets-sort.md b/.changeset/twelve-poets-sort.md new file mode 100644 index 000000000..21e70c3fe --- /dev/null +++ b/.changeset/twelve-poets-sort.md @@ -0,0 +1,5 @@ +--- +'@powersync/lib-services-framework': patch +--- + +Add bigint support for codec validations diff --git a/libs/lib-services/src/schema/json-schema/keywords.ts b/libs/lib-services/src/schema/json-schema/keywords.ts index 06ee104a1..5a9b91824 100644 --- a/libs/lib-services/src/schema/json-schema/keywords.ts +++ b/libs/lib-services/src/schema/json-schema/keywords.ts @@ -4,7 +4,7 @@ export const BufferNodeType: ajv.KeywordDefinition = { keyword: 'nodeType', metaSchema: { type: 'string', - enum: ['buffer', 'date'] + enum: ['bigint', 'buffer', 'date'] }, error: { message: ({ schemaCode }) => { @@ -13,6 +13,9 @@ export const BufferNodeType: ajv.KeywordDefinition = { }, code(context) { switch (context.schema) { + case 'bigint': { + return context.fail(ajv._`typeof ${context.data} != 'bigint'`); + } case 'buffer': { return context.fail(ajv._`!Buffer.isBuffer(${context.data})`); } diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json index 941fd9fe3..738ccccf3 100644 --- a/modules/module-convex/package.json +++ b/modules/module-convex/package.json @@ -35,6 +35,7 @@ "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", + "ajv": "^8.18.0", "bson": "^6.10.4", "ts-codec": "^1.3.0" }, diff --git a/modules/module-convex/src/client/ConvexAPITypes.ts b/modules/module-convex/src/client/ConvexAPITypes.ts new file mode 100644 index 000000000..a2b5dab4b --- /dev/null +++ b/modules/module-convex/src/client/ConvexAPITypes.ts @@ -0,0 +1,112 @@ +import { BufferNodeType } from '@powersync/lib-services-framework'; +import AJV from 'ajv'; +import * as t from 'ts-codec'; + +export const ConvexRawDocument = t + .object({ + _id: t.string.optional(), + _table: t.string.optional(), + _deleted: t.boolean.optional() + }) + .and(t.record(t.any)); + +export type ConvexRawDocument = t.Encoded; + +// Placeholder to represent bigints +export const bigint = t.codec( + 'bigint', + (decoded: bigint) => decoded, + (encoded: bigint) => encoded +); + +// Placeholder which allows validating bigint payloads +export const bigintParser = t.createParser(bigint._tag, () => ({ + nodeType: 'bigint' +})); + +export const ConvexDocumentDelta = ConvexRawDocument.and( + t.object({ + _ts: bigint + }) +); + +export type ConvexDocumentDelta = t.Encoded; + +export const ConvexTableSchema = t.object({ + tableName: t.string, + // JSON schema object + schema: t.record(t.any) +}); + +export type ConvexTableSchema = t.Encoded; + +export const ConvexJsonSchemasResult = t.object({ + tables: t.array(ConvexTableSchema) +}); + +export type ConvexJsonSchemasResult = t.Encoded; + +export const ConvexListSnapshotResult = t.object({ + snapshot: bigint, + cursor: t.string.or(t.Null), + hasMore: t.boolean, + values: t.array(ConvexRawDocument) +}); + +export type ConvexListSnapshotResult = t.Encoded; + +// These validators help assert the API structure response. +// These could be disabled for production. +export const ensureConvexListSnapshotResult = ensureResponseFormatValidator(ConvexListSnapshotResult); + +export const ConvexDocumentDeltasResult = t.object({ + cursor: bigint, + hasMore: t.boolean, + values: t.array(ConvexDocumentDelta) +}); + +export type ConvexDocumentDeltasResult = t.Encoded; + +// These validators help assert the API structure response. +// These could be disabled for production. +export const ensureConvexDocumentDeltasResult = ensureResponseFormatValidator(ConvexDocumentDeltasResult); + +export interface ConvexListSnapshotOptions { + snapshot?: string; + cursor?: string; + tableName?: string; + signal?: AbortSignal; +} + +export interface ConvexDocumentDeltasOptions { + cursor?: string; + signal?: AbortSignal; +} + +/** + * Performs a validation which ensures the input data matches the codec specification. + */ +export function ensureResponseFormatValidator( + codec: Codec +): (data: unknown) => t.Encoded { + const schema = t.generateJSONSchema(codec, { + parsers: [bigintParser], + allowAdditional: true + }); + const ajv = new AJV.Ajv({ + allErrors: true, + keywords: [BufferNodeType] + }); + + const validator = ajv.compile(schema); + return (data: unknown) => { + const isValid = validator(data); + if (!isValid) { + // This does not result in leaking failed data, it only logs the keys which failed validation + throw new Error( + `Invalid data received. Got parsing errors when checking data format. Keys which failed validation: ${ajv.errors?.map((e) => e.propertyName).join(', ')}` + ); + } + return data as t.Encoded; + }; +} diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index 701aad5d9..88a01f01c 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -1,55 +1,37 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { setTimeout as delay } from 'timers/promises'; +import * as t from 'ts-codec'; import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; import { NormalizedConvexConnectionConfig } from '../types/types.js'; +import { + ConvexDocumentDeltasOptions, + ConvexDocumentDeltasResult, + ConvexJsonSchemasResult, + ConvexListSnapshotOptions, + ConvexListSnapshotResult, + ensureConvexDocumentDeltasResult, + ensureConvexListSnapshotResult, + ensureResponseFormatValidator +} from './ConvexAPITypes.js'; const CONVEX_REQUEST_TIMEOUT_MS = 60_000; -export interface ConvexRawDocument { - _id?: string; - _table?: string; - _deleted?: boolean; - [key: string]: any; -} - -export interface ConvexDocumentDelta extends ConvexRawDocument { - _ts: bigint; -} - -export interface ConvexTableSchema { - tableName: string; - schema: Record; -} - -export interface ConvexJsonSchemasResult { - tables: ConvexTableSchema[]; - raw: Record; -} - -export interface ConvexListSnapshotOptions { - snapshot?: string; - cursor?: string; - tableName?: string; - signal?: AbortSignal; -} +const RawJsonSchemaResponse = t.record( + t.object({ + type: t.string, + properties: t.record(t.any) + }) +); -export interface ConvexListSnapshotResult { - snapshot: string; - cursor: string | null; - hasMore: boolean; - values: ConvexRawDocument[]; -} +const ensureRawJsonSchemaResponse = ensureResponseFormatValidator(RawJsonSchemaResponse); -export interface ConvexDocumentDeltasOptions { - cursor?: string; +type GetRequestParams = { + path: string; + params?: Record; signal?: AbortSignal; -} - -export interface ConvexDocumentDeltasResult { - cursor: string; - hasMore: boolean; - values: ConvexDocumentDelta[]; -} + extraHeaders?: Record; + includeJsonFormat?: boolean; +}; export class ConvexApiError extends Error { readonly status?: number; @@ -69,65 +51,55 @@ export class ConvexApiClient { constructor(private readonly config: NormalizedConvexConnectionConfig) {} async getJsonSchemas(options?: { signal?: AbortSignal }): Promise { - const payload = await this.requestJson({ - endpoint: 'json_schemas', - signal: options?.signal, - allowStreamingFallback: true - }); - + const raw = await this.performTypedGetRequest( + { + path: '/api/json_schemas', + signal: options?.signal + }, + ensureRawJsonSchemaResponse + ); + // Convex returns this as a map of {[tableName]: JSONSchema} for tables return { - tables: extractTableSchemas(payload), - raw: payload + tables: Object.entries(raw).map(([tableName, schema]) => ({ + tableName, + schema + })) }; } async listSnapshot(options: ConvexListSnapshotOptions): Promise { - const payload = await this.requestJson({ - endpoint: 'list_snapshot', - params: { - snapshot: options.snapshot, - cursor: options.cursor, - //TODO: NB: test this against cloud. Seems to be `tableName` in the docs, but `table_name` when self-hosting. - table_name: options.tableName + return await this.performTypedGetRequest( + { + path: '/api/list_snapshot', + params: { + snapshot: options.snapshot, + cursor: options.cursor, + table_name: options.tableName + }, + signal: options.signal }, - signal: options.signal, - allowStreamingFallback: true - }); - - const values = parseValues(payload); - const snapshot = parseSnapshot(payload, options.snapshot); - const cursor = payload.cursor == null ? null : stringifyCursor(payload.cursor); - - return { - snapshot, - cursor, - hasMore: parseHasMore(payload), - values - }; + ensureConvexListSnapshotResult + ); } async documentDeltas(options: ConvexDocumentDeltasOptions): Promise { - const payload = await this.requestJson({ - endpoint: 'document_deltas', - params: { - cursor: options.cursor + return await this.performTypedGetRequest( + { + path: '/api/document_deltas', + params: { + cursor: options.cursor + }, + signal: options.signal }, - signal: options.signal, - allowStreamingFallback: true - }); - - return { - cursor: stringifyCursor(payload.cursor), - hasMore: parseHasMore(payload), - values: parseValues(payload) as ConvexDocumentDelta[] - }; + ensureConvexDocumentDeltasResult + ); } async getGlobalSnapshotCursor(options?: { signal?: AbortSignal }): Promise { const page = await this.listSnapshot({ signal: options?.signal }); - return page.snapshot; + return page.snapshot.toString(); } async getHeadCursor(options?: { signal?: AbortSignal }): Promise { @@ -139,7 +111,7 @@ export class ConvexApiClient { await this.performRequest({ method: 'POST', - path: '/api/mutation', + url: new URL('/api/mutation', this.config.deployment_url), body: { path: `${CONVEX_CHECKPOINT_TABLE}:createCheckpoint`, args: {}, @@ -148,65 +120,61 @@ export class ConvexApiClient { signal: options?.signal, extraHeaders: { 'Content-Type': 'application/json' - }, - includeJsonFormat: false + } }); } - private async requestJson(options: { - endpoint: string; - params?: Record; - signal?: AbortSignal; - allowStreamingFallback?: boolean; - }): Promise> { - await this.assertHostAllowed(); + private async performGetRequest(options: GetRequestParams) { + const { path, params = {}, signal, extraHeaders, includeJsonFormat } = options; + + // `params` are mapped to url search params for GET requests + const url = new URL(path, this.config.deployment_url); + if (includeJsonFormat ?? true) { + url.searchParams.set('format', 'json'); + } + + for (const [key, value] of Object.entries(params)) { + if (value == null) { + continue; + } + url.searchParams.set(key, `${value}`); + } - const primaryPath = `/api/${options.endpoint}`; - const fallbackPath = `/api/streaming_export/${options.endpoint}`; + return this.performRequest({ + method: 'GET', + url, + extraHeaders, + signal + }); + } + private async performTypedGetRequest( + options: GetRequestParams, + validator: (data: unknown) => ResponseType + ) { + const rawResponse = await this.performGetRequest(options); try { - return await this.performRequest({ - path: primaryPath, - params: options.params, - signal: options.signal + return validator(rawResponse); + } catch (ex) { + throw new ConvexApiError({ + message: `Failed to validate Convex API request response format`, + retryable: false, + cause: ex }); - } catch (error) { - if ( - options.allowStreamingFallback && - error instanceof ConvexApiError && - error.status == 404 && - primaryPath != fallbackPath - ) { - return await this.performRequest({ - path: fallbackPath, - params: options.params, - signal: options.signal - }); - } - throw error; } } + /** + * Performs a request which expects a JSON response + */ private async performRequest(options: { - method?: 'GET' | 'POST'; - path: string; - params?: Record; + method: 'GET' | 'POST'; + url: URL; body?: unknown; signal?: AbortSignal; extraHeaders?: Record; - includeJsonFormat?: boolean; }): Promise> { - const url = new URL(options.path, this.config.deployment_url); - if (options.includeJsonFormat ?? true) { - url.searchParams.set('format', 'json'); - } - - for (const [key, value] of Object.entries(options.params ?? {})) { - if (value == null) { - continue; - } - url.searchParams.set(key, `${value}`); - } + const { method, url, body, extraHeaders = {}, signal: requestSignal } = options; const timeout = new AbortController(); const timeoutPromise = delay(CONVEX_REQUEST_TIMEOUT_MS, undefined, { @@ -216,26 +184,31 @@ export class ConvexApiClient { }); const signals = [timeout.signal]; - if (options.signal) { - signals.push(options.signal); + if (requestSignal) { + signals.push(requestSignal); } const signal = AbortSignal.any(signals); try { const response = await fetch(url, { - method: options.method ?? 'GET', + method, headers: { Authorization: `Convex ${this.config.deploy_key}`, Accept: 'application/json', - ...(options.extraHeaders ?? {}) + ...extraHeaders }, - body: options.body == null ? undefined : JSON.stringify(options.body), + body: body == null ? undefined : JSON.stringify(options.body), signal }); const text = await response.text(); - const json = text == '' ? {} : safeParseJson(text); + let json: unknown; + try { + json = JSONBig.parse(text); + } catch (ex) { + // The response could not be json, this should only happen for !ok responses. + } if (!response.ok) { const retryable = response.status == 429 || response.status >= 500; @@ -317,154 +290,6 @@ export function isCursorExpiredError(error: unknown): boolean { ); } -function parseHasMore(payload: Record): boolean { - return Boolean(payload.has_more ?? payload.hasMore ?? false); -} - -function parseValues(payload: Record): ConvexRawDocument[] { - const values = payload.values ?? payload.page ?? []; - if (!Array.isArray(values)) { - return []; - } - - return values.filter((value) => isRecord(value)) as ConvexRawDocument[]; -} - -function extractTableSchemas(payload: Record): ConvexTableSchema[] { - const resolved = new Map(); - - const tableList = payload.tables; - if (Array.isArray(tableList)) { - for (const table of tableList) { - if (!isRecord(table)) { - continue; - } - const tableName = table.tableName ?? table.table_name ?? table.name; - if (typeof tableName != 'string' || tableName.length == 0) { - continue; - } - resolved.set(tableName, { - tableName, - schema: isRecord(table.schema) ? table.schema : {} - }); - } - } else if (isRecord(tableList)) { - for (const [tableName, schema] of Object.entries(tableList)) { - resolved.set(tableName, { - tableName, - schema: isRecord(schema) ? schema : {} - }); - } - } - - const schemaMap = payload.schema; - if (isRecord(schemaMap)) { - for (const [tableName, schema] of Object.entries(schemaMap)) { - if (!resolved.has(tableName)) { - resolved.set(tableName, { - tableName, - schema: isRecord(schema) ? schema : {} - }); - } - } - } - - // Self-hosted Convex may return table schemas as the top-level object: - // { "table_a": { ...json schema... }, "table_b": { ...json schema... } }. - for (const [tableName, schema] of Object.entries(payload)) { - if (resolved.has(tableName) || RESERVED_SCHEMA_KEYS.has(tableName)) { - continue; - } - - if (!looksLikeJsonSchema(schema)) { - continue; - } - - resolved.set(tableName, { - tableName, - schema: isRecord(schema) ? schema : {} - }); - } - - return [...resolved.values()].sort((a, b) => a.tableName.localeCompare(b.tableName)); -} - -function stringifyCursor(value: unknown): string { - if (typeof value == 'string') { - if (value.length == 0) { - throw new ConvexApiError({ - message: 'Convex cursor cannot be empty', - retryable: false, - body: value - }); - } - return value; - } - - if (typeof value == 'number' || typeof value == 'bigint') { - return `${value}`; - } - - throw new ConvexApiError({ - message: `Convex cursor is missing or invalid: ${JSON.stringify(value)}`, - retryable: false, - body: value - }); -} - -function parseSnapshot(payload: Record, requestedSnapshot?: string): string { - const responseSnapshot = payload.snapshot ?? payload.snapshot_ts ?? payload.snapshotTs; - if (responseSnapshot != null) { - return stringifyCursor(responseSnapshot); - } - - if (requestedSnapshot != null && requestedSnapshot.length > 0) { - return requestedSnapshot; - } - - throw new ConvexApiError({ - message: 'Convex list_snapshot response is missing snapshot', - retryable: false, - body: payload - }); -} - -function safeParseJson(value: string): unknown { - try { - return JSONBig.parse(value); - } catch { - return { raw: value }; - } -} - function isRecord(value: unknown): value is Record { return typeof value == 'object' && value != null && !Array.isArray(value); } - -const RESERVED_SCHEMA_KEYS = new Set([ - 'tables', - 'schema', - 'snapshot', - 'cursor', - 'values', - 'value', - 'page', - 'hasMore', - 'has_more', - 'error', - 'errors' -]); - -function looksLikeJsonSchema(value: unknown): boolean { - if (!isRecord(value)) { - return false; - } - - return ( - typeof value.type == 'string' || - typeof value.$schema == 'string' || - Array.isArray(value.required) || - isRecord(value.properties) || - isRecord(value.schema) - ); -} diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index 2f8e7b286..f0549ad89 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -15,8 +15,8 @@ export function toConvexLsn(cursor: string | bigint): string { /** * Converts the decimal timestamp LSN to a JS Date. */ -export function lsnToDate(cursor: string): Date { - return new Date(Number(BigInt(cursor) / 1_000_000n)); +export function lsnToDate(cursor: bigint): Date { + return new Date(Number(cursor / 1_000_000n)); } function assertValidConvexLsn(cursor: string) { diff --git a/modules/module-convex/src/common/convex-to-sqlite.ts b/modules/module-convex/src/common/convex-to-sqlite.ts index 2acfddd3e..fd3e384d5 100644 --- a/modules/module-convex/src/common/convex-to-sqlite.ts +++ b/modules/module-convex/src/common/convex-to-sqlite.ts @@ -6,7 +6,7 @@ import { SqliteInputRow, toSyncRulesRow } from '@powersync/service-sync-rules'; -import { ConvexRawDocument } from '../client/ConvexApiClient.js'; +import { ConvexRawDocument } from '../client/ConvexAPITypes.js'; /** * From Convex docs: diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index a33f88c9d..9ff37fdd7 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -19,12 +19,8 @@ import { import { HydratedSyncRules, TablePattern } from '@powersync/service-sync-rules'; import { ReplicationMetric } from '@powersync/service-types'; import { setTimeout as delay } from 'timers/promises'; -import { - ConvexListSnapshotResult, - ConvexRawDocument, - ConvexTableSchema, - isCursorExpiredError -} from '../client/ConvexApiClient.js'; +import { ConvexListSnapshotResult, ConvexRawDocument, ConvexTableSchema } from '../client/ConvexAPITypes.js'; +import { isCursorExpiredError } from '../client/ConvexApiClient.js'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { lsnToDate, parseConvexLsn, toConvexLsn, ZERO_LSN } from '../common/ConvexLSN.js'; import { extractProperties, toSqliteInputRow } from '../common/convex-to-sqlite.js'; @@ -155,13 +151,13 @@ export class ConvexStream { // Resolve source tables up-front to warm table metadata and sync-rule matching. await this.resolveAllSourceTables(batch); - let cursor = parseConvexLsn(resumeFromLsn); + let cursor = BigInt(resumeFromLsn); let lastTransactionTimestamp: bigint | null = null; while (!this.abortSignal.aborted) { const page = await this.connections.client .documentDeltas({ - cursor, + cursor: cursor.toString(), signal: this.abortSignal }) .catch((error) => { @@ -210,7 +206,7 @@ export class ConvexStream { } lastTransactionTimestamp = transactionTimestamp; - const table = await this.getOrResolveTable(batch, tableName, nextCursor, snapshottedTablesInPage); + const table = await this.getOrResolveTable(batch, tableName, nextCursor.toString(), snapshottedTablesInPage); if (table == null || !table.syncAny) { continue; } @@ -401,7 +397,7 @@ export class ConvexStream { throw error; }); - if (snapshotCursor != page.snapshot) { + if (snapshotCursor != page.snapshot.toString()) { throw new ReplicationAssertionError( `Convex snapshot cursor changed while snapshotting ${table.qualifiedName}: ${snapshotCursor} -> ${page.snapshot}` ); diff --git a/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts b/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts new file mode 100644 index 000000000..c43319b5c --- /dev/null +++ b/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts @@ -0,0 +1,153 @@ +import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; +import { normalizeConnectionConfig } from '@module/types/types.js'; +import type { DatabaseSchema } from '@powersync/service-types'; +import { randomUUID } from 'crypto'; +import { describe, expect, test } from 'vitest'; +import { env } from '../env.js'; +import { ConvexStreamTestContext } from '../test-utils/ConvexStreamTestContext.js'; +import { INITIALIZED_MONGO_STORAGE_FACTORY } from '../test-utils/util.js'; + +const TEST_CONNECTION_OPTIONS = normalizeConnectionConfig({ + type: 'convex', + deploy_key: env.CONVEX_DEPLOY_KEY, + deployment_url: env.CONVEX_URL +}); + +function normalizeSchemaForSnapshot(schema: DatabaseSchema[]): DatabaseSchema[] { + const snapshottedTables = new Set(['lists', 'todos']); + + return schema.map((database) => ({ + ...database, + tables: [...database.tables] + .filter((table) => snapshottedTables.has(table.name)) + .sort((a, b) => a.name.localeCompare(b.name)) + .map((table) => ({ + columns: [...table.columns] + .sort((a, b) => a.name.localeCompare(b.name)) + .map((column) => ({ + internal_type: column.internal_type, + name: column.name, + pg_type: column.pg_type, + sqlite_type: column.sqlite_type, + type: column.type + })), + name: table.name + })) + })); +} + +describe.skipIf(!env.CONVEX_DEPLOY_KEY)('ConvexStream ConvexRouteAPIAdapter tests', function () { + test('retrieves the testing Convex schema in the expected service schema format', async () => { + /** + * It seems like Convex requires the table to contain populated columns in order to report + * valuable column information. + */ + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY.factory, {}); + // Create an item + await context.backend.client.mutation(context.backend.api.lists.createBatch, { + lists: [ + { + name: 'a string name', + uuid: randomUUID(), + archived: 1, + attributes: { color: 'red' }, + created_at: new Date().toISOString(), + owner: 'an owner', + owner_id: randomUUID(), + settings: { + color: 'red', + is_public: true, + theme: 'theme' + }, + tags: ['one', 'two'] + } + ] + }); + await using adapter = new ConvexRouteAPIAdapter(TEST_CONNECTION_OPTIONS); + const schema = await adapter.getConnectionSchema(); + expect(schema).toMatchObject( + expect.arrayContaining([ + { + name: 'convex', + tables: expect.arrayContaining([ + { + name: 'lists', + columns: expect.arrayContaining([ + { + internal_type: 'id', + name: '_id', + pg_type: 'id', + sqlite_type: 2, + type: 'id' + }, + { + internal_type: 'float64', + name: 'archived', + pg_type: 'float64', + sqlite_type: 8, + type: 'float64' + }, + { + internal_type: 'object', + name: 'attributes', + pg_type: 'object', + sqlite_type: 2, + type: 'object' + }, + { + internal_type: 'string', + name: 'created_at', + pg_type: 'string', + sqlite_type: 2, + type: 'string' + }, + { + internal_type: 'string', + name: 'name', + pg_type: 'string', + sqlite_type: 2, + type: 'string' + }, + { + internal_type: 'string', + name: 'owner', + pg_type: 'string', + sqlite_type: 2, + type: 'string' + }, + { + internal_type: 'string', + name: 'owner_id', + pg_type: 'string', + sqlite_type: 2, + type: 'string' + }, + { + internal_type: 'object', + name: 'settings', + pg_type: 'object', + sqlite_type: 2, + type: 'object' + }, + { + internal_type: 'array', + name: 'tags', + pg_type: 'array', + sqlite_type: 2, + type: 'array' + }, + { + internal_type: 'string', + name: 'uuid', + pg_type: 'string', + sqlite_type: 2, + type: 'string' + } + ]) + } + ]) + } + ]) + ); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 109a2bad9..697d525f0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -213,6 +213,9 @@ importers: '@powersync/service-types': specifier: workspace:* version: link:../../packages/types + ajv: + specifier: ^8.18.0 + version: 8.18.0 bson: specifier: ^6.10.4 version: 6.10.4 @@ -857,7 +860,7 @@ importers: version: 3.1.14 ts-node: specifier: ^10.9.2 - version: 10.9.2(@types/node@22.16.2)(typescript@6.0.3) + version: 10.9.2(@types/node@25.5.0)(typescript@6.0.3) test-client: dependencies: @@ -7490,14 +7493,14 @@ snapshots: ts-codec@1.3.0: {} - ts-node@10.9.2(@types/node@22.16.2)(typescript@6.0.3): + ts-node@10.9.2(@types/node@25.5.0)(typescript@6.0.3): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.12 '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 22.16.2 + '@types/node': 25.5.0 acorn: 8.15.0 acorn-walk: 8.3.5 arg: 4.1.3 From 071406ba549a3d14e8d36e535d9d3e73b0c2d52e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 12 May 2026 18:11:02 +0200 Subject: [PATCH 63/86] fix mocked tests --- modules/module-convex/package.json | 2 +- .../src/client/ConvexAPITypes.ts | 11 ++ .../src/client/ConvexApiClient.ts | 12 +- .../test/src/ConvexApiClient.test.ts | 113 ++---------------- .../test/src/ConvexStream.test.ts | 42 ++++--- .../resuming_snapshots.integration.test.ts | 6 + pnpm-lock.yaml | 16 +-- 7 files changed, 60 insertions(+), 142 deletions(-) diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json index 738ccccf3..85d4a35fc 100644 --- a/modules/module-convex/package.json +++ b/modules/module-convex/package.json @@ -44,7 +44,7 @@ "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-postgres-storage": "workspace:*", "@convex-dev/auth": "^0.0.92", - "convex": "^1.37.0", + "convex": "^1.38.0", "dotenv": "^16.4.5" } } diff --git a/modules/module-convex/src/client/ConvexAPITypes.ts b/modules/module-convex/src/client/ConvexAPITypes.ts index a2b5dab4b..ef3b21456 100644 --- a/modules/module-convex/src/client/ConvexAPITypes.ts +++ b/modules/module-convex/src/client/ConvexAPITypes.ts @@ -83,6 +83,17 @@ export interface ConvexDocumentDeltasOptions { signal?: AbortSignal; } +export const RawJsonSchemaResponse = t.record( + t.object({ + type: t.string, + properties: t.record(t.any) + }) +); + +export type RawJsonSchemaResponse = t.Encoded; + +export const ensureRawJsonSchemaResponse = ensureResponseFormatValidator(RawJsonSchemaResponse); + /** * Performs a validation which ensures the input data matches the codec specification. */ diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index 88a01f01c..03ae0587b 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -1,6 +1,5 @@ import { JSONBig } from '@powersync/service-jsonbig'; import { setTimeout as delay } from 'timers/promises'; -import * as t from 'ts-codec'; import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; import { NormalizedConvexConnectionConfig } from '../types/types.js'; import { @@ -11,20 +10,11 @@ import { ConvexListSnapshotResult, ensureConvexDocumentDeltasResult, ensureConvexListSnapshotResult, - ensureResponseFormatValidator + ensureRawJsonSchemaResponse } from './ConvexAPITypes.js'; const CONVEX_REQUEST_TIMEOUT_MS = 60_000; -const RawJsonSchemaResponse = t.record( - t.object({ - type: t.string, - properties: t.record(t.any) - }) -); - -const ensureRawJsonSchemaResponse = ensureResponseFormatValidator(RawJsonSchemaResponse); - type GetRequestParams = { path: string; params?: Record; diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index b68dc89c1..c495a8f12 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -1,6 +1,8 @@ import { ConvexApiClient } from '@module/client/ConvexApiClient.js'; +import { ConvexListSnapshotResult, RawJsonSchemaResponse } from '@module/client/ConvexAPITypes.js'; import { CONVEX_CHECKPOINT_TABLE } from '@module/common/ConvexCheckpoints.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; +import { JSONBig } from '@powersync/service-jsonbig'; import { afterEach, describe, expect, it, vi } from 'vitest'; const baseConfig = normalizeConnectionConfig({ @@ -8,7 +10,7 @@ const baseConfig = normalizeConnectionConfig({ deployment_url: 'https://example.convex.cloud', deploy_key: 'test-key' }); -const SNAPSHOT_CURSOR = '1770335566197683000'; +const SNAPSHOT_CURSOR = 1770335566197683000n; describe('ConvexApiClient', () => { afterEach(() => { @@ -19,10 +21,8 @@ describe('ConvexApiClient', () => { const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( new Response( JSON.stringify({ - tables: { - users: { properties: { _id: { type: 'string' } } } - } - }), + users: { type: 'table', properties: { _id: { type: 'string' } } } + } satisfies RawJsonSchemaResponse), { status: 200 } ) ); @@ -38,71 +38,6 @@ describe('ConvexApiClient', () => { expect((init?.headers as Record).Authorization).toBe('Convex test-key'); }); - it('falls back to /api/streaming_export path on 404', async () => { - const fetchSpy = vi - .spyOn(globalThis, 'fetch') - .mockResolvedValueOnce(new Response('{}', { status: 404 })) - .mockResolvedValueOnce( - new Response( - JSON.stringify({ - snapshot: SNAPSHOT_CURSOR, - cursor: 'next-page', - has_more: false, - values: [] - }), - { status: 200 } - ) - ); - - const client = new ConvexApiClient(baseConfig); - const page = await client.listSnapshot({ tableName: 'users' }); - - expect(page.snapshot).toBe(SNAPSHOT_CURSOR); - expect(fetchSpy.mock.calls.length).toBe(2); - expect(String(fetchSpy.mock.calls[1]![0])).toContain('/api/streaming_export/list_snapshot'); - }); - - it('reuses requested snapshot when response omits snapshot', async () => { - vi.spyOn(globalThis, 'fetch').mockResolvedValue( - new Response( - JSON.stringify({ - cursor: 'next-page', - has_more: true, - values: [] - }), - { status: 200 } - ) - ); - - const client = new ConvexApiClient(baseConfig); - const page = await client.listSnapshot({ - snapshot: SNAPSHOT_CURSOR, - tableName: 'lists' - }); - - expect(page.snapshot).toBe(SNAPSHOT_CURSOR); - expect(page.cursor).toBe('next-page'); - }); - - it('fails when first list_snapshot page omits snapshot', async () => { - vi.spyOn(globalThis, 'fetch').mockResolvedValue( - new Response( - JSON.stringify({ - cursor: 'next-page', - has_more: true, - values: [] - }), - { status: 200 } - ) - ); - - const client = new ConvexApiClient(baseConfig); - await expect(client.listSnapshot({ tableName: 'lists' })).rejects.toMatchObject({ - message: expect.stringContaining('missing snapshot'), - retryable: false - }); - }); - it('preserves high-precision numeric snapshot values', async () => { vi.spyOn(globalThis, 'fetch').mockResolvedValue( new Response( @@ -114,54 +49,26 @@ describe('ConvexApiClient', () => { const client = new ConvexApiClient(baseConfig); const page = await client.listSnapshot({ tableName: 'lists' }); - expect(page.snapshot).toBe('1770335566197682922'); + expect(page.snapshot).toBe(1770335566197682922n); expect(page.cursor).toContain('"tablet":"X0yj4Cm7GfuikfsSBm9QCQ"'); expect(page.hasMore).toBe(true); }); - it('parses self-hosted top-level json_schemas table map', async () => { - vi.spyOn(globalThis, 'fetch').mockResolvedValue( - new Response( - JSON.stringify({ - todos: { - type: 'object', - properties: { - _id: { type: 'string' } - } - }, - lists: { - type: 'object', - properties: { - _id: { type: 'string' }, - name: { type: 'string' } - } - } - }), - { status: 200 } - ) - ); - - const client = new ConvexApiClient(baseConfig); - const result = await client.getJsonSchemas(); - - expect(result.tables.map((table) => table.tableName)).toEqual(['lists', 'todos']); - }); - it('sends table_name as snake_case query parameter in list_snapshot', async () => { const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( new Response( - JSON.stringify({ + JSONBig.stringify({ snapshot: SNAPSHOT_CURSOR, cursor: null, - has_more: false, + hasMore: false, values: [] - }), + } satisfies ConvexListSnapshotResult), { status: 200 } ) ); const client = new ConvexApiClient(baseConfig); - await client.listSnapshot({ tableName: 'lists', snapshot: SNAPSHOT_CURSOR }); + await client.listSnapshot({ tableName: 'lists', snapshot: SNAPSHOT_CURSOR.toString() }); const url = String(fetchSpy.mock.calls[0]![0]); expect(url).toContain('table_name=lists'); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index c4ab26841..3e9f96229 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -1,3 +1,4 @@ +import { ConvexDocumentDeltasResult, ConvexJsonSchemasResult } from '@module/client/ConvexAPITypes.js'; import { toConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; import { BinaryConvexSnapshotProgressCursor } from '@module/replication/ConvexSnapshotProgresCursor.js'; import { ConvexStream } from '@module/replication/ConvexStream.js'; @@ -6,12 +7,12 @@ import { TablePattern } from '@powersync/service-sync-rules'; import { ReplicationMetric } from '@powersync/service-types'; import { describe, expect, it, vi } from 'vitest'; -const CURSOR_100 = '1772817606884944100'; -const CURSOR_101 = '1772817606884944101'; -const CURSOR_102 = '1772817606884944102'; -const CURSOR_200 = '1772817606884944200'; -const CURSOR_300 = '1772817606884944300'; -const CURSOR_301 = '1772817606884944301'; +const CURSOR_100 = 1772817606884944100n; +const CURSOR_101 = 1772817606884944101n; +const CURSOR_102 = 1772817606884944102n; +const CURSOR_200 = 1772817606884944200n; +const CURSOR_300 = 1772817606884944300n; +const CURSOR_301 = 1772817606884944301n; function createFakeStorage(options?: { snapshotDone?: boolean; @@ -459,18 +460,21 @@ describe('ConvexStream', () => { connectionId: '1', config: { pollingIntervalMs: 1 }, client: { - getJsonSchemas: async () => ({ - tables: [{ tableName: 'users', schema: {} }], - raw: {} - }), - documentDeltas: async () => ({ - cursor: CURSOR_102, - hasMore: false, - values: [ - { _table: 'users', _id: 'u1', _ts: BigInt(CURSOR_102), name: 'Later' }, - { _table: 'users', _id: 'u2', _ts: BigInt(CURSOR_101), name: 'Earlier' } - ] - }) + getJsonSchemas: async () => { + return { + tables: [{ tableName: 'users', schema: {} }] + } satisfies ConvexJsonSchemasResult; + }, + documentDeltas: async () => { + return { + cursor: CURSOR_102, + hasMore: false, + values: [ + { _table: 'users', _id: 'u1', _ts: BigInt(CURSOR_102), name: 'Later' }, + { _table: 'users', _id: 'u2', _ts: BigInt(CURSOR_101), name: 'Earlier' } + ] + } satisfies ConvexDocumentDeltasResult; + } } } as any }); @@ -532,7 +536,7 @@ describe('ConvexStream', () => { expect(listSnapshot).toHaveBeenCalledTimes(1); expect(listSnapshot).toHaveBeenCalledWith({ tableName: 'projects_archive', - snapshot: CURSOR_101, + snapshot: CURSOR_101.toString(), cursor: undefined, signal: abortController.signal }); diff --git a/modules/module-convex/test/src/integration/resuming_snapshots.integration.test.ts b/modules/module-convex/test/src/integration/resuming_snapshots.integration.test.ts index 6340ef7cb..e014502e1 100644 --- a/modules/module-convex/test/src/integration/resuming_snapshots.integration.test.ts +++ b/modules/module-convex/test/src/integration/resuming_snapshots.integration.test.ts @@ -54,6 +54,8 @@ async function testResumingReplication( name: `list-${index}` })) }); + // Delay to avoid TooManyWrites error from Convex + await new Promise((r) => setTimeout(r, 1_000)); // create a single row to track deleted items const deletableListId = randomUUID(); @@ -82,6 +84,8 @@ async function testResumingReplication( description: `todo-${index}` })) }); + // Delay to avoid TooManyWrites error from Convex + await new Promise((r) => setTimeout(r, 1_000)); // twice in order to get many todos (see limits above) await backend.client.mutation(backend.api.todos.createBatch, { todos: Array.from({ length: 2_000 }).map((_, index) => ({ @@ -90,6 +94,8 @@ async function testResumingReplication( description: `todo-${index}` })) }); + // Delay to avoid TooManyWrites error from Convex + await new Promise((r) => setTimeout(r, 1_000)); let stopped = new Promise((resolve) => { context.storage!.registerListener({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 697d525f0..6d8eecd1d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -225,7 +225,7 @@ importers: devDependencies: '@convex-dev/auth': specifier: ^0.0.92 - version: 0.0.92(@auth/core@0.37.4)(convex@1.37.0) + version: 0.0.92(@auth/core@0.37.4)(convex@1.38.0) '@powersync/service-core-tests': specifier: workspace:* version: link:../../packages/service-core-tests @@ -236,8 +236,8 @@ importers: specifier: workspace:* version: link:../module-postgres-storage convex: - specifier: ^1.37.0 - version: 1.37.0 + specifier: ^1.38.0 + version: 1.38.0 dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -2470,8 +2470,8 @@ packages: convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} - convex@1.37.0: - resolution: {integrity: sha512-xGSx5edIsXCEex3OU2U2N0oyB/cOa9qGwKiImF9yOWqjqZgOkx39idtpdlwNBTBSt4S30oAvs4yeXY5xxPIX3A==} + convex@1.38.0: + resolution: {integrity: sha512-122AC6y5lUS7mr39cluLw9+TOtRX5d/XxeivHhHObs/NTXoVvOnIgDzexVcxaz6Rk0oLFSoydSR1rDCltEz/0A==} engines: {node: '>=18.0.0', npm: '>=7.0.0'} hasBin: true peerDependencies: @@ -4614,12 +4614,12 @@ snapshots: '@colors/colors@1.6.0': {} - '@convex-dev/auth@0.0.92(@auth/core@0.37.4)(convex@1.37.0)': + '@convex-dev/auth@0.0.92(@auth/core@0.37.4)(convex@1.38.0)': dependencies: '@auth/core': 0.37.4 '@oslojs/crypto': 1.0.1 '@oslojs/encoding': 1.1.0 - convex: 1.37.0 + convex: 1.38.0 cookie: 1.0.2 is-network-error: 1.3.1 jose: 5.10.0 @@ -6052,7 +6052,7 @@ snapshots: convert-source-map@2.0.0: {} - convex@1.37.0: + convex@1.38.0: dependencies: esbuild: 0.27.0 prettier: 3.4.2 From b719e4e24ca0cc0de643aa379f380556e8d72dd4 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 12 May 2026 18:24:28 +0200 Subject: [PATCH 64/86] cleanup --- .../test/src/ConvexRouteAPIAdapter.test.ts | 28 ++++++++++--------- .../test-utils/AbstractStreamTestContext.ts | 1 - 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index 894a60bbf..2d0296b41 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -1,4 +1,5 @@ import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; +import { ConvexJsonSchemasResult } from '@module/client/ConvexAPITypes.js'; import { toConvexLsn } from '@module/common/ConvexLSN.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; import { ExpressionType, SqlSyncRules } from '@powersync/service-sync-rules'; @@ -21,21 +22,22 @@ function createAdapter() { }); (adapter as any).connectionManager.client = { - getJsonSchemas: async () => ({ - tables: [ - { - tableName: 'users', - schema: { - properties: { - _id: { type: 'string' }, - age: { type: 'integer' }, - avatar: { type: 'bytes' } + getJsonSchemas: async () => { + return { + tables: [ + { + tableName: 'users', + schema: { + properties: { + _id: { type: 'string' }, + age: { type: 'integer' }, + avatar: { type: 'bytes' } + } } } - } - ], - raw: {} - }), + ] + } satisfies ConvexJsonSchemasResult; + }, getHeadCursor: async () => HEAD_CURSOR, createWriteCheckpointMarker: async () => undefined }; diff --git a/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts b/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts index 477ed89a9..33265f124 100644 --- a/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts +++ b/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts @@ -12,7 +12,6 @@ import { StorageDataHelpers } from './StorageDataHelpers.js'; import { bucketRequest } from './general-utils.js'; import { fromAsync } from './stream_utils.js'; -// TODO: Might want to share this with other replication module tests export abstract class AbstractStreamTestContext implements AsyncDisposable { protected abortController = new AbortController(); protected syncRulesContent?: storage.PersistedSyncRulesContent; From 1d8e9a06f4cde53a3d7b52042ef5be4235941957 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 13 May 2026 11:03:49 +0200 Subject: [PATCH 65/86] cleanup schema parsing --- .../src/api/ConvexRouteAPIAdapter.ts | 8 +- .../src/client/ConvexAPITypes.ts | 11 +- modules/module-convex/src/common/ConvexLSN.ts | 28 +- .../src/common/convex-to-sqlite.ts | 247 ++++++++++-------- .../src/replication/ConvexStream.ts | 10 +- .../module-convex/test/src/ConvexLSN.test.ts | 12 +- .../test/src/ConvexRouteAPIAdapter.test.ts | 5 +- .../test/src/ConvexStream.test.ts | 39 +-- .../module-convex/test/src/test-utils/util.ts | 30 ++- 9 files changed, 230 insertions(+), 160 deletions(-) diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index e69507849..e1f4e9696 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -8,7 +8,7 @@ import { import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; -import { toConvexLsn } from '../common/ConvexLSN.js'; +import { parseConvexLsn } from '../common/ConvexLSN.js'; import { extractProperties, readConvexFieldType, toExpressionTypeFromConvexType } from '../common/convex-to-sqlite.js'; import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; import * as types from '../types/types.js'; @@ -128,7 +128,7 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { async createReplicationHead(callback: ReplicationHeadCallback): Promise { const head = await this.connectionManager.client.getHeadCursor(); - const result = await callback(toConvexLsn(head)); + const result = await callback(parseConvexLsn(head)); await this.connectionManager.client.createWriteCheckpointMarker(); return result; } @@ -148,8 +148,8 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { _id: { type: 'id' } }) .sort(([a], [b]) => a.localeCompare(b)) - .map(([columnName, property]) => { - const jsonType = readConvexFieldType(property); + .map(([columnName, jsonSchemaProperty]) => { + const jsonType = readConvexFieldType(jsonSchemaProperty); const sqliteType = toExpressionTypeFromConvexType(jsonType); return { diff --git a/modules/module-convex/src/client/ConvexAPITypes.ts b/modules/module-convex/src/client/ConvexAPITypes.ts index ef3b21456..bbc796b00 100644 --- a/modules/module-convex/src/client/ConvexAPITypes.ts +++ b/modules/module-convex/src/client/ConvexAPITypes.ts @@ -67,8 +67,6 @@ export const ConvexDocumentDeltasResult = t.object({ export type ConvexDocumentDeltasResult = t.Encoded; -// These validators help assert the API structure response. -// These could be disabled for production. export const ensureConvexDocumentDeltasResult = ensureResponseFormatValidator(ConvexDocumentDeltasResult); export interface ConvexListSnapshotOptions { @@ -95,7 +93,14 @@ export type RawJsonSchemaResponse = t.Encoded; export const ensureRawJsonSchemaResponse = ensureResponseFormatValidator(RawJsonSchemaResponse); /** - * Performs a validation which ensures the input data matches the codec specification. + * Performs a validation which ensures the Convex API response data matches the codec specification. + * This was added after noticing the original implementation was coercing various permutations of + * response fields e.g. `has_more` and `hasMore` in responses. There were comments that the + * self hosted and cloud Convex implementations might have returned different responses. + * In testing, with these validations, I could not see any actual discrepency in responses. + * Having these checks could help spot potential changes to the API - however they do come at a cost. + * We could disable this in prod builds or remove in the future. For now, while the API seems fickle, it could + * be nice to have a safety net. */ export function ensureResponseFormatValidator( codec: Codec diff --git a/modules/module-convex/src/common/ConvexLSN.ts b/modules/module-convex/src/common/ConvexLSN.ts index f0549ad89..cd7810ef7 100644 --- a/modules/module-convex/src/common/ConvexLSN.ts +++ b/modules/module-convex/src/common/ConvexLSN.ts @@ -1,24 +1,30 @@ -export const ZERO_LSN = '0'; // Convex replication cursors are fixed-width decimal timestamps. const CONVEX_TIMESTAMP_DIGITS = 19; -export function parseConvexLsn(lsn: string): string { - return toConvexLsn(lsn); -} - -export function toConvexLsn(cursor: string | bigint): string { - const asString = `${cursor}`; - assertValidConvexLsn(asString); - return asString; -} +export const ZERO_LSN = '0'; /** * Converts the decimal timestamp LSN to a JS Date. */ -export function lsnToDate(cursor: bigint): Date { +export function lsnCursorToDate(cursor: bigint): Date { return new Date(Number(cursor / 1_000_000n)); } +/** + * Parse and validate an incoming LSN. + * Sources for input are usually the `list-snapshot` response - which returns a bigint + * or the current stored LSN - which is stored as a string. + */ +export function parseConvexLsn(input: string | bigint): string { + return toConvexLsn(input); +} + +export function toConvexLsn(input: string | bigint): string { + const asString = `${input}`; + assertValidConvexLsn(asString); + return asString; +} + function assertValidConvexLsn(cursor: string) { if (cursor.length == 0) { throw new Error('Convex cursor cannot be empty'); diff --git a/modules/module-convex/src/common/convex-to-sqlite.ts b/modules/module-convex/src/common/convex-to-sqlite.ts index fd3e384d5..8466794e6 100644 --- a/modules/module-convex/src/common/convex-to-sqlite.ts +++ b/modules/module-convex/src/common/convex-to-sqlite.ts @@ -8,6 +8,34 @@ import { } from '@powersync/service-sync-rules'; import { ConvexRawDocument } from '../client/ConvexAPITypes.js'; +export enum SupportedJSONSchemaPropertyType { + ID = 'id', + STRING = 'string', + BYTES = 'bytes', + ARRAY = 'array', + OBJECT = 'object', + RECORD = 'record', + NULL = 'null', + INT64 = 'int64', + FLOAT64 = 'float64', + BOOLEAN = 'boolean', + UNKNOWN = 'unknown' +} + +export const CONVEX_TO_SQLITE_TYPE_MAP: Record = { + [SupportedJSONSchemaPropertyType.ID]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.STRING]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.BYTES]: ExpressionType.BLOB, + [SupportedJSONSchemaPropertyType.ARRAY]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.OBJECT]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.RECORD]: ExpressionType.TEXT, + [SupportedJSONSchemaPropertyType.NULL]: ExpressionType.NONE, + [SupportedJSONSchemaPropertyType.INT64]: ExpressionType.INTEGER, + [SupportedJSONSchemaPropertyType.FLOAT64]: ExpressionType.REAL, + [SupportedJSONSchemaPropertyType.BOOLEAN]: ExpressionType.INTEGER, + [SupportedJSONSchemaPropertyType.UNKNOWN]: ExpressionType.TEXT +} as const; + /** * From Convex docs: * Every document in Convex automatically has two system fields: @@ -19,7 +47,10 @@ import { ConvexRawDocument } from '../client/ConvexAPITypes.js'; */ const INTERNAL_KEYS = new Set(['_table', '_deleted', '_ts', '_component', '_creationTime']); -export function toSqliteInputRow(change: ConvexRawDocument, properties?: Record): SqliteInputRow { +export function toSqliteInputRow( + change: ConvexRawDocument, + jsonSchemaProperties?: Record +): SqliteInputRow { const row: DatabaseInputRow = {}; for (const [key, value] of Object.entries(change)) { @@ -27,94 +58,73 @@ export function toSqliteInputRow(change: ConvexRawDocument, properties?: Record< continue; } - row[key] = toConvexDatabaseValue(value, readConvexFieldType(properties?.[key])); + row[key] = toDatabaseValue(value, readConvexFieldType(jsonSchemaProperties?.[key])); } return toSyncRulesRow(row); } -export function toExpressionTypeFromConvexType(type: string | undefined): ExpressionType { - switch (normalizeConvexType(type)) { +/** + * Normalizes the Convex Table columns JSON schema property type + * to a set of common supported JSON schema property types. + */ +function normalizeConvexJsonSchemaType(type: string | undefined): SupportedJSONSchemaPropertyType { + const normalized = type?.trim().toLowerCase(); + switch (normalized) { + case SupportedJSONSchemaPropertyType.ID: + case SupportedJSONSchemaPropertyType.STRING: + case SupportedJSONSchemaPropertyType.BYTES: + case SupportedJSONSchemaPropertyType.ARRAY: + case SupportedJSONSchemaPropertyType.OBJECT: + case SupportedJSONSchemaPropertyType.RECORD: + case SupportedJSONSchemaPropertyType.NULL: + return normalized; + case 'integer': case 'int64': - return ExpressionType.INTEGER; + return SupportedJSONSchemaPropertyType.INT64; + case 'number': + case 'float': case 'float64': - return ExpressionType.REAL; + return SupportedJSONSchemaPropertyType.FLOAT64; + case 'bool': case 'boolean': - return ExpressionType.INTEGER; - case 'bytes': - return ExpressionType.BLOB; - case 'null': - return ExpressionType.NONE; - case 'array': - case 'object': - case 'record': - return ExpressionType.TEXT; - case 'id': - case 'string': - case 'unknown': + return SupportedJSONSchemaPropertyType.BOOLEAN; + case 'bytea': + case 'blob': + return SupportedJSONSchemaPropertyType.BYTES; default: - return ExpressionType.TEXT; + return SupportedJSONSchemaPropertyType.UNKNOWN; } } -export function extractProperties(schema: Record) { - const direct = schema.properties; - if (isRecord(direct)) { - return direct; - } - - const nested = schema.schema?.properties; - if (isRecord(nested)) { - return nested; - } - - return {}; +/** + * Connverts a Convex JSON Schema property type to A SQLite ExpressionType + */ +export function toExpressionTypeFromConvexType(type: string | undefined): ExpressionType { + return CONVEX_TO_SQLITE_TYPE_MAP[normalizeConvexJsonSchemaType(type)]; } -export function readConvexFieldType(value: unknown): string { - if (!isRecord(value)) { - return 'unknown'; - } - - const format = typeof value.format == 'string' ? normalizeConvexType(value.format) : null; - if (format == 'bytes' || format == 'id') { - return format; - } - - const contentEncoding = typeof value.contentEncoding == 'string' ? value.contentEncoding.toLowerCase() : null; - if (contentEncoding == 'base64') { - return 'bytes'; - } - - const directType = typeof value.type == 'string' ? value.type : null; - if (directType != null) { - return normalizeConvexType(directType); - } - - if (Array.isArray(value.type)) { - const firstString = value.type.find((entry) => typeof entry == 'string'); - if (typeof firstString == 'string') { - return normalizeConvexType(firstString); - } - } - - const alternateType = - typeof value.valueType == 'string' - ? value.valueType - : typeof value.fieldType == 'string' - ? value.fieldType - : typeof value.kind == 'string' - ? value.kind - : null; - if (alternateType != null) { - return normalizeConvexType(alternateType); - } - - return 'unknown'; +export function extractProperties(schema: Record): Record { + // Convex returns each table schema as a standard JSON Schema object. + // For example: { "type": "object", "properties": { "name": { "type": "string" } } }. + // The table columns live under `properties`; top-level keys such as `type` + // describe the schema itself and must not be treated as columns. + return isRecord(schema.properties) ? schema.properties : {}; } -function toConvexDatabaseValue(value: unknown, type: string): DatabaseInputValue { - switch (normalizeConvexType(type)) { +/** + * Converts a Convex row value to a DatabaseInputValue. + * We typically receive the Schema for each table from Convex in the form of a JSON schema. + * Each column in a table has a JSON Schema property type. + * + * In some cases, we have observed that the Schema returned from Convex does not include column + * definitions unless if rows exist which have values defined for the column. + * We don't exclusively rely on the jsonSchemaType for this reason. Type mappings are performed using + * type checks in these scenarios. + */ +function toDatabaseValue(value: unknown, jsonSchemaType: string): DatabaseInputValue { + // Use the schema type if available + switch (normalizeConvexJsonSchemaType(jsonSchemaType)) { case 'bytes': return toBytesValue(value); case 'boolean': @@ -126,6 +136,8 @@ function toConvexDatabaseValue(value: unknown, type: string): DatabaseInputValue break; } + // The schema did not match at this point, we continue with runtime type checks + if (value == null) { return null; } @@ -168,6 +180,10 @@ function toConvexDatabaseValue(value: unknown, type: string): DatabaseInputValue return null; } +function isRecord(value: unknown): value is Record { + return typeof value == 'object' && value != null && !Array.isArray(value); +} + function toBytesValue(value: unknown): Uint8Array | null { if (value == null) { return null; @@ -202,35 +218,62 @@ function toBytesValue(value: unknown): Uint8Array | null { return new Uint8Array(buffer); } -function normalizeConvexType(type: string | undefined): string { - const normalized = type?.trim().toLowerCase(); - switch (normalized) { - case 'id': - case 'string': - case 'bytes': - case 'array': - case 'object': - case 'record': - case 'null': - return normalized; - case 'integer': - case 'int64': - return 'int64'; - case 'number': - case 'float': - case 'float64': - return 'float64'; - case 'bool': - case 'boolean': - return 'boolean'; - case 'bytea': - case 'blob': - return 'bytes'; - default: - return normalized ?? 'unknown'; +/** + * Reads the Convex field type from the Convex table column's JSON schema entry. + */ +export function readConvexFieldType(jsonSchemaProperty: unknown): string { + if (!isRecord(jsonSchemaProperty)) { + // Invalid schema property entry received + return 'unknown'; } -} -function isRecord(value: unknown): value is Record { - return typeof value == 'object' && value != null && !Array.isArray(value); + // Convex can expose logical string subtypes through JSON schema `format`. + // For example: { "type": "string", "format": "id" } or { "type": "string", "format": "bytes" }. + // Check this before `type`, otherwise these would be treated as plain strings. + const format = + typeof jsonSchemaProperty.format == 'string' ? normalizeConvexJsonSchemaType(jsonSchemaProperty.format) : null; + if (format == 'bytes' || format == 'id') { + return format; + } + + // Some schema exporters describe binary data as a base64-encoded string. + // For example: { "type": "string", "contentEncoding": "base64" }. + const contentEncoding = + typeof jsonSchemaProperty.contentEncoding == 'string' ? jsonSchemaProperty.contentEncoding.toLowerCase() : null; + if (contentEncoding == 'base64') { + return 'bytes'; + } + + // Standard JSON schema uses a direct `type`. + // For example: { "type": "integer" }, { "type": "number" }, or { "type": "boolean" }. + const directType = typeof jsonSchemaProperty.type == 'string' ? jsonSchemaProperty.type : null; + if (directType != null) { + return normalizeConvexJsonSchemaType(directType); + } + + // Nullable JSON schema fields can use a type array. + // For example: { "type": ["string", "null"] }. Use the first concrete string type. + if (Array.isArray(jsonSchemaProperty.type)) { + const firstString = jsonSchemaProperty.type.find((entry) => typeof entry == 'string'); + if (typeof firstString == 'string') { + return normalizeConvexJsonSchemaType(firstString); + } + } + + // Convex-generated or intermediate schema metadata can carry the same type + // under non-standard keys. For example: { "valueType": "record" }, + // { "fieldType": "bytes" }, or { "kind": "array" }. + const alternateType = + typeof jsonSchemaProperty.valueType == 'string' + ? jsonSchemaProperty.valueType + : typeof jsonSchemaProperty.fieldType == 'string' + ? jsonSchemaProperty.fieldType + : typeof jsonSchemaProperty.kind == 'string' + ? jsonSchemaProperty.kind + : null; + if (alternateType != null) { + return normalizeConvexJsonSchemaType(alternateType); + } + + return 'unknown'; } diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 9ff37fdd7..cdc4aa95c 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -22,7 +22,7 @@ import { setTimeout as delay } from 'timers/promises'; import { ConvexListSnapshotResult, ConvexRawDocument, ConvexTableSchema } from '../client/ConvexAPITypes.js'; import { isCursorExpiredError } from '../client/ConvexApiClient.js'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; -import { lsnToDate, parseConvexLsn, toConvexLsn, ZERO_LSN } from '../common/ConvexLSN.js'; +import { lsnCursorToDate, parseConvexLsn, ZERO_LSN } from '../common/ConvexLSN.js'; import { extractProperties, toSqliteInputRow } from '../common/convex-to-sqlite.js'; import { ConvexConnectionManager } from './ConvexConnectionManager.js'; import { BinaryConvexSnapshotProgressCursor, decodeSnapshotProgressCursor } from './ConvexSnapshotProgresCursor.js'; @@ -168,7 +168,7 @@ export class ConvexStream { }); const nextCursor = page.cursor; - const pageLsn = toConvexLsn(nextCursor); + const pageLsn = parseConvexLsn(nextCursor); let changesInPage = 0; const transactionTimestampsInPage = new Set(); @@ -223,7 +223,7 @@ export class ConvexStream { * mark the start after this point - which means we will have an uncommited change. */ if (!didMarkOldestUncommitedChange) { - this.replicationLag.trackUncommittedChange(lsnToDate(page.cursor)); + this.replicationLag.trackUncommittedChange(lsnCursorToDate(page.cursor)); didMarkOldestUncommitedChange = true; } @@ -318,7 +318,7 @@ export class ConvexStream { }); const snapshotCursor = await this.resolveSnapshotBoundary(snapshotLsn); - const snapshotLsnValue = toConvexLsn(snapshotCursor); + const snapshotLsnValue = parseConvexLsn(snapshotCursor); await batch.setResumeLsn(snapshotLsnValue); const sourceTables = await this.resolveAllSourceTables(batch); @@ -461,7 +461,7 @@ export class ConvexStream { table: SourceTable, snapshotCursor: string ): Promise { - const snapshotLsnValue = toConvexLsn(snapshotCursor); + const snapshotLsnValue = parseConvexLsn(snapshotCursor); const [doneTable] = await batch.markTableSnapshotDone([table], snapshotLsnValue); this.relationCache.update(doneTable); return doneTable; diff --git a/modules/module-convex/test/src/ConvexLSN.test.ts b/modules/module-convex/test/src/ConvexLSN.test.ts index 8bb553698..06d731512 100644 --- a/modules/module-convex/test/src/ConvexLSN.test.ts +++ b/modules/module-convex/test/src/ConvexLSN.test.ts @@ -1,9 +1,9 @@ -import { parseConvexLsn, toConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; +import { parseConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; import { describe, expect, it } from 'vitest'; describe('Convex cursor LSN helpers', () => { it('validates and round-trips the numeric cursor', () => { - const source = toConvexLsn('1772817606884944136'); + const source = parseConvexLsn('1772817606884944136'); const roundTrip = parseConvexLsn(source); expect(source).toBe('1772817606884944136'); @@ -11,8 +11,8 @@ describe('Convex cursor LSN helpers', () => { }); it('sorts lexicographically after validating fixed-width timestamps', () => { - const older = toConvexLsn('1772817606884944136'); - const newer = toConvexLsn('1772817606884944137'); + const older = parseConvexLsn('1772817606884944136'); + const newer = parseConvexLsn('1772817606884944137'); expect(older < newer).toBe(true); }); @@ -43,6 +43,8 @@ describe('Convex cursor LSN helpers', () => { }); it('rejects non-numeric cursors', () => { - expect(() => toConvexLsn('{"tablet":"abc","id":"xyz"}')).toThrow('Convex cursor is not a valid numeric timestamp'); + expect(() => parseConvexLsn('{"tablet":"abc","id":"xyz"}')).toThrow( + 'Convex cursor is not a valid numeric timestamp' + ); }); }); diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index 2d0296b41..56000e671 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -1,6 +1,6 @@ import { ConvexRouteAPIAdapter } from '@module/api/ConvexRouteAPIAdapter.js'; import { ConvexJsonSchemasResult } from '@module/client/ConvexAPITypes.js'; -import { toConvexLsn } from '@module/common/ConvexLSN.js'; +import { parseConvexLsn } from '@module/common/ConvexLSN.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; import { ExpressionType, SqlSyncRules } from '@powersync/service-sync-rules'; import { describe, expect, it, vi } from 'vitest'; @@ -28,6 +28,7 @@ function createAdapter() { { tableName: 'users', schema: { + type: 'object', properties: { _id: { type: 'string' }, age: { type: 'integer' }, @@ -89,7 +90,7 @@ bucket_definitions: (adapter as any).connectionManager.client.createWriteCheckpointMarker = createWriteCheckpointMarker; const result = await adapter.createReplicationHead(async (head) => head); - expect(result).toBe(toConvexLsn(HEAD_CURSOR)); + expect(result).toBe(parseConvexLsn(HEAD_CURSOR)); expect(getHeadCursor).toHaveBeenCalledTimes(1); expect(getHeadCursor).toHaveBeenCalledWith(); expect(createWriteCheckpointMarker).toHaveBeenCalledTimes(1); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 3e9f96229..dcbf6fc84 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -1,5 +1,5 @@ import { ConvexDocumentDeltasResult, ConvexJsonSchemasResult } from '@module/client/ConvexAPITypes.js'; -import { toConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; +import { parseConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; import { BinaryConvexSnapshotProgressCursor } from '@module/replication/ConvexSnapshotProgresCursor.js'; import { ConvexStream } from '@module/replication/ConvexStream.js'; import { SaveOperationTag, SourceTable } from '@powersync/service-core'; @@ -140,7 +140,7 @@ function createFakeStorage(options?: { return { active: true, snapshot_done: options?.snapshotDone ?? false, - checkpoint_lsn: options?.snapshotDone ? toConvexLsn(CURSOR_100) : null, + checkpoint_lsn: options?.snapshotDone ? parseConvexLsn(CURSOR_100) : null, snapshot_lsn: options?.snapshotLsn ?? null }; }, @@ -209,7 +209,7 @@ describe('ConvexStream', () => { config: { pollingIntervalMs: 1 }, client: { getJsonSchemas: async () => ({ - tables: [{ tableName: 'users', schema: {} }], + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], raw: {} }), listSnapshot, @@ -229,8 +229,8 @@ describe('ConvexStream', () => { expect(context.saves.length).toBe(1); expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); expect(context.resumeLsnUpdates.length).toBe(1); - expect(context.allSnapshotDoneLsns).toEqual([toConvexLsn(CURSOR_100)]); - expect(context.commits.at(-1)).toBe(toConvexLsn(CURSOR_100)); + expect(context.allSnapshotDoneLsns).toEqual([parseConvexLsn(CURSOR_100)]); + expect(context.commits.at(-1)).toBe(parseConvexLsn(CURSOR_100)); }); it('decodes bytes fields to Uint8Array during snapshot hydration', async () => { @@ -254,6 +254,7 @@ describe('ConvexStream', () => { { tableName: 'users', schema: { + type: 'object', properties: { avatar: { type: 'bytes' } } @@ -292,7 +293,7 @@ describe('ConvexStream', () => { it('marks snapshot done without re-reading rows when the final page was already flushed', async () => { const context = createFakeStorage({ - snapshotLsn: toConvexLsn(CURSOR_200), + snapshotLsn: parseConvexLsn(CURSOR_200), tableSnapshotStatus: { replicatedCount: 2, totalEstimatedCount: -1, @@ -323,7 +324,7 @@ describe('ConvexStream', () => { config: { polling_interval_ms: 1 }, client: { getJsonSchemas: async () => ({ - tables: [{ tableName: 'users', schema: {} }], + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], raw: {} }), listSnapshot, @@ -341,7 +342,7 @@ describe('ConvexStream', () => { it('fails when table snapshots return a different snapshot boundary', async () => { const context = createFakeStorage({ - snapshotLsn: toConvexLsn(CURSOR_300) + snapshotLsn: parseConvexLsn(CURSOR_300) }); const abortController = new AbortController(); @@ -358,7 +359,7 @@ describe('ConvexStream', () => { config: { pollingIntervalMs: 1 }, client: { getJsonSchemas: async () => ({ - tables: [{ tableName: 'users', schema: {} }], + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], raw: {} }), getGlobalSnapshotCursor: async () => 'should-not-be-called', @@ -378,7 +379,7 @@ describe('ConvexStream', () => { it('streams deltas and commits checkpoint', async () => { const context = createFakeStorage({ snapshotDone: true, - resumeFromLsn: toConvexLsn(CURSOR_100) + resumeFromLsn: parseConvexLsn(CURSOR_100) }); const abortController = new AbortController(); @@ -405,7 +406,7 @@ describe('ConvexStream', () => { config: { pollingIntervalMs: 1 }, client: { getJsonSchemas: async () => ({ - tables: [{ tableName: 'users', schema: {} }], + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], raw: {} }), documentDeltas: async (options: any) => { @@ -433,7 +434,7 @@ describe('ConvexStream', () => { expect(context.saves[0]?.tag).toBe(SaveOperationTag.UPDATE); expect(context.saves[1]?.tag).toBe(SaveOperationTag.DELETE); expect(context.saves[2]?.tag).toBe(SaveOperationTag.UPDATE); - expect(context.commits.at(-1)).toBe(toConvexLsn(CURSOR_102)); + expect(context.commits.at(-1)).toBe(parseConvexLsn(CURSOR_102)); expect(deltaCalls[0]?.tableName).toBeUndefined(); expect(transactionCounts).toEqual([2]); }); @@ -444,7 +445,7 @@ describe('ConvexStream', () => { // This test just verifies the assertion would catch an issue if it ever happened for some reason. const context = createFakeStorage({ snapshotDone: true, - resumeFromLsn: toConvexLsn(CURSOR_100) + resumeFromLsn: parseConvexLsn(CURSOR_100) }); const abortController = new AbortController(); @@ -462,7 +463,7 @@ describe('ConvexStream', () => { client: { getJsonSchemas: async () => { return { - tables: [{ tableName: 'users', schema: {} }] + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }] } satisfies ConvexJsonSchemasResult; }, documentDeltas: async () => { @@ -485,14 +486,14 @@ describe('ConvexStream', () => { it('refreshes metadata before snapshotting a newly discovered wildcard-matched table inline', async () => { const context = createFakeStorage({ snapshotDone: true, - resumeFromLsn: toConvexLsn(CURSOR_100), + resumeFromLsn: parseConvexLsn(CURSOR_100), sourcePatterns: [new TablePattern('convex', 'projects%')] }); const abortController = new AbortController(); let calls = 0; const getJsonSchemas = vi.fn(async () => ({ - tables: [{ tableName: 'users', schema: {} }], + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], raw: {} })); const listSnapshot = vi.fn(async (options: any) => ({ @@ -549,7 +550,7 @@ describe('ConvexStream', () => { it('keeps alive immediately when only checkpoint marker rows are streamed', async () => { const context = createFakeStorage({ snapshotDone: true, - resumeFromLsn: toConvexLsn(CURSOR_100) + resumeFromLsn: parseConvexLsn(CURSOR_100) }); const abortController = new AbortController(); let calls = 0; @@ -567,7 +568,7 @@ describe('ConvexStream', () => { config: { pollingIntervalMs: 1 }, client: { getJsonSchemas: async () => ({ - tables: [{ tableName: 'users', schema: {} }], + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], raw: {} }), documentDeltas: async () => { @@ -595,6 +596,6 @@ describe('ConvexStream', () => { expect(context.saves.length).toBe(0); expect(context.commits.length).toBe(0); - expect(context.keepalives).toEqual([toConvexLsn(CURSOR_101), toConvexLsn(CURSOR_102)]); + expect(context.keepalives).toEqual([parseConvexLsn(CURSOR_101), parseConvexLsn(CURSOR_102)]); }); }); diff --git a/modules/module-convex/test/src/test-utils/util.ts b/modules/module-convex/test/src/test-utils/util.ts index b99263df0..91e7785be 100644 --- a/modules/module-convex/test/src/test-utils/util.ts +++ b/modules/module-convex/test/src/test-utils/util.ts @@ -3,8 +3,6 @@ import { api } from '@testing-convex/_generated/api.js'; import { ConvexHttpClient } from 'convex/browser'; import { SUPPORTED_STORAGE_VERSIONS, TestStorageConfig, TestStorageFactory } from '@powersync/service-core'; -import * as mongo_storage from '@powersync/service-module-mongodb-storage'; -import * as postgres_storage from '@powersync/service-module-postgres-storage'; import { describe, TestOptions } from 'vitest'; import { env } from '../env.js'; @@ -15,14 +13,28 @@ export type TestConvexConnection = { export const TEST_URI = env.CONVEX_URL; -export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ - url: env.MONGO_TEST_URL, - isCI: env.CI -}); +export const INITIALIZED_MONGO_STORAGE_FACTORY: TestStorageConfig = { + tableIdStrings: false, + factory: async (options) => { + const mongo_storage = await import('@powersync/service-module-mongodb-storage'); + const config = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ + url: env.MONGO_TEST_URL, + isCI: env.CI + }); + return config.factory(options); + } +}; -export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.postgresTestSetup({ - url: env.PG_STORAGE_TEST_URL -}); +export const INITIALIZED_POSTGRES_STORAGE_FACTORY: TestStorageConfig = { + tableIdStrings: true, + factory: async (options) => { + const postgres_storage = await import('@powersync/service-module-postgres-storage'); + const config = postgres_storage.test_utils.postgresTestSetup({ + url: env.PG_STORAGE_TEST_URL + }); + return config.factory(options); + } +}; const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS; From 018d18ae6461c9b776f474951946e973ffca401c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 13 May 2026 11:27:18 +0200 Subject: [PATCH 66/86] revert debugging changes to storage --- modules/module-convex/README.md | 26 ++++++------- .../implementation/MongoBucketBatch.ts | 39 +------------------ .../src/storage/batch/PostgresBucketBatch.ts | 33 +--------------- modules/module-postgres/vitest.config.ts | 9 +---- 4 files changed, 17 insertions(+), 90 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 5f001db26..73ed8aa19 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -21,7 +21,7 @@ replication: ## Manual smoke test -1. Simplest is to run the convex demo in the self-host-demo [repo](https://github.com/powersync-ja/self-host-demo) +1. Simplest is to run the react-convex-todolist demo in the powersync-js[repo](https://github.com/powersync-ja/powersync-js) ## Development @@ -119,18 +119,18 @@ The content below is written in an agents.md style describing the behavior of `m - Current runtime mapping in stream writer: -| Convex Type | TS/JS Type | SQLite type | -| ----------- | ----------- | ---------------------------------- | -| Id | string | text | -| Null | null | null | -| Int64 | bigint | integer | -| Float64 | number | real | -| Boolean | boolean | Up to developer - string or number | -| String | string | text | -| Bytes | ArrayBuffer | text | -| Array | Array | text | -| Object | Object | text | -| Record | Record | text | +| Convex Type | TS/JS Type | SQLite type | +| ----------- | ----------- | ----------- | +| Id | string | text | +| Null | null | null | +| Int64 | bigint | integer | +| Float64 | number | real | +| Boolean | boolean | integer | +| String | string | text | +| Bytes | ArrayBuffer | text | +| Array | Array | text | +| Object | Object | text | +| Record | Record | text | - Convex does not expose a native `Date` wire type; timestamps arrive as `number` or `string`. - BLOB values are valid row values but are not valid bucket parameter values. diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 0df1fc391..61fbdbf60 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -29,7 +29,7 @@ import { BucketDefinitionMapping } from './BucketDefinitionMapping.js'; import { PersistedBatch } from './common/PersistedBatch.js'; import { LoadedSourceRecord, SourceRecordStore } from './common/SourceRecordStore.js'; import type { VersionedPowerSyncMongo } from './db.js'; -import { CurrentBucket, SyncRuleDocument } from './models.js'; +import { SyncRuleDocument } from './models.js'; import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; import { MongoIdSequence } from './MongoIdSequence.js'; import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'; @@ -944,7 +944,7 @@ export abstract class MongoBucketBatch return null; } - this.logger.debug(`Saving ${record.tag}:${formatSaveReplicaIds(record)}`); + this.logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); this.batch ??= new OperationBatch(); this.batch.push(new RecordOperation(record)); @@ -1174,38 +1174,3 @@ export abstract class MongoBucketBatch ); } } - -export function currentBucketKey(b: CurrentBucket) { - return `${b.bucket}/${b.table}/${b.id}`; -} - -function formatReplicaId(value: unknown): string { - if (value == null) { - return '-'; - } - - if (typeof value == 'string' || typeof value == 'number' || typeof value == 'bigint' || typeof value == 'boolean') { - return `${value}`; - } - - try { - return JSON.stringify(value); - } catch { - return '[unserializable]'; - } -} - -function formatSaveReplicaIds(record: storage.SaveOptions): string { - switch (record.tag) { - case SaveOperationTag.INSERT: - return formatReplicaId(record.afterReplicaId); - case SaveOperationTag.DELETE: - return formatReplicaId(record.beforeReplicaId); - case SaveOperationTag.UPDATE: { - if (record.beforeReplicaId == null) { - return formatReplicaId(record.afterReplicaId); - } - return `${formatReplicaId(record.beforeReplicaId)}->${formatReplicaId(record.afterReplicaId)}`; - } - } -} diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index b48de12c7..c44a5d11b 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -163,7 +163,7 @@ export class PostgresBucketBatch return null; } - this.logger.debug(`Saving ${record.tag}:${formatSaveReplicaIds(record)}`); + this.logger.debug(`Saving ${record.tag}:${record.before?.id}/${record.after?.id}`); this.batch ??= new OperationBatch(this.options.batch_limits); this.batch.push(new RecordOperation(record)); @@ -1154,34 +1154,3 @@ export const notifySyncRulesUpdate = async (db: lib_postgres.DatabaseClient, upd statement: `NOTIFY ${NOTIFICATION_CHANNEL}, '${models.ActiveCheckpointNotification.encode({ active_checkpoint: update })}'` }); }; - -function formatReplicaId(value: unknown): string { - if (value == null) { - return '-'; - } - - if (typeof value == 'string' || typeof value == 'number' || typeof value == 'bigint' || typeof value == 'boolean') { - return `${value}`; - } - - try { - return JSON.stringify(value); - } catch { - return '[unserializable]'; - } -} - -function formatSaveReplicaIds(record: storage.SaveOptions): string { - switch (record.tag) { - case storage.SaveOperationTag.INSERT: - return formatReplicaId(record.afterReplicaId); - case storage.SaveOperationTag.DELETE: - return formatReplicaId(record.beforeReplicaId); - case storage.SaveOperationTag.UPDATE: { - if (record.beforeReplicaId == null) { - return formatReplicaId(record.afterReplicaId); - } - return `${formatReplicaId(record.beforeReplicaId)}->${formatReplicaId(record.afterReplicaId)}`; - } - } -} diff --git a/modules/module-postgres/vitest.config.ts b/modules/module-postgres/vitest.config.ts index e3d77ae5a..7cc2993f9 100644 --- a/modules/module-postgres/vitest.config.ts +++ b/modules/module-postgres/vitest.config.ts @@ -1,10 +1,3 @@ import { serviceIntegrationTestConfig } from '../test_config'; -const baseConfig = serviceIntegrationTestConfig(__dirname); -baseConfig.resolve = { - ...baseConfig.resolve, - alias: { - ...baseConfig.resolve?.alias - } -}; -export default baseConfig; +export default serviceIntegrationTestConfig(__dirname); From 489315d23688a5d8d08a58d50a58d40c2918098d Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 13 May 2026 11:43:54 +0200 Subject: [PATCH 67/86] cleanup todo comments --- modules/module-convex/src/replication/ConvexStream.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index cdc4aa95c..a21d79936 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -138,8 +138,9 @@ export class ConvexStream { logger: this.logger, zeroLSN: ZERO_LSN, defaultSchema: this.defaultSchema, - // TODO(steven) check this - storeCurrentData: false, //convex currently has a hard document limit of 1MB per document + // Convex document_deltas include the full document state after each mutation, + // so storage does not need to keep current row data to apply partial updates. + storeCurrentData: false, skipExistingRows: false }); @@ -312,7 +313,7 @@ export class ConvexStream { logger: this.logger, zeroLSN: ZERO_LSN, defaultSchema: this.defaultSchema, - // TODO(steven) check this + // Convex snapshots emit complete documents, so no current row state is needed. storeCurrentData: false, skipExistingRows: true }); From ba0daa119369fdacdebed096977b1959877c149f Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 14 May 2026 14:30:40 +0200 Subject: [PATCH 68/86] handle sqlite conversions according to actual schema responses. --- modules/module-convex/README.md | 2 + modules/module-convex/convex/schema.ts | 12 + .../src/api/ConvexRouteAPIAdapter.ts | 8 +- .../src/common/convex-to-sqlite.ts | 298 ++++++------------ .../test/src/ConvexRouteAPIAdapter.test.ts | 3 +- .../test/src/ConvexStream.test.ts | 2 +- .../test/src/convex-to-sqlite.test.ts | 70 ++-- .../ConvexRouteAPIAdapter.integration.test.ts | 6 +- .../ConvexStream.integration.test.ts | 206 ++++++++++-- 9 files changed, 362 insertions(+), 245 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 73ed8aa19..f7e8e2dfd 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -113,6 +113,7 @@ The content below is written in an agents.md style describing the behavior of `m - Convex `json_schemas` does not provide a schema change token or revision cursor that can be checkpointed. - Current behavior uses `json_schemas` for discovery/debug, but does not continuously diff source schema versions. - Schema changes are not automatically supported at this point. If Convex schema changes (tables or columns), developers must review and redeploy Sync Streams rules manually so PowerSync re-resolves the schema and re-replicates affected streams. +- Convex `json_schemas` can omit fields that do not have populated values at the time the schema is fetched. If PowerSync caches that incomplete schema during an initial snapshot, later values for those fields may be converted using runtime inspection only. - Future improvement: cache a canonicalized `json_schemas` hash, poll periodically, and raise diagnostics when schema drift is detected. ## 7) Datatype Mapping @@ -134,6 +135,7 @@ The content below is written in an agents.md style describing the behavior of `m - Convex does not expose a native `Date` wire type; timestamps arrive as `number` or `string`. - BLOB values are valid row values but are not valid bucket parameter values. +- Known limitation: missing `json_schemas` entries can affect type-sensitive fields. This is most visible for Convex `Int64` values, which may be synced inconsistently as strings or bigints when schema metadata was missing, and for `Bytes` values, which may be represented as base64 strings or otherwise not converted as intended. Until schema handling is improved, users should explicitly cast those columns to strings in Sync Streams rules to avoid ambiguity. ## 8) Checkpointing and Consistency diff --git a/modules/module-convex/convex/schema.ts b/modules/module-convex/convex/schema.ts index d0346b376..0717e44e7 100644 --- a/modules/module-convex/convex/schema.ts +++ b/modules/module-convex/convex/schema.ts @@ -50,6 +50,7 @@ export default defineSchema({ // Number types priority: v.optional(v.number()), + points: v.optional(v.int64()), estimated_hours: v.optional(v.float64()), progress_percentage: v.optional(v.float64()), @@ -57,6 +58,7 @@ export default defineSchema({ is_urgent: v.optional(v.boolean()), is_private: v.optional(v.boolean()), has_attachments: v.optional(v.boolean()), + attachment_data: v.optional(v.bytes()), // Array types tags: v.optional(v.array(v.string())), @@ -64,6 +66,15 @@ export default defineSchema({ assigned_users: v.optional(v.array(v.string())), // Object types + details: v.optional( + v.object({ + label: v.string(), + count: v.number(), + nested: v.object({ + enabled: v.boolean() + }) + }) + ), metadata: v.optional(v.record(v.string(), v.any())), custom_fields: v.optional(v.record(v.string(), v.union(v.string(), v.number(), v.boolean()))), @@ -78,6 +89,7 @@ export default defineSchema({ difficulty: v.optional(v.union(v.literal('easy'), v.literal('medium'), v.literal('hard'))), // Null handling + explicit_null: v.optional(v.null()), archived_at: v.optional(v.union(v.null(), v.string())), deleted_by: v.optional(v.union(v.null(), v.string())), diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index e1f4e9696..b99c5b4b4 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -9,7 +9,7 @@ import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { parseConvexLsn } from '../common/ConvexLSN.js'; -import { extractProperties, readConvexFieldType, toExpressionTypeFromConvexType } from '../common/convex-to-sqlite.js'; +import { extractProperties, jsonSchemaToSQLiteType, readConvexFieldJsonType } from '../common/convex-to-sqlite.js'; import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; import * as types from '../types/types.js'; @@ -149,12 +149,12 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { }) .sort(([a], [b]) => a.localeCompare(b)) .map(([columnName, jsonSchemaProperty]) => { - const jsonType = readConvexFieldType(jsonSchemaProperty); - const sqliteType = toExpressionTypeFromConvexType(jsonType); + const jsonType = readConvexFieldJsonType(jsonSchemaProperty); + const sqliteType = jsonSchemaToSQLiteType(jsonType); return { name: columnName, - type: jsonType, + type: jsonSchemaProperty.type ?? 'unknown', sqlite_type: sqliteType.typeFlags, internal_type: jsonType, pg_type: jsonType diff --git a/modules/module-convex/src/common/convex-to-sqlite.ts b/modules/module-convex/src/common/convex-to-sqlite.ts index 8466794e6..022e9d230 100644 --- a/modules/module-convex/src/common/convex-to-sqlite.ts +++ b/modules/module-convex/src/common/convex-to-sqlite.ts @@ -1,4 +1,5 @@ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; +import { JSONBig } from '@powersync/service-jsonbig'; import { DatabaseInputRow, DatabaseInputValue, @@ -14,7 +15,6 @@ export enum SupportedJSONSchemaPropertyType { BYTES = 'bytes', ARRAY = 'array', OBJECT = 'object', - RECORD = 'record', NULL = 'null', INT64 = 'int64', FLOAT64 = 'float64', @@ -28,7 +28,6 @@ export const CONVEX_TO_SQLITE_TYPE_MAP: Record): Record { // Convex returns each table schema as a standard JSON Schema object. // For example: { "type": "object", "properties": { "name": { "type": "string" } } }. @@ -122,59 +76,73 @@ export function extractProperties(schema: Record): Record { return typeof value == 'object' && value != null && !Array.isArray(value); } -function toBytesValue(value: unknown): Uint8Array | null { - if (value == null) { - return null; - } - - if (value instanceof Uint8Array) { - return value; - } - - if (ArrayBuffer.isView(value)) { - return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); - } - - if (value instanceof ArrayBuffer) { - return new Uint8Array(value); - } - - if (typeof value != 'string') { - throw new ServiceError( - ErrorCode.PSYNC_S1004, - `Convex bytes value must be a base64 string or binary buffer, got ${typeof value}` - ); - } - - const normalized = value.replace(/\s+/g, ''); - const buffer = Buffer.from(normalized, 'base64'); - const canonical = buffer.toString('base64').replace(/=+$/g, ''); - if (normalized != '' && canonical != normalized.replace(/=+$/g, '')) { - throw new ServiceError(ErrorCode.PSYNC_S1004, 'Convex bytes value is not valid base64'); - } - - return new Uint8Array(buffer); -} - -/** - * Reads the Convex field type from the Convex table column's JSON schema entry. - */ -export function readConvexFieldType(jsonSchemaProperty: unknown): string { +export function readConvexFieldJsonType(jsonSchemaProperty: unknown): SupportedJSONSchemaPropertyType { if (!isRecord(jsonSchemaProperty)) { // Invalid schema property entry received - return 'unknown'; - } - - // Convex can expose logical string subtypes through JSON schema `format`. - // For example: { "type": "string", "format": "id" } or { "type": "string", "format": "bytes" }. - // Check this before `type`, otherwise these would be treated as plain strings. - const format = - typeof jsonSchemaProperty.format == 'string' ? normalizeConvexJsonSchemaType(jsonSchemaProperty.format) : null; - if (format == 'bytes' || format == 'id') { - return format; - } - - // Some schema exporters describe binary data as a base64-encoded string. - // For example: { "type": "string", "contentEncoding": "base64" }. - const contentEncoding = - typeof jsonSchemaProperty.contentEncoding == 'string' ? jsonSchemaProperty.contentEncoding.toLowerCase() : null; - if (contentEncoding == 'base64') { - return 'bytes'; - } - - // Standard JSON schema uses a direct `type`. - // For example: { "type": "integer" }, { "type": "number" }, or { "type": "boolean" }. - const directType = typeof jsonSchemaProperty.type == 'string' ? jsonSchemaProperty.type : null; - if (directType != null) { - return normalizeConvexJsonSchemaType(directType); - } - - // Nullable JSON schema fields can use a type array. - // For example: { "type": ["string", "null"] }. Use the first concrete string type. - if (Array.isArray(jsonSchemaProperty.type)) { - const firstString = jsonSchemaProperty.type.find((entry) => typeof entry == 'string'); - if (typeof firstString == 'string') { - return normalizeConvexJsonSchemaType(firstString); - } - } - - // Convex-generated or intermediate schema metadata can carry the same type - // under non-standard keys. For example: { "valueType": "record" }, - // { "fieldType": "bytes" }, or { "kind": "array" }. - const alternateType = - typeof jsonSchemaProperty.valueType == 'string' - ? jsonSchemaProperty.valueType - : typeof jsonSchemaProperty.fieldType == 'string' - ? jsonSchemaProperty.fieldType - : typeof jsonSchemaProperty.kind == 'string' - ? jsonSchemaProperty.kind - : null; - if (alternateType != null) { - return normalizeConvexJsonSchemaType(alternateType); - } - - return 'unknown'; + return SupportedJSONSchemaPropertyType.UNKNOWN; + } + + const description = jsonSchemaProperty['$description']; + const type = jsonSchemaProperty['type']; + /** + * An Int64 example + * "points": {"$description": "int64 represented as base10 string", "type": "string"}, + */ + if (description == 'int64 represented as base10 string' && type == 'string') { + return SupportedJSONSchemaPropertyType.INT64; + } else if (description == 'base64 bytes' && type == 'string') { + /** + * Buffer example + * "attachment_data": {"$description": "base64 bytes", "type": "string"}, + */ + return SupportedJSONSchemaPropertyType.BYTES; + } else if (type == 'string' || type == 'id') { + return SupportedJSONSchemaPropertyType.STRING; + } else if (type == 'boolean') { + return SupportedJSONSchemaPropertyType.BOOLEAN; + } else if (type == 'number') { + // number and float64 seem to both be represented as 'number' in the reported JSON schema + return SupportedJSONSchemaPropertyType.FLOAT64; + } else if (type == 'array') { + return SupportedJSONSchemaPropertyType.ARRAY; + } else if (type == 'object') { + return SupportedJSONSchemaPropertyType.OBJECT; + } else if (type == 'null') { + return SupportedJSONSchemaPropertyType.NULL; + } + + return SupportedJSONSchemaPropertyType.UNKNOWN; } diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index 56000e671..f269523cb 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -32,7 +32,7 @@ function createAdapter() { properties: { _id: { type: 'string' }, age: { type: 'integer' }, - avatar: { type: 'bytes' } + avatar: { type: 'string', $description: 'base64 bytes' } } } } @@ -54,6 +54,7 @@ describe('ConvexRouteAPIAdapter', () => { expect(schema[0]?.name).toBe('convex'); expect(schema[0]?.tables[0]?.name).toBe('users'); expect(schema[0]?.tables[0]?.columns.find((column) => column.name == '_id')?.type).toBe('id'); + expect(schema[0]?.tables[0]?.columns.find((column) => column.name == '_creationTime')).toBeUndefined(); expect(schema[0]?.tables[0]?.columns.find((column) => column.name == 'avatar')?.sqlite_type).toBe( ExpressionType.BLOB.typeFlags ); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index dcbf6fc84..878459aac 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -256,7 +256,7 @@ describe('ConvexStream', () => { schema: { type: 'object', properties: { - avatar: { type: 'bytes' } + avatar: { type: 'string', $description: 'base64 bytes' } } } } diff --git a/modules/module-convex/test/src/convex-to-sqlite.test.ts b/modules/module-convex/test/src/convex-to-sqlite.test.ts index fd920da3f..7462c9086 100644 --- a/modules/module-convex/test/src/convex-to-sqlite.test.ts +++ b/modules/module-convex/test/src/convex-to-sqlite.test.ts @@ -1,8 +1,4 @@ -import { - readConvexFieldType, - toExpressionTypeFromConvexType, - toSqliteInputRow -} from '@module/common/convex-to-sqlite.js'; +import { jsonSchemaToSQLiteType, readConvexFieldJsonType, toSqliteInputRow } from '@module/common/convex-to-sqlite.js'; import { applyRowContext, CompatibilityContext, @@ -15,20 +11,12 @@ import { describe, expect, it } from 'vitest'; const context = new CompatibilityContext({ edition: CompatibilityEdition.SYNC_STREAMS }); describe('convex-to-sqlite', () => { - it('maps Convex types to SQLite expression types', () => { - expect(toExpressionTypeFromConvexType('id')).toBe(ExpressionType.TEXT); - expect(toExpressionTypeFromConvexType('bytes')).toBe(ExpressionType.BLOB); - expect(toExpressionTypeFromConvexType('int64')).toBe(ExpressionType.INTEGER); - expect(toExpressionTypeFromConvexType('float64')).toBe(ExpressionType.REAL); - expect(toExpressionTypeFromConvexType('record')).toBe(ExpressionType.TEXT); - expect(toExpressionTypeFromConvexType('null')).toBe(ExpressionType.NONE); - }); - it('detects bytes and id field types from schema metadata', () => { - expect(readConvexFieldType({ type: 'string', format: 'bytes' })).toBe('bytes'); - expect(readConvexFieldType({ type: 'string', format: 'id' })).toBe('id'); - expect(readConvexFieldType({ valueType: 'record' })).toBe('record'); - expect(readConvexFieldType({ contentEncoding: 'base64', type: 'string' })).toBe('bytes'); + expect(jsonSchemaToSQLiteType(readConvexFieldJsonType({ type: 'string', format: 'id' }))).toBe(ExpressionType.TEXT); + expect(jsonSchemaToSQLiteType(readConvexFieldJsonType({ type: 'object' }))).toBe(ExpressionType.TEXT); + expect(jsonSchemaToSQLiteType(readConvexFieldJsonType({ description: 'base64 bytes', type: 'string' }))).toBe( + ExpressionType.TEXT + ); }); it('decodes bytes to Uint8Array and keeps them out of JSON-compatible values', () => { @@ -41,7 +29,7 @@ describe('convex-to-sqlite', () => { }, { _id: { type: 'id' }, - payload: { type: 'bytes' }, + payload: { $description: 'base64 bytes', type: 'string' }, plain_text: { type: 'string' } } ), @@ -58,4 +46,48 @@ describe('convex-to-sqlite', () => { expect(isJsonValue(payload)).toBe(false); expect(row.plain_text).toBe('AQID'); }); + + it('strips Convex metadata fields that should not be reported to storage', () => { + const row = applyRowContext( + toSqliteInputRow( + { + _id: 'doc1', + _creationTime: 1772817606884, + _table: 'users', + _ts: 1772817606884944123n, + name: 'Alice' + }, + { + _id: { type: 'id' }, + _creationTime: { type: 'float64' }, + name: { type: 'string' } + } + ), + context + ); + + expect(row._id).toBe('doc1'); + expect(row._table).toBeUndefined(); + expect(row._ts).toBeUndefined(); + }); + + it('uses declared numeric field types instead of integer-looking runtime values', () => { + const row = applyRowContext( + toSqliteInputRow( + { + int_value: 1, + float_value: 1 + }, + { + int_value: { type: 'int64' }, + float_value: { type: 'float64' } + } + ), + context + ); + + expect(row.int_value).toBe(1); + expect(row.float_value).toBe(1); + expect(typeof row.float_value).toBe('number'); + }); }); diff --git a/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts b/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts index c43319b5c..2e192c032 100644 --- a/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts +++ b/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts @@ -74,9 +74,9 @@ describe.skipIf(!env.CONVEX_DEPLOY_KEY)('ConvexStream ConvexRouteAPIAdapter test name: 'lists', columns: expect.arrayContaining([ { - internal_type: 'id', + internal_type: 'string', name: '_id', - pg_type: 'id', + pg_type: 'string', sqlite_type: 2, type: 'id' }, @@ -85,7 +85,7 @@ describe.skipIf(!env.CONVEX_DEPLOY_KEY)('ConvexStream ConvexRouteAPIAdapter test name: 'archived', pg_type: 'float64', sqlite_type: 8, - type: 'float64' + type: 'number' }, { internal_type: 'object', diff --git a/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts b/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts index 454a4c69c..8c58d91be 100644 --- a/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts +++ b/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts @@ -3,11 +3,17 @@ import { randomUUID } from 'node:crypto'; import { METRICS_HELPER, removeOp } from '@powersync/service-core-tests'; import { JSONBig } from '@powersync/service-jsonbig'; import { ReplicationMetric } from '@powersync/service-types'; + import { describe, expect, test, vi } from 'vitest'; import { env } from '../env.js'; import { ConvexStreamTestContext } from '../test-utils/ConvexStreamTestContext.js'; import { describeWithStorage, StorageVersionTestContext } from '../test-utils/util.js'; +import schema from '@testing-convex/schema.js'; + +type ListsData = typeof schema.tables.lists.validator.type; +type TodosData = Omit; + const BASIC_SYNC_RULES = ` bucket_definitions: global: @@ -166,16 +172,42 @@ bucket_definitions: `); const list = await createList(context, { name: 'parent-list' }); - const todo = await createTodo(context, { - listUuid: list.uuid, - description: 'snapshot-todo' - }); + + const sampleTodo: TodosData = { + uuid: randomUUID(), + description: 'A test', + list_uuid: list.uuid, + title: 'Typed snapshot todo', + notes: 'notes', + category: 'testing', + priority: 3, + points: 9_007_199_254_740_991n, + estimated_hours: 12.5, + progress_percentage: 87.25, + is_urgent: true, + is_private: false, + has_attachments: true, + attachment_data: Uint8Array.from([0, 1, 255]).buffer, + tags: ['convex', 'sqlite'], + assigned_users: ['alice', 'bob'], + details: { label: 'detail', count: 7, nested: { enabled: true } }, + metadata: { count: 3, enabled: true, nested: { label: 'metadata' } }, + custom_fields: { score: 10, ready: true, owner: 'tester' }, + status: 'in_progress', + difficulty: 'hard', + explicit_null: null, + archived_at: '2026-05-14T00:00:00.000Z', + deleted_by: 'debugger' + }; + + const todo = await createTodo(context, sampleTodo); await context.replicateSnapshot(); const streamedList = await createList(context, { name: 'streamed-list' }); const streamedTodo = await createTodo(context, { - listUuid: streamedList.uuid, + ...sampleTodo, + list_uuid: streamedList.uuid, description: 'streamed-todo' }); @@ -186,6 +218,146 @@ bucket_definitions: convexOp('PUT', 'lists', syncedId(streamedList)), convexOp('PUT', 'todos', syncedId(streamedTodo)) ]); + + // Now verifying the to sqlite typings + const firstTodoOp = data[1]; + const parsedData = JSONBig.parse(firstTodoOp.data!) as Record; + + expect(typeof parsedData.uuid).eq('string'), + expect(typeof parsedData.description).eq('string'), + expect(typeof parsedData.list_uuid).eq('string'), + expect(typeof parsedData.description).eq('string'), + expect(typeof parsedData.list_id).eq('string'), + expect(typeof parsedData.title).eq('string'), + expect(typeof parsedData.notes).eq('string'), + expect(typeof parsedData.category).eq('string'), + expect(typeof parsedData.priority).eq('number'), // This is a regular v.number() + expect(typeof parsedData.points).eq('bigint'), + expect(typeof parsedData.estimated_hours).eq('number'), + expect(typeof parsedData.progress_percentage).eq('number'), + expect(typeof parsedData.is_urgent).eq('bigint'), // boolean + expect(typeof parsedData.is_private).eq('bigint'), // boolean + expect(typeof parsedData.has_attachments).eq('bigint'), // boolean + // TODO, should this have been persisted? I believe this requires a sync config cast in order to work. + // expect(typeof parsedData.attachment_data).eq('string'), // buffer + expect(typeof parsedData.tags).eq('string'), // array + expect(typeof parsedData.assigned_users).eq('string'), //array + expect(typeof parsedData.details).eq('string'), + expect(typeof parsedData.explicit_null).eq('object'), // null + expect(typeof parsedData.archived_at).eq('string'); // '2026-05-14T00:00:00.000Z', + }); + + /** + * It seems like the json-schema's route will not return JSON schema values for table + * columns if not populated value exists for the row yet. + * This test simulates this behaviour. We do an initial replication and start streaming + * before adding any todo records. We create a todo record while streaming. + */ + test('Replicate values when initial snapshot did not include data', async () => { + await using context = await ConvexStreamTestContext.open(factory, { storageVersion }); + await context.updateSyncRules(` +bucket_definitions: + global: + data: + - SELECT uuid as id, * FROM lists + - SELECT uuid as id, * FROM todos + +`); + + const list = await createList(context, { name: 'parent-list' }); + + // This one does not include some optional column values. + // Replicating this will cause the schema for todos to be queried and cached. + const firstTodo = await createTodo(context, { + description: 'the first one, to be recorded in initial snapshot', + list_uuid: list.uuid, + uuid: randomUUID() + }); + + // This starts the replication of the initial snapshot and also then streams + await context.replicateSnapshot(); + + await new Promise((r) => setTimeout(r, 1_000)); + + // Create a new todo record + const sampleTodo: TodosData = { + uuid: randomUUID(), + description: 'A test', + list_uuid: list.uuid, + title: 'Typed snapshot todo', + notes: 'notes', + category: 'testing', + priority: 3, + points: 9_007_199_254_740_991n, + estimated_hours: 12.5, + progress_percentage: 87.25, + is_urgent: true, + is_private: false, + has_attachments: true, + attachment_data: Uint8Array.from([0, 1, 255]).buffer, + tags: ['convex', 'sqlite'], + assigned_users: ['alice', 'bob'], + details: { label: 'detail', count: 7, nested: { enabled: true } }, + metadata: { count: 3, enabled: true, nested: { label: 'metadata' } }, + custom_fields: { score: 10, ready: true, owner: 'tester' }, + status: 'in_progress', + difficulty: 'hard', + explicit_null: null, + archived_at: '2026-05-14T00:00:00.000Z', + deleted_by: 'debugger' + }; + + const streamedList = await createList(context, { name: 'streamed-list' }); + const streamedTodo = await createTodo(context, { + ...sampleTodo, + list_uuid: streamedList.uuid, + description: 'streamed-todo' + }); + + const data = await context.getBucketData('global[]'); + expect(data).toMatchObject([ + convexOp('PUT', 'lists', syncedId(list)), + convexOp('PUT', 'todos', syncedId(firstTodo)), + convexOp('PUT', 'lists', syncedId(streamedList)), + convexOp('PUT', 'todos', syncedId(streamedTodo)) + ]); + + // Now verifying the to sqlite typings + const firstTodoOp = data[3]; + const parsedData = JSONBig.parse(firstTodoOp.data!) as Record; + + expect(typeof parsedData.uuid).eq('string'), + expect(typeof parsedData.description).eq('string'), + expect(typeof parsedData.list_uuid).eq('string'), + expect(typeof parsedData.description).eq('string'), + expect(typeof parsedData.list_id).eq('string'), + expect(typeof parsedData.title).eq('string'), + expect(typeof parsedData.notes).eq('string'), + expect(typeof parsedData.category).eq('string'), + expect(typeof parsedData.priority).eq('number'), // This is a regular v.number() + /** + * This is where the logic breaks down. + * We fetched the schema for todos before any values for points (a int64 column) + * were present. This means the json-schemas result did not include an entry for `points`. + * We then receive a string value for the points value in the document-deltas endpoint. + * We don't have a schema for this column, so we can only do runtime inspection for this value - + * which results in us storing the value as is. + * This results in the type being string instead of bigint. + */ + // expect(typeof parsedData.points).eq('bigint'), + expect(typeof parsedData.points).eq('string'), + expect(typeof parsedData.estimated_hours).eq('number'), + expect(typeof parsedData.progress_percentage).eq('number'), + expect(typeof parsedData.is_urgent).eq('bigint'), // boolean + expect(typeof parsedData.is_private).eq('bigint'), // boolean + expect(typeof parsedData.has_attachments).eq('bigint'), // boolean + // TODO, should this have been persisted? I believe this requires a sync config cast in order to work. + // expect(typeof parsedData.attachment_data).eq('string'), // buffer + expect(typeof parsedData.tags).eq('string'), // array + expect(typeof parsedData.assigned_users).eq('string'), //array + expect(typeof parsedData.details).eq('string'), + expect(typeof parsedData.explicit_null).eq('object'), // null + expect(typeof parsedData.archived_at).eq('string'); // '2026-05-14T00:00:00.000Z', }); test('Replication for tables not in the sync rules are ignored', async () => { @@ -200,7 +372,8 @@ bucket_definitions: // Basic sync streams here don't replicate todo rows await createTodo(context, { - listUuid: list.uuid, + uuid: randomUUID(), + list_uuid: list.uuid, description: 'ignored-todo' }); @@ -261,29 +434,14 @@ async function createLists(context: ConvexStreamTestContext, names: string[]) { })); } -async function createTodo( - context: ConvexStreamTestContext, - options: { - listUuid: string; - description: string; - } -) { - const uuid = randomUUID(); +async function createTodo(context: ConvexStreamTestContext, options: TodosData) { const [id] = await context.backend.client.mutation(context.backend.api.todos.createBatch, { - todos: [ - { - uuid, - list_uuid: options.listUuid, - description: options.description - } - ] + todos: [options] }); return { id, - uuid, - listUuid: options.listUuid, - description: options.description + ...options }; } From adb916b43b12c8dcf0459cf27adacbae37f9aa27 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 14 May 2026 16:52:30 +0200 Subject: [PATCH 69/86] cleanup --- modules/module-convex/README.md | 5 +-- .../src/client/ConvexApiClient.ts | 16 ++++----- modules/module-convex/src/types/types.ts | 23 +++++++++++- .../test/src/ConvexApiClient.test.ts | 35 +++++++++++++++++-- modules/module-convex/test/src/types.test.ts | 34 ++++++++++++++++++ 5 files changed, 98 insertions(+), 15 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index f7e8e2dfd..af0fe3838 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -141,8 +141,9 @@ The content below is written in an agents.md style describing the behavior of `m - `createReplicationHead` must: 1. resolve global head cursor, - 2. write a Convex checkpoint marker via `POST /api/mutation` (calls `powersync_checkpoints:createCheckpoint`), - 3. then pass the head to callback. + 2. pass that head to the callback so PowerSync stores the managed write checkpoint mapping, + 3. then write a Convex checkpoint marker via `POST /api/mutation` (calls `powersync_checkpoints:createCheckpoint`). +- The marker must be written after the callback. If the marker is replicated before the managed write checkpoint mapping exists, an idle source can still leave the client waiting for a later observable checkpoint update. - Source marker table: `powersync_checkpoints` - Convex rejects table names starting with `_`, so no leading-underscore variant is used. - The table has a single `last_updated` field; the mutation upserts one row (bounded to one row total). diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index 03ae0587b..073013f69 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -13,8 +13,6 @@ import { ensureRawJsonSchemaResponse } from './ConvexAPITypes.js'; -const CONVEX_REQUEST_TIMEOUT_MS = 60_000; - type GetRequestParams = { path: string; params?: Record; @@ -97,8 +95,6 @@ export class ConvexApiClient { } async createWriteCheckpointMarker(options?: { signal?: AbortSignal }): Promise { - await this.assertHostAllowed(); - await this.performRequest({ method: 'POST', url: new URL('/api/mutation', this.config.deployment_url), @@ -167,10 +163,10 @@ export class ConvexApiClient { const { method, url, body, extraHeaders = {}, signal: requestSignal } = options; const timeout = new AbortController(); - const timeoutPromise = delay(CONVEX_REQUEST_TIMEOUT_MS, undefined, { + const timeoutPromise = delay(this.config.request_timeout_ms, undefined, { signal: timeout.signal }).then(() => { - timeout.abort(new Error(`Convex API request timed out after ${CONVEX_REQUEST_TIMEOUT_MS}ms`)); + timeout.abort(new Error(`Convex API request timed out after ${this.config.request_timeout_ms}ms`)); }); const signals = [timeout.signal]; @@ -181,6 +177,8 @@ export class ConvexApiClient { const signal = AbortSignal.any(signals); try { + await this.assertHostAllowed(url); + const response = await fetch(url, { method, headers: { @@ -248,15 +246,13 @@ export class ConvexApiClient { } } - private async assertHostAllowed(): Promise { + private async assertHostAllowed(url: URL): Promise { if (!this.config.lookup) { return; } - const hostname = new URL(this.config.deployment_url).hostname; - await new Promise((resolve, reject) => { - this.config.lookup!(hostname, {}, (error) => { + this.config.lookup!(url.hostname, {}, (error) => { if (error) { reject(error); } else { diff --git a/modules/module-convex/src/types/types.ts b/modules/module-convex/src/types/types.ts index bbd4f7eb2..5f75f6baf 100644 --- a/modules/module-convex/src/types/types.ts +++ b/modules/module-convex/src/types/types.ts @@ -4,6 +4,8 @@ import { LookupFunction } from 'node:net'; import * as t from 'ts-codec'; export const CONVEX_CONNECTION_TYPE = 'convex' as const; +const DEFAULT_POLLING_INTERVAL_MS = 1_000; +const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; export interface NormalizedConvexConnectionConfig { id: string; @@ -15,6 +17,7 @@ export interface NormalizedConvexConnectionConfig { debug_api: boolean; polling_interval_ms: number; + request_timeout_ms: number; lookup?: LookupFunction; } @@ -25,6 +28,7 @@ export const ConvexConnectionConfig = service_types.configFile.DataSourceConfig. deployment_url: t.string, deploy_key: t.string, polling_interval_ms: t.number.optional(), + request_timeout_ms: t.number.optional(), reject_ip_ranges: t.array(t.string).optional() }) ); @@ -59,6 +63,22 @@ export function normalizeConnectionConfig(options: ConvexConnectionConfig): Norm throw new ServiceError(ErrorCode.PSYNC_S1108, `Convex connection: deploy_key required`); } + const pollingIntervalMs = options.polling_interval_ms ?? DEFAULT_POLLING_INTERVAL_MS; + if (!Number.isFinite(pollingIntervalMs) || pollingIntervalMs <= 0) { + throw new ServiceError( + ErrorCode.PSYNC_S1109, + `Convex connection: polling_interval_ms must be a positive finite number` + ); + } + + const requestTimeoutMs = options.request_timeout_ms ?? DEFAULT_REQUEST_TIMEOUT_MS; + if (!Number.isFinite(requestTimeoutMs) || requestTimeoutMs <= 0) { + throw new ServiceError( + ErrorCode.PSYNC_S1109, + `Convex connection: request_timeout_ms must be a positive finite number` + ); + } + const lookup = makeHostnameLookupFunction(deploymentURL.hostname, { reject_ip_ranges: options.reject_ip_ranges ?? [] }); @@ -72,7 +92,8 @@ export function normalizeConnectionConfig(options: ConvexConnectionConfig): Norm deploy_key: options.deploy_key, debug_api: options.debug_api ?? false, - polling_interval_ms: options.polling_interval_ms ?? 1_000, + polling_interval_ms: pollingIntervalMs, + request_timeout_ms: requestTimeoutMs, lookup }; diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index c495a8f12..dd0990326 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -14,6 +14,7 @@ const SNAPSHOT_CURSOR = 1770335566197683000n; describe('ConvexApiClient', () => { afterEach(() => { + vi.useRealTimers(); vi.restoreAllMocks(); }); @@ -85,6 +86,31 @@ describe('ConvexApiClient', () => { }); }); + it('uses the configured request timeout', async () => { + vi.useFakeTimers(); + vi.spyOn(globalThis, 'fetch').mockImplementation( + (_url, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + reject(init.signal!.reason); + }); + }) + ); + + const client = new ConvexApiClient({ + ...baseConfig, + request_timeout_ms: 25 + }); + + const request = expect(client.getJsonSchemas()).rejects.toMatchObject({ + message: expect.stringContaining('timed out after 25ms'), + retryable: true + }); + + await vi.advanceTimersByTimeAsync(25); + await request; + }); + it('creates write checkpoint markers via mutation', async () => { const fetchSpy = vi .spyOn(globalThis, 'fetch') @@ -117,7 +143,7 @@ describe('ConvexApiClient', () => { }); }); - it('checks the hostname policy before creating checkpoint markers', async () => { + it('checks the hostname policy before every Convex API request', async () => { const fetchSpy = vi .spyOn(globalThis, 'fetch') .mockResolvedValue(new Response(JSON.stringify({ status: 'success' }), { status: 200 })); @@ -130,8 +156,13 @@ describe('ConvexApiClient', () => { lookup }); + await expect(client.getJsonSchemas()).rejects.toThrow('blocked by reject_ip_ranges'); + await expect(client.listSnapshot({ tableName: 'lists' })).rejects.toThrow('blocked by reject_ip_ranges'); + await expect(client.documentDeltas({ cursor: SNAPSHOT_CURSOR.toString() })).rejects.toThrow( + 'blocked by reject_ip_ranges' + ); await expect(client.createWriteCheckpointMarker()).rejects.toThrow('blocked by reject_ip_ranges'); - expect(lookup).toHaveBeenCalledTimes(1); + expect(lookup).toHaveBeenCalledTimes(4); expect(fetchSpy).not.toHaveBeenCalled(); }); }); diff --git a/modules/module-convex/test/src/types.test.ts b/modules/module-convex/test/src/types.test.ts index ef2087604..94627e550 100644 --- a/modules/module-convex/test/src/types.test.ts +++ b/modules/module-convex/test/src/types.test.ts @@ -12,9 +12,21 @@ describe('Convex connection config', () => { expect(config.id).toBe('default'); expect(config.tag).toBe('default'); expect(config.polling_interval_ms).toBe(1000); + expect(config.request_timeout_ms).toBe(60_000); expect(config.deployment_url).toBe('https://example.convex.cloud'); }); + it('normalizes custom request timeout', () => { + const config = normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'https://example.convex.cloud', + deploy_key: 'secret-key', + request_timeout_ms: 30_000 + }); + + expect(config.request_timeout_ms).toBe(30_000); + }); + it('throws for invalid URL', () => { expect(() => normalizeConnectionConfig({ @@ -34,4 +46,26 @@ describe('Convex connection config', () => { }) ).toThrow(); }); + + it.each([-1, 0, Number.NaN, Number.POSITIVE_INFINITY])('throws for invalid polling_interval_ms: %s', (value) => { + expect(() => + normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'https://example.convex.cloud', + deploy_key: 'secret-key', + polling_interval_ms: value + }) + ).toThrow('polling_interval_ms must be a positive finite number'); + }); + + it.each([-1, 0, Number.NaN, Number.POSITIVE_INFINITY])('throws for invalid request_timeout_ms: %s', (value) => { + expect(() => + normalizeConnectionConfig({ + type: CONVEX_CONNECTION_TYPE, + deployment_url: 'https://example.convex.cloud', + deploy_key: 'secret-key', + request_timeout_ms: value + }) + ).toThrow('request_timeout_ms must be a positive finite number'); + }); }); From 05ff780420b7d4b582605b9b9fe0076eb9c29d8b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 14 May 2026 17:17:41 +0200 Subject: [PATCH 70/86] cleanup readmes --- docs/convex/convex-write-checkpoints.md | 9 +++++---- modules/module-convex/README.md | 12 ++++++++++-- ...gresCursor.ts => ConvexSnapshotProgressCursor.ts} | 0 .../module-convex/src/replication/ConvexStream.ts | 2 +- modules/module-convex/test/src/ConvexStream.test.ts | 2 +- 5 files changed, 17 insertions(+), 8 deletions(-) rename modules/module-convex/src/replication/{ConvexSnapshotProgresCursor.ts => ConvexSnapshotProgressCursor.ts} (100%) diff --git a/docs/convex/convex-write-checkpoints.md b/docs/convex/convex-write-checkpoints.md index f3848daad..7bd46c508 100644 --- a/docs/convex/convex-write-checkpoints.md +++ b/docs/convex/convex-write-checkpoints.md @@ -28,14 +28,15 @@ For Convex, the source position is the Convex replication cursor. The current implementation in `createReplicationHead`: 1. calls `getHeadCursor()` to read the current global Convex head, -2. calls `createWriteCheckpointMarker()` to run the - `powersync_checkpoints:createCheckpoint` Convex mutation, -3. invokes the callback with the original head cursor. +2. invokes the callback with the original head cursor so PowerSync stores the + managed write checkpoint mapping, +3. calls `createWriteCheckpointMarker()` to run the + `powersync_checkpoints:createCheckpoint` Convex mutation. The callback stores the managed write checkpoint in bucket storage with the original head as the replication head. The marker write is intentionally not the write checkpoint position. It is a later Convex mutation whose job is to advance -the Convex delta stream beyond the stored head. +the Convex delta stream beyond the stored head after the managed mapping exists. The key invariant is that PowerSync must observe a checkpoint update at or past the stored head after the managed write checkpoint mapping exists. Other source diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index af0fe3838..ce67176e6 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -128,14 +128,22 @@ The content below is written in an agents.md style describing the behavior of `m | Float64 | number | real | | Boolean | boolean | integer | | String | string | text | -| Bytes | ArrayBuffer | text | +| Bytes | ArrayBuffer | blob | | Array | Array | text | | Object | Object | text | | Record | Record | text | - Convex does not expose a native `Date` wire type; timestamps arrive as `number` or `string`. - BLOB values are valid row values but are not valid bucket parameter values. -- Known limitation: missing `json_schemas` entries can affect type-sensitive fields. This is most visible for Convex `Int64` values, which may be synced inconsistently as strings or bigints when schema metadata was missing, and for `Bytes` values, which may be represented as base64 strings or otherwise not converted as intended. Until schema handling is improved, users should explicitly cast those columns to strings in Sync Streams rules to avoid ambiguity. +- Value conversion flow: + 1. PowerSync requests snapshots or document deltas from Convex's Streaming Export APIs. + 2. The JSON response is parsed into raw Convex documents, where row columns are represented as JSON object fields. + 3. PowerSync queries Convex's `json_schemas` endpoint for table schema metadata. + 4. The raw document values and schema metadata are used together to convert Convex values to SQLite-compatible values. +- Known limitation: Convex JSON documents are close to SQLite-compatible input, but some values need schema metadata to preserve their intended type. Boolean values are handled internally as SQLite integers, but `Int64` and `Bytes` values are ambiguous on the JSON wire: + - Convex `Int64` values arrive in raw documents as JSON strings. With schema metadata, PowerSync converts those strings to SQLite integers backed by `bigint`. Without schema metadata, the same value is indistinguishable from an ordinary string and may be synced as text. + - Convex `Bytes` values arrive in raw documents as base64 strings. With schema metadata, PowerSync decodes them to SQLite blobs. Without schema metadata, the value is indistinguishable from an ordinary string and may be synced as text. +- Convex's `json_schemas` endpoint appears to infer table schemas from table summaries instead of using the TypeScript schema as a complete source of truth. This means fields can be absent from `json_schemas` until populated data exists. For columns that may suffer from this ambiguity, users should explicitly cast those values in Sync Streams rules, commonly to `TEXT`, until Convex exposes complete schema metadata. ## 8) Checkpointing and Consistency diff --git a/modules/module-convex/src/replication/ConvexSnapshotProgresCursor.ts b/modules/module-convex/src/replication/ConvexSnapshotProgressCursor.ts similarity index 100% rename from modules/module-convex/src/replication/ConvexSnapshotProgresCursor.ts rename to modules/module-convex/src/replication/ConvexSnapshotProgressCursor.ts diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index a21d79936..8a33273b1 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -25,7 +25,7 @@ import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { lsnCursorToDate, parseConvexLsn, ZERO_LSN } from '../common/ConvexLSN.js'; import { extractProperties, toSqliteInputRow } from '../common/convex-to-sqlite.js'; import { ConvexConnectionManager } from './ConvexConnectionManager.js'; -import { BinaryConvexSnapshotProgressCursor, decodeSnapshotProgressCursor } from './ConvexSnapshotProgresCursor.js'; +import { BinaryConvexSnapshotProgressCursor, decodeSnapshotProgressCursor } from './ConvexSnapshotProgressCursor.js'; export interface ConvexStreamOptions { connections: ConvexConnectionManager; diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 878459aac..194810190 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -1,6 +1,6 @@ import { ConvexDocumentDeltasResult, ConvexJsonSchemasResult } from '@module/client/ConvexAPITypes.js'; import { parseConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; -import { BinaryConvexSnapshotProgressCursor } from '@module/replication/ConvexSnapshotProgresCursor.js'; +import { BinaryConvexSnapshotProgressCursor } from '@module/replication/ConvexSnapshotProgressCursor.js'; import { ConvexStream } from '@module/replication/ConvexStream.js'; import { SaveOperationTag, SourceTable } from '@powersync/service-core'; import { TablePattern } from '@powersync/service-sync-rules'; From 70f7a5f8429c2382a37b26ff3eb1ff36478ea4e5 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 14 May 2026 17:28:41 +0200 Subject: [PATCH 71/86] cleanup convex api validations --- modules/module-convex/README.md | 1 + modules/module-convex/package.json | 1 - .../src/client/ConvexAPITypes.ts | 46 ++++++++++--------- pnpm-lock.yaml | 3 -- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index ce67176e6..b54ff32f3 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -104,6 +104,7 @@ The content below is written in an agents.md style describing the behavior of `m - Auth header: `Authorization: Convex `. - Always request `format=json`. - Parse large numeric JSON using `JSONBig`. +- Convex API response shape validation is disabled by default. Set `POWERSYNC_DEV_CHECK_CONVEX_RESPONSES` before service startup to validate responses with the shared ts-codec JSON schema validator while debugging API compatibility. - Retry classification: - retryable: network, timeout, 429, 5xx. - non-retryable: malformed responses, auth/config issues. diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json index 85d4a35fc..6dda9180b 100644 --- a/modules/module-convex/package.json +++ b/modules/module-convex/package.json @@ -35,7 +35,6 @@ "@powersync/service-jsonbig": "workspace:*", "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", - "ajv": "^8.18.0", "bson": "^6.10.4", "ts-codec": "^1.3.0" }, diff --git a/modules/module-convex/src/client/ConvexAPITypes.ts b/modules/module-convex/src/client/ConvexAPITypes.ts index bbc796b00..d24ab151b 100644 --- a/modules/module-convex/src/client/ConvexAPITypes.ts +++ b/modules/module-convex/src/client/ConvexAPITypes.ts @@ -1,5 +1,4 @@ -import { BufferNodeType } from '@powersync/lib-services-framework'; -import AJV from 'ajv'; +import { schema } from '@powersync/lib-services-framework'; import * as t from 'ts-codec'; export const ConvexRawDocument = t @@ -55,8 +54,9 @@ export const ConvexListSnapshotResult = t.object({ export type ConvexListSnapshotResult = t.Encoded; -// These validators help assert the API structure response. -// These could be disabled for production. +const CHECK_CONVEX_RESPONSES_ENV = 'POWERSYNC_DEV_CHECK_CONVEX_RESPONSES'; + +// These validators optionally assert the API response shape in development. export const ensureConvexListSnapshotResult = ensureResponseFormatValidator(ConvexListSnapshotResult); export const ConvexDocumentDeltasResult = t.object({ @@ -93,34 +93,38 @@ export type RawJsonSchemaResponse = t.Encoded; export const ensureRawJsonSchemaResponse = ensureResponseFormatValidator(RawJsonSchemaResponse); /** - * Performs a validation which ensures the Convex API response data matches the codec specification. - * This was added after noticing the original implementation was coercing various permutations of - * response fields e.g. `has_more` and `hasMore` in responses. There were comments that the - * self hosted and cloud Convex implementations might have returned different responses. - * In testing, with these validations, I could not see any actual discrepency in responses. - * Having these checks could help spot potential changes to the API - however they do come at a cost. - * We could disable this in prod builds or remove in the future. For now, while the API seems fickle, it could - * be nice to have a safety net. + * Optionally validates that Convex API response data matches the codec specification. + * + * These checks were originally added because earlier iterations of this client + * handled several Convex API route and field-name permutations, such as falling + * back between `has_more` and `hasMore`. That suggested there may have been + * response differences between Convex Cloud and self-hosted Convex deployments. + * + * The current response typings have been verified with cloud and self-hosted + * integration tests, so we do not need to pay this validation cost in + * production. The checks are still useful while developing against Convex API + * changes, so they are kept as an opt-in guardrail. + * + * Set POWERSYNC_DEV_CHECK_CONVEX_RESPONSES to enable this development safety net. */ export function ensureResponseFormatValidator( codec: Codec ): (data: unknown) => t.Encoded { - const schema = t.generateJSONSchema(codec, { + if (!process.env[CHECK_CONVEX_RESPONSES_ENV]) { + return (data: unknown) => data as t.Encoded; + } + + const validator = schema.createTsCodecValidator(codec, { parsers: [bigintParser], allowAdditional: true }); - const ajv = new AJV.Ajv({ - allErrors: true, - keywords: [BufferNodeType] - }); - const validator = ajv.compile(schema); return (data: unknown) => { - const isValid = validator(data); - if (!isValid) { + const result = validator.validate(data); + if (!result.valid) { // This does not result in leaking failed data, it only logs the keys which failed validation throw new Error( - `Invalid data received. Got parsing errors when checking data format. Keys which failed validation: ${ajv.errors?.map((e) => e.propertyName).join(', ')}` + `Invalid data received. Got parsing errors when checking data format. Errors: ${result.errors.join(', ')}` ); } return data as t.Encoded; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d8eecd1d..232693f35 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -213,9 +213,6 @@ importers: '@powersync/service-types': specifier: workspace:* version: link:../../packages/types - ajv: - specifier: ^8.18.0 - version: 8.18.0 bson: specifier: ^6.10.4 version: 6.10.4 From 580f87a0ec8b1508ace74ac0ecf54fdc0e599aaa Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 15 May 2026 09:04:05 +0200 Subject: [PATCH 72/86] update note about int64 values --- modules/module-convex/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index b54ff32f3..6453767af 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -144,7 +144,7 @@ The content below is written in an agents.md style describing the behavior of `m - Known limitation: Convex JSON documents are close to SQLite-compatible input, but some values need schema metadata to preserve their intended type. Boolean values are handled internally as SQLite integers, but `Int64` and `Bytes` values are ambiguous on the JSON wire: - Convex `Int64` values arrive in raw documents as JSON strings. With schema metadata, PowerSync converts those strings to SQLite integers backed by `bigint`. Without schema metadata, the same value is indistinguishable from an ordinary string and may be synced as text. - Convex `Bytes` values arrive in raw documents as base64 strings. With schema metadata, PowerSync decodes them to SQLite blobs. Without schema metadata, the value is indistinguishable from an ordinary string and may be synced as text. -- Convex's `json_schemas` endpoint appears to infer table schemas from table summaries instead of using the TypeScript schema as a complete source of truth. This means fields can be absent from `json_schemas` until populated data exists. For columns that may suffer from this ambiguity, users should explicitly cast those values in Sync Streams rules, commonly to `TEXT`, until Convex exposes complete schema metadata. +- Convex's `json_schemas` endpoint appears to infer table schemas from table summaries instead of using the TypeScript schema as a complete source of truth. This means fields can be absent from `json_schemas` until populated data exists. For `Int64` columns that may suffer from this ambiguity, users should explicitly cast those values in Sync Streams rules using `CAST(value AS INTEGER)` so either a raw string or already-converted `bigint` is emitted as a SQLite integer. ## 8) Checkpointing and Consistency From 38a819a0fe0f54b5f884fca0b30c3e154f586ffe Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 15 May 2026 14:28:04 +0200 Subject: [PATCH 73/86] don't use json-schemas for sqlite type conversion --- modules/module-convex/README.md | 35 +++--- .../src/common/convex-to-sqlite.ts | 104 ++++-------------- .../src/replication/ConvexStream.ts | 19 +--- .../test/src/ConvexRouteAPIAdapter.test.ts | 2 +- .../test/src/ConvexStream.test.ts | 4 +- .../test/src/convex-to-sqlite.test.ts | 76 +++++-------- .../ConvexStream.integration.test.ts | 20 +--- 7 files changed, 74 insertions(+), 186 deletions(-) diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 6453767af..0d5bb2d84 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -114,37 +114,32 @@ The content below is written in an agents.md style describing the behavior of `m - Convex `json_schemas` does not provide a schema change token or revision cursor that can be checkpointed. - Current behavior uses `json_schemas` for discovery/debug, but does not continuously diff source schema versions. - Schema changes are not automatically supported at this point. If Convex schema changes (tables or columns), developers must review and redeploy Sync Streams rules manually so PowerSync re-resolves the schema and re-replicates affected streams. -- Convex `json_schemas` can omit fields that do not have populated values at the time the schema is fetched. If PowerSync caches that incomplete schema during an initial snapshot, later values for those fields may be converted using runtime inspection only. +- Convex `json_schemas` can omit fields that do not have populated values at the time the schema is fetched. To keep replicated values consistent, stream row conversion does not use `json_schemas` metadata for datatype coercion. - Future improvement: cache a canonicalized `json_schemas` hash, poll periodically, and raise diagnostics when schema drift is detected. ## 7) Datatype Mapping - Current runtime mapping in stream writer: -| Convex Type | TS/JS Type | SQLite type | -| ----------- | ----------- | ----------- | -| Id | string | text | -| Null | null | null | -| Int64 | bigint | integer | -| Float64 | number | real | -| Boolean | boolean | integer | -| String | string | text | -| Bytes | ArrayBuffer | blob | -| Array | Array | text | -| Object | Object | text | -| Record | Record | text | +| Convex Type | JSON wire / Sync Streams value | SQLite type | +| ----------- | ------------------------------ | ----------- | +| Id | string | text | +| Null | null | null | +| Int64 | base10 string | text | +| Float64 | number | real | +| Boolean | boolean | integer | +| String | string | text | +| Bytes | base64 string | text | +| Array | Array | text | +| Object | Object | text | +| Record | Record | text | - Convex does not expose a native `Date` wire type; timestamps arrive as `number` or `string`. -- BLOB values are valid row values but are not valid bucket parameter values. - Value conversion flow: 1. PowerSync requests snapshots or document deltas from Convex's Streaming Export APIs. 2. The JSON response is parsed into raw Convex documents, where row columns are represented as JSON object fields. - 3. PowerSync queries Convex's `json_schemas` endpoint for table schema metadata. - 4. The raw document values and schema metadata are used together to convert Convex values to SQLite-compatible values. -- Known limitation: Convex JSON documents are close to SQLite-compatible input, but some values need schema metadata to preserve their intended type. Boolean values are handled internally as SQLite integers, but `Int64` and `Bytes` values are ambiguous on the JSON wire: - - Convex `Int64` values arrive in raw documents as JSON strings. With schema metadata, PowerSync converts those strings to SQLite integers backed by `bigint`. Without schema metadata, the same value is indistinguishable from an ordinary string and may be synced as text. - - Convex `Bytes` values arrive in raw documents as base64 strings. With schema metadata, PowerSync decodes them to SQLite blobs. Without schema metadata, the value is indistinguishable from an ordinary string and may be synced as text. -- Convex's `json_schemas` endpoint appears to infer table schemas from table summaries instead of using the TypeScript schema as a complete source of truth. This means fields can be absent from `json_schemas` until populated data exists. For `Int64` columns that may suffer from this ambiguity, users should explicitly cast those values in Sync Streams rules using `CAST(value AS INTEGER)` so either a raw string or already-converted `bigint` is emitted as a SQLite integer. + 3. PowerSync converts the raw JSON-compatible values to SQLite-compatible values without using `json_schemas` metadata for datatype coercion. +- Convex's `json_schemas` endpoint appears to infer table schemas from table summaries instead of using the TypeScript schema as a complete source of truth. This means fields can be absent from `json_schemas` until populated data exists. To avoid changing replicated value types based on whether schema metadata was available, Sync Streams see the stable JSON wire representation. For `Int64` columns, users should explicitly cast those values in Sync Streams rules using `CAST(value AS INTEGER)`. ## 8) Checkpointing and Consistency diff --git a/modules/module-convex/src/common/convex-to-sqlite.ts b/modules/module-convex/src/common/convex-to-sqlite.ts index 022e9d230..eb109b746 100644 --- a/modules/module-convex/src/common/convex-to-sqlite.ts +++ b/modules/module-convex/src/common/convex-to-sqlite.ts @@ -1,4 +1,3 @@ -import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; import { JSONBig } from '@powersync/service-jsonbig'; import { DatabaseInputRow, @@ -25,11 +24,11 @@ export enum SupportedJSONSchemaPropertyType { export const CONVEX_TO_SQLITE_TYPE_MAP: Record = { [SupportedJSONSchemaPropertyType.ID]: ExpressionType.TEXT, [SupportedJSONSchemaPropertyType.STRING]: ExpressionType.TEXT, - [SupportedJSONSchemaPropertyType.BYTES]: ExpressionType.BLOB, + [SupportedJSONSchemaPropertyType.BYTES]: ExpressionType.TEXT, [SupportedJSONSchemaPropertyType.ARRAY]: ExpressionType.TEXT, [SupportedJSONSchemaPropertyType.OBJECT]: ExpressionType.TEXT, [SupportedJSONSchemaPropertyType.NULL]: ExpressionType.NONE, - [SupportedJSONSchemaPropertyType.INT64]: ExpressionType.INTEGER, + [SupportedJSONSchemaPropertyType.INT64]: ExpressionType.TEXT, [SupportedJSONSchemaPropertyType.FLOAT64]: ExpressionType.REAL, [SupportedJSONSchemaPropertyType.BOOLEAN]: ExpressionType.INTEGER, [SupportedJSONSchemaPropertyType.UNKNOWN]: ExpressionType.TEXT @@ -41,10 +40,7 @@ export function jsonSchemaToSQLiteType(jsonType: SupportedJSONSchemaPropertyType return CONVEX_TO_SQLITE_TYPE_MAP[jsonType]; } -export function toSqliteInputRow( - change: ConvexRawDocument, - jsonSchemaProperties?: Record -): SqliteInputRow { +export function toSqliteInputRow(change: ConvexRawDocument): SqliteInputRow { const row: DatabaseInputRow = {}; for (const [key, value] of Object.entries(change)) { @@ -52,7 +48,7 @@ export function toSqliteInputRow( continue; } - row[key] = toDatabaseValue(value, readConvexFieldJsonType(jsonSchemaProperties?.[key])); + row[key] = toDatabaseValue(value); } return toSyncRulesRow(row); @@ -68,84 +64,24 @@ export function extractProperties(schema: Record): Record { diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 8a33273b1..e7df65d07 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -23,7 +23,7 @@ import { ConvexListSnapshotResult, ConvexRawDocument, ConvexTableSchema } from ' import { isCursorExpiredError } from '../client/ConvexApiClient.js'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { lsnCursorToDate, parseConvexLsn, ZERO_LSN } from '../common/ConvexLSN.js'; -import { extractProperties, toSqliteInputRow } from '../common/convex-to-sqlite.js'; +import { toSqliteInputRow } from '../common/convex-to-sqlite.js'; import { ConvexConnectionManager } from './ConvexConnectionManager.js'; import { BinaryConvexSnapshotProgressCursor, decodeSnapshotProgressCursor } from './ConvexSnapshotProgressCursor.js'; @@ -51,7 +51,6 @@ export class ConvexStream { private replicationLag = new ReplicationLagTracker(); private tableSchemaCache: ConvexTableSchema[] | null = null; - private tableSchemaPropertiesByName = new Map>(); private lastKeepaliveAt = 0; private lastTouchedAt = performance.now(); @@ -359,7 +358,6 @@ export class ConvexStream { table: SourceTable, snapshotCursor: string ): Promise<{ table: SourceTable }> { - const tableProperties = this.getTableSchemaProperties(table.name); const snapshotProgress = decodeSnapshotProgressCursor(table.snapshotStatus?.lastKey); let pageCursor = snapshotProgress.cursor; let replicatedCount = table.snapshotStatus?.replicatedCount ?? 0; @@ -415,7 +413,7 @@ export class ConvexStream { continue; } - const row = this.toSqliteRow(rawDocument, tableProperties); + const row = this.toSqliteRow(rawDocument); await batch.save({ tag: SaveOperationTag.INSERT, sourceTable: latestTable, @@ -640,7 +638,7 @@ export class ConvexStream { return true; } - const after = this.toSqliteRow(change, this.getTableSchemaProperties(table.name)); + const after = this.toSqliteRow(change); await batch.save({ tag: SaveOperationTag.UPDATE, sourceTable: table, @@ -653,8 +651,8 @@ export class ConvexStream { return true; } - private toSqliteRow(change: ConvexRawDocument, properties?: Record) { - return this.syncRules.applyRowContext(toSqliteInputRow(change, properties)); + private toSqliteRow(change: ConvexRawDocument) { + return this.syncRules.applyRowContext(toSqliteInputRow(change)); } private async getAllTableSchemas(options?: { force?: boolean }): Promise { @@ -664,16 +662,9 @@ export class ConvexStream { const schema = await this.connections.client.getJsonSchemas({ signal: this.abortSignal }); this.tableSchemaCache = schema.tables; - this.tableSchemaPropertiesByName = new Map( - schema.tables.map((table) => [table.tableName, extractProperties(table.schema)]) - ); return schema.tables; } - private getTableSchemaProperties(tableName: string): Record | undefined { - return this.tableSchemaPropertiesByName.get(tableName); - } - private touch() { if (performance.now() - this.lastTouchedAt < 1_000) { return; diff --git a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts index f269523cb..5b64ddf56 100644 --- a/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts +++ b/modules/module-convex/test/src/ConvexRouteAPIAdapter.test.ts @@ -56,7 +56,7 @@ describe('ConvexRouteAPIAdapter', () => { expect(schema[0]?.tables[0]?.columns.find((column) => column.name == '_id')?.type).toBe('id'); expect(schema[0]?.tables[0]?.columns.find((column) => column.name == '_creationTime')).toBeUndefined(); expect(schema[0]?.tables[0]?.columns.find((column) => column.name == 'avatar')?.sqlite_type).toBe( - ExpressionType.BLOB.typeFlags + ExpressionType.TEXT.typeFlags ); await adapter.shutdown(); diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 194810190..af5c94728 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -233,7 +233,7 @@ describe('ConvexStream', () => { expect(context.commits.at(-1)).toBe(parseConvexLsn(CURSOR_100)); }); - it('decodes bytes fields to Uint8Array during snapshot hydration', async () => { + it('keeps bytes fields as base64 strings during snapshot hydration', async () => { const context = createFakeStorage(); const abortController = new AbortController(); @@ -288,7 +288,7 @@ describe('ConvexStream', () => { await stream.initReplication(); expect(context.saves).toHaveLength(1); - expect(context.saves[0]?.after.avatar).toEqual(Uint8Array.of(1, 2, 3)); + expect(context.saves[0]?.after.avatar).toBe('AQID'); }); it('marks snapshot done without re-reading rows when the final page was already flushed', async () => { diff --git a/modules/module-convex/test/src/convex-to-sqlite.test.ts b/modules/module-convex/test/src/convex-to-sqlite.test.ts index 7462c9086..a0710f883 100644 --- a/modules/module-convex/test/src/convex-to-sqlite.test.ts +++ b/modules/module-convex/test/src/convex-to-sqlite.test.ts @@ -3,8 +3,7 @@ import { applyRowContext, CompatibilityContext, CompatibilityEdition, - ExpressionType, - isJsonValue + ExpressionType } from '@powersync/service-sync-rules'; import { describe, expect, it } from 'vitest'; @@ -14,55 +13,40 @@ describe('convex-to-sqlite', () => { it('detects bytes and id field types from schema metadata', () => { expect(jsonSchemaToSQLiteType(readConvexFieldJsonType({ type: 'string', format: 'id' }))).toBe(ExpressionType.TEXT); expect(jsonSchemaToSQLiteType(readConvexFieldJsonType({ type: 'object' }))).toBe(ExpressionType.TEXT); - expect(jsonSchemaToSQLiteType(readConvexFieldJsonType({ description: 'base64 bytes', type: 'string' }))).toBe( + expect(jsonSchemaToSQLiteType(readConvexFieldJsonType({ $description: 'base64 bytes', type: 'string' }))).toBe( ExpressionType.TEXT ); + expect( + jsonSchemaToSQLiteType( + readConvexFieldJsonType({ $description: 'int64 represented as base10 string', type: 'string' }) + ) + ).toBe(ExpressionType.TEXT); }); - it('decodes bytes to Uint8Array and keeps them out of JSON-compatible values', () => { + it('keeps bytes as base64 strings without using schema metadata for row conversion', () => { const row = applyRowContext( - toSqliteInputRow( - { - _id: 'doc1', - payload: 'AQID', - plain_text: 'AQID' - }, - { - _id: { type: 'id' }, - payload: { $description: 'base64 bytes', type: 'string' }, - plain_text: { type: 'string' } - } - ), + toSqliteInputRow({ + _id: 'doc1', + payload: 'AQID', + plain_text: 'AQID' + }), context ); - const payload = row.payload; expect(row._id).toBe('doc1'); - expect(payload).toEqual(Uint8Array.of(1, 2, 3)); - expect(payload instanceof Uint8Array).toBe(true); - if (!(payload instanceof Uint8Array)) { - throw new Error('Expected payload to be Uint8Array'); - } - expect(isJsonValue(payload)).toBe(false); + expect(row.payload).toBe('AQID'); expect(row.plain_text).toBe('AQID'); }); it('strips Convex metadata fields that should not be reported to storage', () => { const row = applyRowContext( - toSqliteInputRow( - { - _id: 'doc1', - _creationTime: 1772817606884, - _table: 'users', - _ts: 1772817606884944123n, - name: 'Alice' - }, - { - _id: { type: 'id' }, - _creationTime: { type: 'float64' }, - name: { type: 'string' } - } - ), + toSqliteInputRow({ + _id: 'doc1', + _creationTime: 1772817606884, + _table: 'users', + _ts: 1772817606884944123n, + name: 'Alice' + }), context ); @@ -71,22 +55,16 @@ describe('convex-to-sqlite', () => { expect(row._ts).toBeUndefined(); }); - it('uses declared numeric field types instead of integer-looking runtime values', () => { + it('keeps raw JSON wire values instead of applying declared numeric field types', () => { const row = applyRowContext( - toSqliteInputRow( - { - int_value: 1, - float_value: 1 - }, - { - int_value: { type: 'int64' }, - float_value: { type: 'float64' } - } - ), + toSqliteInputRow({ + int_value: '9007199254740991', + float_value: 1 + }), context ); - expect(row.int_value).toBe(1); + expect(row.int_value).toBe('9007199254740991'); expect(row.float_value).toBe(1); expect(typeof row.float_value).toBe('number'); }); diff --git a/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts b/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts index 8c58d91be..a6401294d 100644 --- a/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts +++ b/modules/module-convex/test/src/integration/ConvexStream.integration.test.ts @@ -232,14 +232,13 @@ bucket_definitions: expect(typeof parsedData.notes).eq('string'), expect(typeof parsedData.category).eq('string'), expect(typeof parsedData.priority).eq('number'), // This is a regular v.number() - expect(typeof parsedData.points).eq('bigint'), + expect(typeof parsedData.points).eq('string'), // int64 values use the raw Convex JSON wire type expect(typeof parsedData.estimated_hours).eq('number'), expect(typeof parsedData.progress_percentage).eq('number'), expect(typeof parsedData.is_urgent).eq('bigint'), // boolean expect(typeof parsedData.is_private).eq('bigint'), // boolean expect(typeof parsedData.has_attachments).eq('bigint'), // boolean - // TODO, should this have been persisted? I believe this requires a sync config cast in order to work. - // expect(typeof parsedData.attachment_data).eq('string'), // buffer + expect(typeof parsedData.attachment_data).eq('string'), // base64 bytes expect(typeof parsedData.tags).eq('string'), // array expect(typeof parsedData.assigned_users).eq('string'), //array expect(typeof parsedData.details).eq('string'), @@ -335,24 +334,13 @@ bucket_definitions: expect(typeof parsedData.notes).eq('string'), expect(typeof parsedData.category).eq('string'), expect(typeof parsedData.priority).eq('number'), // This is a regular v.number() - /** - * This is where the logic breaks down. - * We fetched the schema for todos before any values for points (a int64 column) - * were present. This means the json-schemas result did not include an entry for `points`. - * We then receive a string value for the points value in the document-deltas endpoint. - * We don't have a schema for this column, so we can only do runtime inspection for this value - - * which results in us storing the value as is. - * This results in the type being string instead of bigint. - */ - // expect(typeof parsedData.points).eq('bigint'), - expect(typeof parsedData.points).eq('string'), + expect(typeof parsedData.points).eq('string'), // int64 values use the raw Convex JSON wire type expect(typeof parsedData.estimated_hours).eq('number'), expect(typeof parsedData.progress_percentage).eq('number'), expect(typeof parsedData.is_urgent).eq('bigint'), // boolean expect(typeof parsedData.is_private).eq('bigint'), // boolean expect(typeof parsedData.has_attachments).eq('bigint'), // boolean - // TODO, should this have been persisted? I believe this requires a sync config cast in order to work. - // expect(typeof parsedData.attachment_data).eq('string'), // buffer + expect(typeof parsedData.attachment_data).eq('string'), // base64 bytes expect(typeof parsedData.tags).eq('string'), // array expect(typeof parsedData.assigned_users).eq('string'), //array expect(typeof parsedData.details).eq('string'), From 0b8ea26c27882d88f7dd8add89f710e5dc38cb60 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 18 May 2026 11:22:12 +0200 Subject: [PATCH 74/86] updates for schema change handling --- docs/convex/schema-change-handling.md | 148 ++++++++++++++++++ modules/module-convex/README.md | 17 +- modules/module-convex/convex/schema.ts | 5 + .../src/replication/ConvexStream.ts | 64 +++----- .../test/src/ConvexStream.test.ts | 41 ++--- .../ConvexRouteAPIAdapter.integration.test.ts | 20 +++ 6 files changed, 218 insertions(+), 77 deletions(-) create mode 100644 docs/convex/schema-change-handling.md diff --git a/docs/convex/schema-change-handling.md b/docs/convex/schema-change-handling.md new file mode 100644 index 000000000..9e7299b64 --- /dev/null +++ b/docs/convex/schema-change-handling.md @@ -0,0 +1,148 @@ +# Convex schema change handling + +This note explains why the Convex replicator does not implement the same +schema-change detection flow used by relational and CDC-based replicators. + +## What conventional schema change handling is for + +In source modules such as Postgres, MySQL, MongoDB, and SQL Server, schema or +relation change handling is used to keep PowerSync's source table metadata in +step with the source database. Typical reasons include: + +1. detecting changed replica identity columns, +2. refreshing cached table or relation metadata, +3. handling table rename, drop, and truncate events, +4. detecting source object identity changes, +5. deciding whether a table must be dropped, truncated, or re-snapshotted, +6. updating column metadata used by row decoding. + +Those concerns matter when the replication stream contains DDL, relation +messages, capture-instance changes, table ids, or column metadata that affect how +PowerSync should interpret subsequent row changes. + +## Why Convex is different + +Convex replication uses the Streaming Export APIs: + +- `json_schemas` for table-name discovery and schema preview, +- `list_snapshot` for initial table snapshots, +- `document_deltas` for ongoing changes. + +The replicated row shape comes from the actual JSON document returned by +`list_snapshot` and `document_deltas`. It does not come from `json_schemas`. +The Convex row converter iterates over the document payload, skips PowerSync's +internal Convex metadata fields, and converts the JSON-compatible values into +Sync Streams input values. + +That means ordinary Convex document shape changes flow through as data changes: + +- if a field is added, later snapshot/delta payloads include the field, +- if a field is removed, later payloads omit the field, +- if a field changes type, later payloads carry the new JSON value, +- if a migration updates existing documents, those updates are normal Convex + writes and should appear in `document_deltas`. + +Convex tables also always use `_id` as the replication identity in this module. +There is no source-specific primary key or replica identity definition to track, +so the usual "replica identity changed" path does not apply. + +## How table discovery works + +Exact table patterns in Sync Streams rules do not need `json_schemas` for +replication. The table name is already present in the sync rule, and the stream +can resolve that table directly. + +Wildcard table patterns do need a source of table names during initial +replication. For those patterns, the Convex replicator uses `json_schemas` to +expand the wildcard into concrete table names before snapshotting. + +An integration test verifies the key assumption behind that behavior: +`json_schemas` lists schema-defined tables even when those tables contain no +documents. This means initial wildcard expansion can discover empty tables +without waiting for a first row write. + +## New tables while streaming + +When `document_deltas` contains a row for a table that is not in the relation +cache, the replicator checks whether that table is selected by the current Sync +Streams rules. If it is selected, the table is resolved in storage and marked as +snapshot-complete at the current delta cursor. + +The replicator does not run an inline snapshot for that newly observed table. +The reasoning is: + +1. existing wildcard-selected tables were already discovered through + `json_schemas` during initial replication, +2. a table that appears later in `document_deltas` has appeared because a source + write occurred, +3. the delta payload contains the full document state needed for replication, +4. an empty newly-created table has no data to snapshot. + +So the delta stream itself is the source of truth for newly observed table data +after the initial snapshot boundary. + +## When a re-snapshot is still needed + +Convex does not need a re-snapshot merely because a field was added, removed, or +changed type. Those are document changes and should replicate through normal +deltas. + +A re-snapshot is still required for cases that change the selected data set or +invalidate the source cursor model: + +- initial replication of tables selected by the current Sync Streams rules, +- a Sync Streams deployment that selects new existing data, +- restarting initial replication from an incomplete snapshot, +- a lost or expired Convex cursor that requires restarting from a fresh + snapshot boundary. + +These are selection or consistency-boundary events, not schema-metadata drift +events. + +## Table drops + +The Convex stream does not continuously diff `json_schemas` to detect that a +table has disappeared from the source schema. + +Validation against the Convex dashboard showed that deleting a table from the +dashboard does not emit per-document `_deleted` rows in `document_deltas`. That +means PowerSync will not automatically remove previously replicated rows for that +table, and clients can continue to see stale synced data. + +If a table needs to be decommissioned while preserving replication correctness, +clear the table before deleting it. In the Convex dashboard, use the "Clear +Table" action first, then delete the table after those document removals have +replicated. Deleting documents through Convex mutations is also valid when that +path emits document delete deltas. Otherwise, treat dashboard table deletion or +schema-only table removal as a sync-rule/deployment state change: review the +affected rules and re-replicate or clear affected PowerSync state as needed. + +## Role of `json_schemas` + +`json_schemas` remains useful for: + +- validating that the Convex deployment is reachable, +- checking that required support tables such as `powersync_checkpoints` exist, +- presenting database schema information through API/debug routes, +- expanding wildcard table patterns at initial replication time. + +It is intentionally not used for runtime row decoding or schema drift detection. +Convex `json_schemas` can omit field detail until data exists, and using it for +type coercion would make replicated values depend on metadata availability +rather than on the source JSON payload. + +## Summary + +For Convex, schema changes are best understood as document-shape changes rather +than DDL events. Since PowerSync replicates the JSON payloads directly and `_id` +is always the replica identity, conventional schema-change detection has little +value for correctness. + +The important guarantees are instead: + +1. initial wildcard expansion can discover all schema-defined tables, +2. snapshot reads use a pinned Convex snapshot cursor, +3. streaming reads the unfiltered global `document_deltas` cursor in order, +4. new selected tables observed in deltas are resolved without an inline + snapshot, +5. migrations that update data are replicated as normal document changes. diff --git a/modules/module-convex/README.md b/modules/module-convex/README.md index 0d5bb2d84..512585e1c 100644 --- a/modules/module-convex/README.md +++ b/modules/module-convex/README.md @@ -78,7 +78,7 @@ The content below is written in an agents.md style describing the behavior of `m - Start from persisted resume LSN. - Poll `document_deltas` using frequency configured in `polling_interval_ms` - Always stream globally (no `tableName` filter), then filter locally by selected Sync Streams tables. - - If a table is first seen in a `document_deltas` page and matches Sync Streams, snapshot it inline at that page boundary and skip that table's delta rows from the same page, because the snapshot already includes them. + - If a table is first seen in a `document_deltas` page and matches Sync Streams, resolve it and apply the delta row directly. Do not snapshot it inline; initial wildcard expansion already discovers schema-defined tables through `json_schemas`, and the delta payload is the source of truth for later writes. ## 3) Hard Invariants (Do Not Break) @@ -109,13 +109,16 @@ The content below is written in an agents.md style describing the behavior of `m - retryable: network, timeout, 429, 5xx. - non-retryable: malformed responses, auth/config issues. -## 6) Schema Change Caveat +## 6) Schema Changes -- Convex `json_schemas` does not provide a schema change token or revision cursor that can be checkpointed. -- Current behavior uses `json_schemas` for discovery/debug, but does not continuously diff source schema versions. -- Schema changes are not automatically supported at this point. If Convex schema changes (tables or columns), developers must review and redeploy Sync Streams rules manually so PowerSync re-resolves the schema and re-replicates affected streams. -- Convex `json_schemas` can omit fields that do not have populated values at the time the schema is fetched. To keep replicated values consistent, stream row conversion does not use `json_schemas` metadata for datatype coercion. -- Future improvement: cache a canonicalized `json_schemas` hash, poll periodically, and raise diagnostics when schema drift is detected. +- Conventional schema change handling is mainly used to detect changed replica identity columns, update cached table metadata, drop/rename tables, and trigger a table re-snapshot when DDL changed storage semantics. +- Convex tables always use `_id` as the replication identity, so there is no equivalent replica identity drift to detect. +- Stream row conversion uses the JSON document returned by `list_snapshot` and `document_deltas`, not Convex `json_schemas` metadata. Added fields, removed fields, and type changes are therefore replicated through normal document mutations. +- Convex data migrations are expected to run as writes/mutations over live documents. Those updates should appear in `document_deltas` and be replicated without a schema-driven re-snapshot. +- Exact table patterns are resolved directly from Sync Streams rules. `json_schemas` is only used for initial wildcard table expansion and API/debug schema previews. +- A re-snapshot is still required for initial replication, a sync-rule deployment that selects new existing data, or a lost/expired cursor, but not merely because a Convex field was added, removed, or changed type. +- Table drops are not (yet) detected by continuously diffing `json_schemas`. Validation showed that deleting a table from the Convex dashboard does not emit per-document `_deleted` rows in `document_deltas`, so previously replicated rows can remain synced to clients. Use the dashboard "Clear Table" action before deleting a table, or delete documents through mutation paths that emit document deltas. Otherwise, handle dashboard/schema-only table removal as a sync-rule/deployment state change and clear/re-replicate affected PowerSync state. +- See [Convex schema change handling](../../docs/convex/schema-change-handling.md) for the detailed rationale and limitations. ## 7) Datatype Mapping diff --git a/modules/module-convex/convex/schema.ts b/modules/module-convex/convex/schema.ts index 0717e44e7..fa4209f25 100644 --- a/modules/module-convex/convex/schema.ts +++ b/modules/module-convex/convex/schema.ts @@ -108,5 +108,10 @@ export default defineSchema({ powersync_checkpoints: defineTable({ last_updated: v.float64() + }), + + // An empty table which only is used to check if the json-schemas endpoint returns data for empty tables. + schema_only_probe: defineTable({ + marker: v.optional(v.string()) }) }); diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index e7df65d07..dfbc881ca 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -19,7 +19,7 @@ import { import { HydratedSyncRules, TablePattern } from '@powersync/service-sync-rules'; import { ReplicationMetric } from '@powersync/service-types'; import { setTimeout as delay } from 'timers/promises'; -import { ConvexListSnapshotResult, ConvexRawDocument, ConvexTableSchema } from '../client/ConvexAPITypes.js'; +import { ConvexListSnapshotResult, ConvexRawDocument } from '../client/ConvexAPITypes.js'; import { isCursorExpiredError } from '../client/ConvexApiClient.js'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { lsnCursorToDate, parseConvexLsn, ZERO_LSN } from '../common/ConvexLSN.js'; @@ -50,8 +50,6 @@ export class ConvexStream { private readonly relationCache = new RelationCache(getCacheIdentifier); private replicationLag = new ReplicationLagTracker(); - private tableSchemaCache: ConvexTableSchema[] | null = null; - private lastKeepaliveAt = 0; private lastTouchedAt = performance.now(); @@ -173,8 +171,6 @@ export class ConvexStream { let changesInPage = 0; const transactionTimestampsInPage = new Set(); let sawCheckpointMarker = false; - const snapshottedTablesInPage = new Set(); - let didMarkOldestUncommitedChange = false; /** @@ -206,13 +202,10 @@ export class ConvexStream { } lastTransactionTimestamp = transactionTimestamp; - const table = await this.getOrResolveTable(batch, tableName, nextCursor.toString(), snapshottedTablesInPage); + const table = await this.getOrResolveTable(batch, tableName, nextCursor.toString()); if (table == null || !table.syncAny) { continue; } - if (snapshottedTablesInPage.has(tableName)) { - continue; - } /** * This tracks the begining of a new transaction which is not yet commited. @@ -502,17 +495,7 @@ export class ConvexStream { return []; } - const availableTableNames = (await this.getAllTableSchemas()).map((table) => table.tableName); - - const matchedTableNames = availableTableNames - .filter((tableName) => { - if (tablePattern.isWildcard) { - return tableName.startsWith(tablePattern.tablePrefix); - } - return tableName == tablePattern.name; - }) - .filter((tableName) => !isConvexCheckpointTable(tableName)) - .sort(); + const matchedTableNames = await this.resolveTablePattern(tablePattern); if (!tablePattern.isWildcard && matchedTableNames.length == 0) { this.logger.warn(`Table ${tablePattern.schema}.${tablePattern.name} not found`); @@ -535,8 +518,7 @@ export class ConvexStream { private async getOrResolveTable( batch: storage.BucketStorageBatch, tableName: string, - snapshotCursor: string, - snapshottedTablesInPage: Set + snapshotCursor: string ): Promise { const descriptor: SourceEntityDescriptor = { schema: this.defaultSchema, @@ -554,20 +536,13 @@ export class ConvexStream { return null; } - await this.getAllTableSchemas({ force: true }); - let table = await this.processTable(batch, descriptor); if (!table.snapshotComplete && table.syncAny) { - this.logger.info(`New table discovered while streaming: [${table.qualifiedName}]`); - await batch.truncate([table]); - table = await batch.updateTableProgress(table, { - totalEstimatedCount: -1, - replicatedCount: 0, - lastKey: null - }); + this.logger.info( + `New table discovered while streaming: [${table.qualifiedName}], applying deltas without snapshot` + ); + [table] = await batch.markTableSnapshotDone([table], parseConvexLsn(snapshotCursor)); this.relationCache.update(table); - table = (await this.snapshotTable(batch, table, snapshotCursor)).table; - snapshottedTablesInPage.add(tableName); } return table; @@ -614,6 +589,19 @@ export class ConvexStream { return resolved.table; } + private async resolveTablePattern(tablePattern: TablePattern): Promise { + if (!tablePattern.isWildcard) { + return isConvexCheckpointTable(tablePattern.name) ? [] : [tablePattern.name]; + } + + const schema = await this.connections.client.getJsonSchemas({ signal: this.abortSignal }); + return schema.tables + .map((table) => table.tableName) + .filter((tableName) => tableName.startsWith(tablePattern.tablePrefix)) + .filter((tableName) => !isConvexCheckpointTable(tableName)) + .sort(); + } + private async writeChange( batch: storage.BucketStorageBatch, table: SourceTable, @@ -655,16 +643,6 @@ export class ConvexStream { return this.syncRules.applyRowContext(toSqliteInputRow(change)); } - private async getAllTableSchemas(options?: { force?: boolean }): Promise { - if (!options?.force && this.tableSchemaCache != null) { - return this.tableSchemaCache; - } - - const schema = await this.connections.client.getJsonSchemas({ signal: this.abortSignal }); - this.tableSchemaCache = schema.tables; - return schema.tables; - } - private touch() { if (performance.now() - this.lastTouchedAt < 1_000) { return; diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index af5c94728..ac4572b87 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -1,4 +1,4 @@ -import { ConvexDocumentDeltasResult, ConvexJsonSchemasResult } from '@module/client/ConvexAPITypes.js'; +import { ConvexDocumentDeltasResult } from '@module/client/ConvexAPITypes.js'; import { parseConvexLsn, ZERO_LSN } from '@module/common/ConvexLSN.js'; import { BinaryConvexSnapshotProgressCursor } from '@module/replication/ConvexSnapshotProgressCursor.js'; import { ConvexStream } from '@module/replication/ConvexStream.js'; @@ -177,6 +177,10 @@ describe('ConvexStream', () => { const context = createFakeStorage(); const abortController = new AbortController(); const snapshotCalls: any[] = []; + const getJsonSchemas = vi.fn(async () => ({ + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], + raw: {} + })); const listSnapshot = vi.fn(async (options: any) => { snapshotCalls.push(options ?? {}); if (options?.tableName == null) { @@ -208,10 +212,7 @@ describe('ConvexStream', () => { connectionId: '1', config: { pollingIntervalMs: 1 }, client: { - getJsonSchemas: async () => ({ - tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], - raw: {} - }), + getJsonSchemas, listSnapshot, getGlobalSnapshotCursor: async (options?: any) => (await listSnapshot(options)).snapshot } @@ -226,6 +227,7 @@ describe('ConvexStream', () => { expect(snapshotCalls[1]?.tableName).toBe('users'); expect(snapshotCalls[1]?.cursor).toBeUndefined(); expect(snapshotCalls[1]?.snapshot).toBe(CURSOR_100); + expect(getJsonSchemas).not.toHaveBeenCalled(); expect(context.saves.length).toBe(1); expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); expect(context.resumeLsnUpdates.length).toBe(1); @@ -461,11 +463,6 @@ describe('ConvexStream', () => { connectionId: '1', config: { pollingIntervalMs: 1 }, client: { - getJsonSchemas: async () => { - return { - tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }] - } satisfies ConvexJsonSchemasResult; - }, documentDeltas: async () => { return { cursor: CURSOR_102, @@ -483,7 +480,7 @@ describe('ConvexStream', () => { await expect(stream.streamChanges()).rejects.toThrow(/out-of-order _ts values/); }); - it('refreshes metadata before snapshotting a newly discovered wildcard-matched table inline', async () => { + it('resolves a newly discovered wildcard-matched table from document deltas without snapshotting', async () => { const context = createFakeStorage({ snapshotDone: true, resumeFromLsn: parseConvexLsn(CURSOR_100), @@ -496,12 +493,7 @@ describe('ConvexStream', () => { tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }], raw: {} })); - const listSnapshot = vi.fn(async (options: any) => ({ - snapshot: CURSOR_101, - cursor: null, - hasMore: false, - values: [{ _table: 'projects_archive', _id: 'p1', name: 'From snapshot' }] - })); + const listSnapshot = vi.fn(); const stream = new ConvexStream({ abortSignal: abortController.signal, @@ -523,7 +515,7 @@ describe('ConvexStream', () => { return { cursor: CURSOR_101, hasMore: false, - values: [{ _table: 'projects_archive', _id: 'p1', name: 'From delta' }] + values: [{ _table: 'projects_archive', _id: 'p1', _ts: CURSOR_101, name: 'From delta' }] }; } } @@ -533,16 +525,11 @@ describe('ConvexStream', () => { await stream.streamChanges(); expect(calls).toBeGreaterThan(0); - expect(getJsonSchemas).toHaveBeenCalledTimes(2); - expect(listSnapshot).toHaveBeenCalledTimes(1); - expect(listSnapshot).toHaveBeenCalledWith({ - tableName: 'projects_archive', - snapshot: CURSOR_101.toString(), - cursor: undefined, - signal: abortController.signal - }); + expect(getJsonSchemas).toHaveBeenCalledTimes(1); + expect(listSnapshot).not.toHaveBeenCalled(); expect(context.saves.length).toBe(1); - expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); + expect(context.saves[0]?.tag).toBe(SaveOperationTag.UPDATE); + expect(context.saves[0]?.after.name).toBe('From delta'); expect(context.saves[0]?.sourceTable.name).toBe('projects_archive'); expect(context.tables.get('projects_archive')?.snapshotComplete).toBe(true); }); diff --git a/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts b/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts index 2e192c032..d58bb6de6 100644 --- a/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts +++ b/modules/module-convex/test/src/integration/ConvexRouteAPIAdapter.integration.test.ts @@ -37,6 +37,26 @@ function normalizeSchemaForSnapshot(schema: DatabaseSchema[]): DatabaseSchema[] } describe.skipIf(!env.CONVEX_DEPLOY_KEY)('ConvexStream ConvexRouteAPIAdapter tests', function () { + test('json_schemas lists schema-defined tables without documents', async () => { + /** + * The Convex stream uses json_schemas for initial wildcard table expansion. + * If json_schemas only listed tables that already contain documents, then a + * wildcard sync rule could miss an empty-but-schema-defined table at the + * initial snapshot boundary and would need a later inline snapshot when the + * first delta appears. This verifies that table names are available even + * before any documents exist, so new table deltas can be applied directly. + */ + await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY.factory, {}); + + const schemas = await context.connectionManager.client.getJsonSchemas(); + const tableNames = schemas.tables.map((table) => table.tableName); + + expect(tableNames).toContain('schema_only_probe'); + + const page = await context.connectionManager.client.listSnapshot({ tableName: 'schema_only_probe' }); + expect(page.values).toHaveLength(0); + }); + test('retrieves the testing Convex schema in the expected service schema format', async () => { /** * It seems like Convex requires the table to contain populated columns in order to report From 30459a19d208efa717639790e7b67115d4b343af Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 18 May 2026 13:19:10 +0200 Subject: [PATCH 75/86] AI feedback - only return table names if they are present in the json-schemas list of tables (for non-wildcard table patterns). --- .../src/replication/ConvexStream.ts | 13 ++--- .../test/src/ConvexStream.test.ts | 50 ++++++++++++++++++- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index dfbc881ca..b1b3df176 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -590,16 +590,17 @@ export class ConvexStream { } private async resolveTablePattern(tablePattern: TablePattern): Promise { - if (!tablePattern.isWildcard) { - return isConvexCheckpointTable(tablePattern.name) ? [] : [tablePattern.name]; - } - const schema = await this.connections.client.getJsonSchemas({ signal: this.abortSignal }); - return schema.tables + const availableTableNames = schema.tables .map((table) => table.tableName) - .filter((tableName) => tableName.startsWith(tablePattern.tablePrefix)) .filter((tableName) => !isConvexCheckpointTable(tableName)) .sort(); + + if (!tablePattern.isWildcard) { + return availableTableNames.includes(tablePattern.name) ? [tablePattern.name] : []; + } + + return availableTableNames.filter((tableName) => tableName.startsWith(tablePattern.tablePrefix)); } private async writeChange( diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index ac4572b87..6563df9ee 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -227,7 +227,7 @@ describe('ConvexStream', () => { expect(snapshotCalls[1]?.tableName).toBe('users'); expect(snapshotCalls[1]?.cursor).toBeUndefined(); expect(snapshotCalls[1]?.snapshot).toBe(CURSOR_100); - expect(getJsonSchemas).not.toHaveBeenCalled(); + expect(getJsonSchemas).toHaveBeenCalledTimes(1); expect(context.saves.length).toBe(1); expect(context.saves[0]?.tag).toBe(SaveOperationTag.INSERT); expect(context.resumeLsnUpdates.length).toBe(1); @@ -235,6 +235,51 @@ describe('ConvexStream', () => { expect(context.commits.at(-1)).toBe(parseConvexLsn(CURSOR_100)); }); + it('does not snapshot exact tables missing from json_schemas', async () => { + const context = createFakeStorage(); + const abortController = new AbortController(); + const listSnapshot = vi.fn(async (options: any) => { + if (options?.tableName == null) { + return { + snapshot: CURSOR_100, + cursor: null, + hasMore: false, + values: [] + }; + } + + throw new Error(`Unexpected table snapshot for ${options.tableName}`); + }); + + const stream = new ConvexStream({ + abortSignal: abortController.signal, + storage: context.storage as any, + metrics: { + getCounter: () => ({ add: () => {} }) + } as any, + connections: { + schema: 'convex', + connectionTag: 'default', + connectionId: '1', + config: { pollingIntervalMs: 1 }, + client: { + getJsonSchemas: async () => ({ + tables: [], + raw: {} + }), + listSnapshot, + getGlobalSnapshotCursor: async (options?: any) => (await listSnapshot(options)).snapshot + } + } as any + }); + + await stream.initReplication(); + + expect(listSnapshot).toHaveBeenCalledTimes(1); + expect(context.saves).toHaveLength(0); + expect(context.allSnapshotDoneLsns).toEqual([parseConvexLsn(CURSOR_100)]); + }); + it('keeps bytes fields as base64 strings during snapshot hydration', async () => { const context = createFakeStorage(); const abortController = new AbortController(); @@ -463,6 +508,9 @@ describe('ConvexStream', () => { connectionId: '1', config: { pollingIntervalMs: 1 }, client: { + getJsonSchemas: async () => ({ + tables: [{ tableName: 'users', schema: { type: 'object', properties: {} } }] + }), documentDeltas: async () => { return { cursor: CURSOR_102, From 92503045b441634903545ca236c72edddfcaada0 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 19 May 2026 16:40:24 +0200 Subject: [PATCH 76/86] update table resolve logic --- .../src/api/ConvexRouteAPIAdapter.ts | 22 ++--- .../src/replication/ConvexStream.ts | 93 +++++++++++-------- .../test/src/ConvexStream.test.ts | 54 ++++++++--- 3 files changed, 101 insertions(+), 68 deletions(-) diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index b99c5b4b4..f9d6179a1 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -1,10 +1,4 @@ -import { - api, - ParseSyncRulesOptions, - ReplicationHeadCallback, - ReplicationLagOptions, - SourceTable -} from '@powersync/service-core'; +import { api, ParseSyncRulesOptions, ReplicationHeadCallback, ReplicationLagOptions } from '@powersync/service-core'; import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; @@ -201,23 +195,19 @@ function createTableInfo(options: { const tableName = options.tableName ?? (options.tablePattern.isWildcard ? options.tablePattern.tablePrefix : options.tablePattern.name); - const sourceTable = new SourceTable({ - id: tableName, + const ref = { connectionTag: options.connectionTag, - objectId: tableName, schema: options.tablePattern.schema, - name: tableName, - replicaIdColumns: [{ name: '_id' }], - snapshotComplete: true - }); + name: tableName + } satisfies sync_rules.SourceTableRef; return { schema: options.tablePattern.schema, name: tableName, pattern: options.tablePattern.isWildcard ? options.tablePattern.tablePattern : undefined, replication_id: ['_id'], - data_queries: options.syncRules.tableSyncsData(sourceTable), - parameter_queries: options.syncRules.tableSyncsParameters(sourceTable), + data_queries: options.syncRules.tableSyncsData(ref), + parameter_queries: options.syncRules.tableSyncsParameters(ref), errors: options.errors ?? [] }; } diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index b1b3df176..db42069f9 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -202,8 +202,8 @@ export class ConvexStream { } lastTransactionTimestamp = transactionTimestamp; - const table = await this.getOrResolveTable(batch, tableName, nextCursor.toString()); - if (table == null || !table.syncAny) { + const tables = await this.getOrResolveTables(batch, tableName, nextCursor.toString()); + if (tables == null || tables.length == 0) { continue; } @@ -220,16 +220,22 @@ export class ConvexStream { didMarkOldestUncommitedChange = true; } - const changed = await this.writeChange(batch, table, change); - if (!changed) { - continue; + let wroteChange = false; + for (const table of tables) { + if (!table.syncAny) { + continue; + } + const changed = await this.writeChange(batch, table, change); + wroteChange = wroteChange || changed; } - // Convex assigns one _ts commit timestamp to every write in a mutation. - // document_deltas may return multiple mutations in one page, so transaction - // metrics are counted by distinct _ts values, not by delta pages. - changesInPage += 1; - transactionTimestampsInPage.add(transactionTimestamp.toString()); + if (wroteChange) { + // Convex assigns one _ts commit timestamp to every write in a mutation. + // document_deltas may return multiple mutations in one page, so transaction + // metrics are counted by distinct _ts values, not by delta pages. + changesInPage += 1; + transactionTimestampsInPage.add(transactionTimestamp.toString()); + } } if (changesInPage > 0) { @@ -472,12 +478,20 @@ export class ConvexStream { } private async resolveAllSourceTables(batch: storage.BucketStorageBatch): Promise { - const sourceTables = this.syncRules.getSourceTables(); + const sourceTablePatterns = this.syncRules.getSourceTables(); const resolved: SourceTable[] = []; + const seenSourceTableIds = new Set(); - for (const tablePattern of sourceTables) { + for (const tablePattern of sourceTablePatterns) { const tables = await this.resolveQualifiedTableNames(batch, tablePattern); - resolved.push(...tables); + for (const table of tables) { + const id = `${table.id}`; + if (seenSourceTableIds.has(id)) { + continue; + } + seenSourceTableIds.add(id); + resolved.push(table); + } } return resolved; @@ -503,49 +517,54 @@ export class ConvexStream { const resolved: SourceTable[] = []; for (const tableName of matchedTableNames) { - const table = await this.processTable(batch, { + const tables = await this.processTables(batch, { + connectionTag: this.connections.connectionTag, schema: this.defaultSchema, name: tableName, objectId: tableName, replicaIdColumns: [{ name: '_id' }] }); - resolved.push(table); + resolved.push(...tables); } return resolved; } - private async getOrResolveTable( + private async getOrResolveTables( batch: storage.BucketStorageBatch, tableName: string, snapshotCursor: string - ): Promise { + ): Promise { + if (!this.isTableSelectedBySyncRules(tableName)) { + return null; + } + const descriptor: SourceEntityDescriptor = { schema: this.defaultSchema, + connectionTag: this.connections.connectionTag, name: tableName, objectId: tableName, replicaIdColumns: [{ name: '_id' }] }; - const existing = this.relationCache.get(descriptor); + const existing = this.relationCache.getAll(descriptor); if (existing) { return existing; } - if (!this.isTableSelectedBySyncRules(tableName)) { - return null; - } - - let table = await this.processTable(batch, descriptor); - if (!table.snapshotComplete && table.syncAny) { + let tables = await this.processTables(batch, descriptor); + const snapshotCandidates = tables.filter((table) => !table.snapshotComplete && table.syncAny); + if (snapshotCandidates.length > 0) { this.logger.info( - `New table discovered while streaming: [${table.qualifiedName}], applying deltas without snapshot` + `New table discovered while streaming: [${descriptor.schema}.${descriptor.name}], applying deltas without snapshot` ); - [table] = await batch.markTableSnapshotDone([table], parseConvexLsn(snapshotCursor)); - this.relationCache.update(table); + const doneTables = await batch.markTableSnapshotDone(snapshotCandidates, parseConvexLsn(snapshotCursor)); + const doneTableById = new Map(doneTables.map((table) => [table.id, table])); + tables = tables.map((table) => doneTableById.get(table.id) ?? table); + this.relationCache.updateAll(descriptor, tables); } - return table; + return tables; } private isTableSelectedBySyncRules(tableName: string): boolean { @@ -569,24 +588,21 @@ export class ConvexStream { return false; } - private async processTable( + private async processTables( batch: storage.BucketStorageBatch, descriptor: SourceEntityDescriptor - ): Promise { - const resolved = await this.storage.resolveTable({ - group_id: this.storage.group_id, + ): Promise { + const resolved = await batch.resolveTables({ connection_id: Number.parseInt(this.connections.connectionId) || 1, - connection_tag: this.connections.connectionTag, - entity_descriptor: descriptor, - sync_rules: this.syncRules + source: descriptor }); if (resolved.dropTables.length > 0) { await batch.drop(resolved.dropTables); } - this.relationCache.update(resolved.table); - return resolved.table; + this.relationCache.updateAll(descriptor, resolved.tables); + return resolved.tables; } private async resolveTablePattern(tablePattern: TablePattern): Promise { @@ -657,7 +673,8 @@ export class ConvexStream { } function getCacheIdentifier(source: SourceEntityDescriptor | SourceTable): string { - return `${source.schema}.${source.name}`; + const connectionTag = source instanceof SourceTable ? source.ref.connectionTag : source.connectionTag; + return `${connectionTag}.${source.schema}.${source.name}`; } function readTableName(change: ConvexRawDocument): string | null { diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 6563df9ee..464bd2dd0 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -52,12 +52,16 @@ function createFakeStorage(options?: { const table = new SourceTable({ id: `${nextTableId++}`, - connectionTag: 'default', + ref: { + connectionTag: 'default', + schema: 'convex', + name + }, objectId: name, - schema: 'convex', - name, replicaIdColumns: [{ name: '_id' }], - snapshotComplete: tableOptions?.snapshotComplete ?? false + snapshotComplete: tableOptions?.snapshotComplete ?? false, + bucketDataSources: [], + parameterLookupSources: [] }); if (tableOptions?.snapshotStatus) { table.snapshotStatus = { @@ -130,12 +134,36 @@ function createFakeStorage(options?: { } }; + const syncRules = { + getSourceTables: () => options?.sourcePatterns ?? [new TablePattern('convex', 'users')], + applyRowContext: (row: Record) => row, + getMatchingSources: () => ({ + bucketDataSources: [], + parameterLookupSources: [] + }), + tableTriggersEvent: () => false + }; + + for (const sourceTable of tables.values()) { + sourceTable.syncData = true; + sourceTable.syncParameters = false; + sourceTable.syncEvent = false; + } + + batch.resolveTables = vi.fn(async ({ source }: any) => { + const resolvedTable = getOrCreateTable(source.name); + resolvedTable.syncData = true; + resolvedTable.syncParameters = false; + resolvedTable.syncEvent = false; + return { + tables: [resolvedTable], + dropTables: [] + }; + }); + const storage = { group_id: 1, - getParsedSyncRules: () => ({ - getSourceTables: () => options?.sourcePatterns ?? [new TablePattern('convex', 'users')], - applyRowContext: (row: Record) => row - }), + getParsedSyncRules: () => syncRules, async getStatus() { return { active: true, @@ -143,20 +171,18 @@ function createFakeStorage(options?: { checkpoint_lsn: options?.snapshotDone ? parseConvexLsn(CURSOR_100) : null, snapshot_lsn: options?.snapshotLsn ?? null }; - }, + } + }; + Object.assign(storage, { clear: vi.fn(async () => undefined), populatePersistentChecksumCache: vi.fn(async () => ({ buckets: 0 })), - resolveTable: vi.fn(async ({ entity_descriptor }: any) => ({ - table: getOrCreateTable(entity_descriptor.name), - dropTables: [] - })), createWriter: vi.fn(async (_options: any) => batch), startBatch: vi.fn(async (_options: any, callback: (batch: any) => Promise) => { await callback(batch); return { flushed_op: 1n }; }), reportError: vi.fn(async () => undefined) - }; + }); return { storage, From 93710001a7e484344ff86a0ff8111c5d2ce2cfc4 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 28 May 2026 09:04:37 +0200 Subject: [PATCH 77/86] use storage logger for replication job --- modules/module-convex/src/replication/ConvexReplicationJob.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/module-convex/src/replication/ConvexReplicationJob.ts b/modules/module-convex/src/replication/ConvexReplicationJob.ts index cc77b5265..e446eb349 100644 --- a/modules/module-convex/src/replication/ConvexReplicationJob.ts +++ b/modules/module-convex/src/replication/ConvexReplicationJob.ts @@ -1,4 +1,4 @@ -import { container, logger as defaultLogger } from '@powersync/lib-services-framework'; +import { container } from '@powersync/lib-services-framework'; import { replication } from '@powersync/service-core'; import { ConvexConnectionManagerFactory } from './ConvexConnectionManagerFactory.js'; import { ConvexCursorExpiredError, ConvexStream } from './ConvexStream.js'; @@ -14,7 +14,7 @@ export class ConvexReplicationJob extends replication.AbstractReplicationJob { constructor(options: ConvexReplicationJobOptions) { super(options); this.connectionFactory = options.connectionFactory; - this.logger = defaultLogger.child({ prefix: `[powersync_${this.options.storage.group_id}] ` }); + this.logger = options.storage.logger; } async keepAlive() { From 5f7927a44c79774627a7cda3a28c062345874f9e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 28 May 2026 09:33:41 +0200 Subject: [PATCH 78/86] cleanup document deltas cursor casting and table resolving --- .../src/replication/ConvexStream.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index db42069f9..91f89f7d0 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -149,13 +149,13 @@ export class ConvexStream { // Resolve source tables up-front to warm table metadata and sync-rule matching. await this.resolveAllSourceTables(batch); - let cursor = BigInt(resumeFromLsn); + let cursor: string = resumeFromLsn; let lastTransactionTimestamp: bigint | null = null; while (!this.abortSignal.aborted) { const page = await this.connections.client .documentDeltas({ - cursor: cursor.toString(), + cursor: cursor, signal: this.abortSignal }) .catch((error) => { @@ -165,7 +165,8 @@ export class ConvexStream { throw error; }); - const nextCursor = page.cursor; + // We receive the cursor as a bigint, but we track it as a string + const nextCursor = page.cursor.toString(); const pageLsn = parseConvexLsn(nextCursor); let changesInPage = 0; @@ -202,8 +203,8 @@ export class ConvexStream { } lastTransactionTimestamp = transactionTimestamp; - const tables = await this.getOrResolveTables(batch, tableName, nextCursor.toString()); - if (tables == null || tables.length == 0) { + const tables = await this.getOrResolveTables(batch, tableName, pageLsn); + if (tables.length == 0) { continue; } @@ -533,10 +534,10 @@ export class ConvexStream { private async getOrResolveTables( batch: storage.BucketStorageBatch, tableName: string, - snapshotCursor: string - ): Promise { + snapshotLSN: string + ): Promise { if (!this.isTableSelectedBySyncRules(tableName)) { - return null; + return []; } const descriptor: SourceEntityDescriptor = { @@ -558,7 +559,7 @@ export class ConvexStream { this.logger.info( `New table discovered while streaming: [${descriptor.schema}.${descriptor.name}], applying deltas without snapshot` ); - const doneTables = await batch.markTableSnapshotDone(snapshotCandidates, parseConvexLsn(snapshotCursor)); + const doneTables = await batch.markTableSnapshotDone(snapshotCandidates, snapshotLSN); const doneTableById = new Map(doneTables.map((table) => [table.id, table])); tables = tables.map((table) => doneTableById.get(table.id) ?? table); this.relationCache.updateAll(descriptor, tables); From 64cd807d8b4db41cb8324134eeadcb64a087eaca Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 28 May 2026 09:38:12 +0200 Subject: [PATCH 79/86] cleanup connection id logic - match CDC stream implementation --- .../src/replication/ConvexStream.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 91f89f7d0..6df11a2a6 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -86,6 +86,22 @@ export class ConvexStream { return this.abortSignal.aborted; } + get connectionId() { + const { connectionId } = this.connections; + // Default to 1 if not set + if (!connectionId) { + return 1; + } + /** + * This is often `"default"` (string) which will parse to `NaN` + */ + const parsed = Number.parseInt(connectionId); + if (isNaN(parsed)) { + return 1; + } + return parsed; + } + async replicate() { try { this.initialSnapshotPromise = this.initReplication(); @@ -594,7 +610,7 @@ export class ConvexStream { descriptor: SourceEntityDescriptor ): Promise { const resolved = await batch.resolveTables({ - connection_id: Number.parseInt(this.connections.connectionId) || 1, + connection_id: this.connectionId, source: descriptor }); From 1a746822cf38f0ce69777a17611a6a1fc2e705db Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 28 May 2026 10:54:24 +0200 Subject: [PATCH 80/86] remove duplicate Test context implemenation and duplicate slow tests --- .../module-convex/test/src/slow_tests.test.ts | 92 -------- modules/module-convex/test/src/util.ts | 204 ------------------ 2 files changed, 296 deletions(-) delete mode 100644 modules/module-convex/test/src/slow_tests.test.ts delete mode 100644 modules/module-convex/test/src/util.ts diff --git a/modules/module-convex/test/src/slow_tests.test.ts b/modules/module-convex/test/src/slow_tests.test.ts deleted file mode 100644 index 87a13dcf4..000000000 --- a/modules/module-convex/test/src/slow_tests.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { describe, expect, test } from 'vitest'; - -import { env } from './env.js'; -import { ConvexStreamTestContext, INITIALIZED_MONGO_STORAGE_FACTORY, makeConvexConnectionManager } from './util.js'; - -describe.runIf(env.SLOW_TESTS && !!env.CONVEX_DEPLOY_KEY)('convex slow tests', { timeout: 120_000 }, function () { - test('connects to Convex and lists table schemas', async () => { - await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); - const schemas = await context.client.getJsonSchemas(); - - expect(schemas.tables.length).toBeGreaterThan(0); - - const tableNames = schemas.tables.map((t) => t.tableName); - expect(tableNames).toContain('lists'); - }); - - test('list_snapshot returns data from Convex', async () => { - const mgr = makeConvexConnectionManager(); - const page = await mgr.client.listSnapshot({ tableName: 'lists' }); - - expect(page.snapshot).toBeDefined(); - // If this fails, seed data first: - // curl -X POST http://127.0.0.1:3210/api/mutation \ - // -H 'Content-Type: application/json' \ - // -H 'Authorization: Convex ' \ - // -d '{"path":"seed:seedLists","args":{"count":10},"format":"json"}' - expect(page.values.length).toBeGreaterThan(0); - - await mgr.end(); - }); - - test('snapshot replicates existing data into buckets', async () => { - await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); - - await context.updateSyncRules(` -bucket_definitions: - global: - data: - - SELECT _id as id, name FROM "lists" - `); - - await context.replicateSnapshot(); - await context.markSnapshotQueryable(); - - const checksum = await context.getChecksum('global[]'); - expect(checksum).toBeDefined(); - expect(checksum!.count).toBeGreaterThan(0); - }); - - test('snapshot replicates with wildcard table pattern', async () => { - await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); - await context.updateSyncRules(` -bucket_definitions: - global: - data: - - SELECT _id as id, * FROM "lists" - `); - - await context.replicateSnapshot(); - await context.markSnapshotQueryable(); - - const data = await context.getBucketData('global[]'); - expect(data.length).toBeGreaterThan(0); - }); - - test('delta streaming starts from snapshot cursor', async () => { - await using context = await ConvexStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY); - await context.updateSyncRules(` -bucket_definitions: - global: - data: - - SELECT _id as id, name FROM "lists" - `); - - await context.replicateSnapshot(); - - // Start streaming in background; it should poll deltas without error. - const streamPromise = context.startStreaming(); - - // Give delta polling a couple of cycles to confirm it doesn't crash. - await new Promise((resolve) => setTimeout(resolve, 2_000)); - - // Abort streaming gracefully. - context.abort(); - await streamPromise.catch(() => {}); - - // Checkpoint should still be valid after streaming ran. - const checksum = await context.getChecksum('global[]'); - expect(checksum).toBeDefined(); - expect(checksum!.count).toBeGreaterThan(0); - }); -}); diff --git a/modules/module-convex/test/src/util.ts b/modules/module-convex/test/src/util.ts deleted file mode 100644 index 9870e5988..000000000 --- a/modules/module-convex/test/src/util.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { - BucketStorageFactory, - createCoreReplicationMetrics, - initializeCoreReplicationMetrics, - InternalOpId, - OplogEntry, - storage, - SyncRulesBucketStorage, - updateSyncRulesFromYaml -} from '@powersync/service-core'; -import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; -import * as mongo_storage from '@powersync/service-module-mongodb-storage'; - -import { ZERO_LSN } from '@module/common/ConvexLSN.js'; -import { ConvexConnectionManager } from '@module/replication/ConvexConnectionManager.js'; -import { ConvexStream, ConvexStreamOptions } from '@module/replication/ConvexStream.js'; -import { normalizeConnectionConfig } from '@module/types/types.js'; -import { env } from './env.js'; - -export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoTestStorageFactoryGenerator({ - url: env.MONGO_TEST_URL, - isCI: env.CI -}); - -export function makeConvexConnectionManager() { - if (!env.CONVEX_DEPLOY_KEY) { - throw new Error('CONVEX_DEPLOY_KEY is required for slow tests'); - } - - const rawConfig = { - type: 'convex' as const, - deployment_url: env.CONVEX_URL, - deploy_key: env.CONVEX_DEPLOY_KEY, - polling_interval_ms: 200 - }; - - const config = { ...rawConfig, ...normalizeConnectionConfig(rawConfig) }; - - return new ConvexConnectionManager(config); -} - -export class ConvexStreamTestContext { - private _stream?: ConvexStream; - private abortController = new AbortController(); - private streamPromise?: Promise; - private syncRulesContent?: storage.PersistedSyncRulesContent; - public storage?: SyncRulesBucketStorage; - - static async open(factoryOrConfig: storage.TestStorageFactory | storage.TestStorageConfig) { - const factory = typeof factoryOrConfig === 'function' ? factoryOrConfig : factoryOrConfig.factory; - const f = await factory({}); - const connectionManager = makeConvexConnectionManager(); - return new ConvexStreamTestContext(f, connectionManager); - } - - constructor( - public factory: BucketStorageFactory, - public connectionManager: ConvexConnectionManager - ) { - createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); - initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); - } - - abort() { - this.abortController.abort(); - } - - async dispose() { - this.abort(); - await this.streamPromise?.catch(() => {}); - await this.factory[Symbol.asyncDispose](); - await this.connectionManager.end(); - } - - async [Symbol.asyncDispose]() { - await this.dispose(); - } - - get client() { - return this.connectionManager.client; - } - - private requireSyncRulesContent(): storage.PersistedSyncRulesContent { - if (this.syncRulesContent == null) { - throw new Error('Call updateSyncRules() first'); - } - return this.syncRulesContent; - } - - async updateSyncRules(content: string) { - const syncRules = await this.factory.updateSyncRules(updateSyncRulesFromYaml(content, { validate: true })); - this.syncRulesContent = syncRules; - this.storage = this.factory.getInstance(syncRules); - return this.storage!; - } - - get stream(): ConvexStream { - if (this.storage == null) { - throw new Error('Call updateSyncRules() first'); - } - if (this._stream) { - return this._stream; - } - - const options: ConvexStreamOptions = { - storage: this.storage, - metrics: METRICS_HELPER.metricsEngine, - connections: this.connectionManager, - abortSignal: this.abortController.signal - }; - this._stream = new ConvexStream(options); - return this._stream; - } - - async replicateSnapshot() { - await this.stream.initReplication(); - } - - startStreaming() { - this.streamPromise = this.stream.streamChanges(); - return this.streamPromise; - } - - async waitForCheckpoint(options?: { timeout?: number }): Promise { - const timeout = options?.timeout ?? 30_000; - const start = Date.now(); - - while (Date.now() - start < timeout) { - const activeStorage = await this.factory.getActiveStorage(); - const cp = await activeStorage?.getCheckpoint(); - if (cp != null && cp.lsn) { - return cp.checkpoint; - } - await new Promise((resolve) => setTimeout(resolve, 100)); - } - - throw new Error(`Timeout waiting for checkpoint after ${timeout}ms`); - } - - async getBucketData(bucket: string, options?: { timeout?: number }): Promise { - const checkpoint = await this.waitForCheckpoint(options); - const syncRules = this.requireSyncRulesContent(); - let request = test_utils.bucketRequest(syncRules, bucket, 0n); - let data: OplogEntry[] = []; - - while (true) { - const batch = this.storage!.getBucketDataBatch(checkpoint, [request]); - const batches = await test_utils.fromAsync(batch); - data = data.concat(batches[0]?.chunkData.data ?? []); - if (batches.length == 0 || !batches[0]!.chunkData.has_more) { - break; - } - request = test_utils.bucketRequest(syncRules, bucket, BigInt(batches[0]!.chunkData.next_after)); - } - - return data; - } - - async getChecksums(buckets: string[], options?: { timeout?: number }) { - const checkpoint = await this.waitForCheckpoint(options); - const syncRules = this.requireSyncRulesContent(); - return this.storage!.getChecksums( - checkpoint, - buckets.map((bucket) => { - const request = test_utils.bucketRequest(syncRules, bucket, 0n); - return { - bucket: request.bucket, - source: request.source - }; - }) - ); - } - - async getChecksum(bucket: string, options?: { timeout?: number }) { - const checkpoint = await this.waitForCheckpoint(options); - const syncRules = this.requireSyncRulesContent(); - const request = test_utils.bucketRequest(syncRules, bucket, 0n); - const map = await this.storage!.getChecksums(checkpoint, [{ bucket: request.bucket, source: request.source }]); - return map.get(request.bucket); - } - - /** - * After snapshot, manually advance the checkpoint so bucket data is queryable - * without requiring full delta streaming to catch up. - */ - async markSnapshotQueryable() { - const status = await this.storage!.getStatus(); - if (!status.checkpoint_lsn) { - throw new Error('No checkpoint LSN available - run snapshot first'); - } - - await this.storage!.startBatch( - { - defaultSchema: 'convex', - zeroLSN: ZERO_LSN, - storeCurrentData: false, - skipExistingRows: false - }, - async (batch) => { - await batch.keepalive(status.checkpoint_lsn!); - } - ); - } -} From db2548dc0c20468a204445a57cf1dee0c0a67a1f Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 28 May 2026 10:54:46 +0200 Subject: [PATCH 81/86] update keepalive logic --- modules/module-convex/src/replication/ConvexStream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 6df11a2a6..21bd84e48 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -279,7 +279,7 @@ export class ConvexStream { this.replicationLag.clearUncommittedChange(); } this.lastKeepaliveAt = Date.now(); - } else if (nextCursor != cursor && Date.now() - this.lastKeepaliveAt > 60_000) { + } else if (nextCursor == cursor && Date.now() - this.lastKeepaliveAt > 60_000) { const { checkpointBlocked } = await batch.keepalive(pageLsn); if (!checkpointBlocked) { this.replicationLag.clearUncommittedChange(); From 21e1d1dc826f55484fe77759426ccb3c30681cdc Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 28 May 2026 11:12:18 +0200 Subject: [PATCH 82/86] remove convex type field from normalized config --- modules/module-convex/src/module/ConvexModule.ts | 5 +---- .../src/replication/ConvexConnectionManager.ts | 4 ++-- modules/module-convex/src/types/types.ts | 9 +++++++-- .../ConvexRouteAPIAdapter.integration.test.ts | 9 +-------- .../test/src/test-utils/ConvexStreamTestContext.ts | 5 ++--- modules/module-convex/test/src/test-utils/util.ts | 2 +- 6 files changed, 14 insertions(+), 20 deletions(-) diff --git a/modules/module-convex/src/module/ConvexModule.ts b/modules/module-convex/src/module/ConvexModule.ts index 69b1abe06..bbbd0f777 100644 --- a/modules/module-convex/src/module/ConvexModule.ts +++ b/modules/module-convex/src/module/ConvexModule.ts @@ -45,10 +45,7 @@ export class ConvexModule extends replication.ReplicationModule { diff --git a/modules/module-convex/src/replication/ConvexConnectionManager.ts b/modules/module-convex/src/replication/ConvexConnectionManager.ts index 2749e63ad..369b83d32 100644 --- a/modules/module-convex/src/replication/ConvexConnectionManager.ts +++ b/modules/module-convex/src/replication/ConvexConnectionManager.ts @@ -1,7 +1,7 @@ import { BaseObserver } from '@powersync/lib-services-framework'; import { DEFAULT_TAG } from '@powersync/service-sync-rules'; import { ConvexApiClient } from '../client/ConvexApiClient.js'; -import { ResolvedConvexConnectionConfig } from '../types/types.js'; +import { NormalizedConvexConnectionConfig } from '../types/types.js'; export interface ConvexConnectionManagerListener { onEnded?: () => void; @@ -13,7 +13,7 @@ export class ConvexConnectionManager extends BaseObserver { const start = Date.now(); - const api = new ConvexRouteAPIAdapter(this.connectionManager.config); - const lsn = await api.createReplicationHead(async (lsn) => lsn); + const lsn = await this.connectionManager.client.getHeadCursor(); + await this.connectionManager.client.createWriteCheckpointMarker(); // This old API needs a persisted checkpoint id. // Since we don't use LSNs anymore, the only way to get that is to wait. diff --git a/modules/module-convex/test/src/test-utils/util.ts b/modules/module-convex/test/src/test-utils/util.ts index 91e7785be..5bc275d4b 100644 --- a/modules/module-convex/test/src/test-utils/util.ts +++ b/modules/module-convex/test/src/test-utils/util.ts @@ -76,7 +76,7 @@ export const RAW_TEST_CONNECTION_OPTIONS: types.ConvexConnectionConfig = { deployment_url: env.CONVEX_URL } as const; -export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig(RAW_TEST_CONNECTION_OPTIONS); +export const TEST_CONNECTION_OPTIONS = types.resolveConvexConnectionConfig(RAW_TEST_CONNECTION_OPTIONS); export function connectConvex(): TestConvexConnection { return { From fab44c1da64278eda39e0e3c3cbfac8f9178c462 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 28 May 2026 11:37:25 +0200 Subject: [PATCH 83/86] update mocked test --- .../test/src/ConvexStream.test.ts | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/modules/module-convex/test/src/ConvexStream.test.ts b/modules/module-convex/test/src/ConvexStream.test.ts index 464bd2dd0..aea76e9ef 100644 --- a/modules/module-convex/test/src/ConvexStream.test.ts +++ b/modules/module-convex/test/src/ConvexStream.test.ts @@ -608,7 +608,7 @@ describe('ConvexStream', () => { expect(context.tables.get('projects_archive')?.snapshotComplete).toBe(true); }); - it('keeps alive immediately when only checkpoint marker rows are streamed', async () => { + it('keeps alive on idle startup and immediately when only checkpoint marker rows are streamed', async () => { const context = createFakeStorage({ snapshotDone: true, resumeFromLsn: parseConvexLsn(CURSOR_100) @@ -635,16 +635,24 @@ describe('ConvexStream', () => { documentDeltas: async () => { calls += 1; if (calls == 1) { + // Convex does not advance the cursor when there are no deltas. + // Since this is the first poll after startup, the periodic + // keepalive path still runs immediately because lastKeepaliveAt + // starts at 0. return { - cursor: CURSOR_101, - hasMore: true, + cursor: CURSOR_100, + hasMore: false, values: [] }; } setTimeout(() => abortController.abort(), 0); + // A checkpoint marker is a real Convex delta, so the cursor advances. + // The row is ignored as replicated data, but marker-only pages must + // still advance the storage checkpoint immediately so managed write + // checkpoints can become visible without waiting for the 60s throttle. return { - cursor: CURSOR_102, + cursor: CURSOR_101, hasMore: false, values: [{ _table: 'powersync_checkpoints', _id: 'cp1' }] }; @@ -657,6 +665,9 @@ describe('ConvexStream', () => { expect(context.saves.length).toBe(0); expect(context.commits.length).toBe(0); - expect(context.keepalives).toEqual([parseConvexLsn(CURSOR_101), parseConvexLsn(CURSOR_102)]); + // The idle startup page and marker-only page both keep the checkpoint moving, + // but for different reasons: the first uses the same-cursor idle keepalive + // path, and the second uses the marker-only immediate keepalive path. + expect(context.keepalives).toEqual([parseConvexLsn(CURSOR_100), parseConvexLsn(CURSOR_101)]); }); }); From 86867b3ce57d1d4cc18eaf21ff21e9da140ccdf4 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 28 May 2026 16:46:13 +0200 Subject: [PATCH 84/86] implement and share checkSourceConfiguration --- .../src/api/ConvexRouteAPIAdapter.ts | 7 +- .../module-convex/src/module/ConvexModule.ts | 80 ++----------- .../replication/check-source-configuration.ts | 113 ++++++++++++++++++ 3 files changed, 128 insertions(+), 72 deletions(-) create mode 100644 modules/module-convex/src/replication/check-source-configuration.ts diff --git a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts index f9d6179a1..5c0dccaa7 100644 --- a/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts +++ b/modules/module-convex/src/api/ConvexRouteAPIAdapter.ts @@ -5,6 +5,7 @@ import { isConvexCheckpointTable } from '../common/ConvexCheckpoints.js'; import { parseConvexLsn } from '../common/ConvexLSN.js'; import { extractProperties, jsonSchemaToSQLiteType, readConvexFieldJsonType } from '../common/convex-to-sqlite.js'; import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; +import { checkSourceConfiguration } from '../replication/check-source-configuration.js'; import * as types from '../types/types.js'; export class ConvexRouteAPIAdapter implements api.RouteAPI { @@ -25,11 +26,11 @@ export class ConvexRouteAPIAdapter implements api.RouteAPI { }; try { - await this.connectionManager.client.getJsonSchemas(); + const { connected, errors } = await checkSourceConfiguration(this.config, { readOnly: true }); return { ...base, - connected: true, - errors: [] + connected, + errors: errors.map((e) => ({ level: 'fatal', message: e })) }; } catch (error) { return { diff --git a/modules/module-convex/src/module/ConvexModule.ts b/modules/module-convex/src/module/ConvexModule.ts index bbbd0f777..67e606726 100644 --- a/modules/module-convex/src/module/ConvexModule.ts +++ b/modules/module-convex/src/module/ConvexModule.ts @@ -1,3 +1,4 @@ +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; import { api, ConfigurationFileSyncRulesProvider, @@ -7,8 +8,7 @@ import { TearDownOptions } from '@powersync/service-core'; import { ConvexRouteAPIAdapter } from '../api/ConvexRouteAPIAdapter.js'; -import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; -import { ConvexConnectionManager } from '../replication/ConvexConnectionManager.js'; +import { checkSourceConfiguration } from '../replication/check-source-configuration.js'; import { ConvexConnectionManagerFactory } from '../replication/ConvexConnectionManagerFactory.js'; import { ConvexErrorRateLimiter } from '../replication/ConvexErrorRateLimiter.js'; import { ConvexReplicator } from '../replication/ConvexReplicator.js'; @@ -59,75 +59,17 @@ export class ConvexModule extends replication.ReplicationModule { - const connectionManager = new ConvexConnectionManager(normalizedConfig); - const missingMutatorErrorFragment = ` -Define a mutator for the PowerSync service to use - -convex/powersync_checkpoints.ts -\`\`\`TypeScript -import { mutation } from './_generated/server.js'; - -export const createCheckpoint = mutation({ - args: {}, - handler: async (ctx) => { - const existing = await ctx.db.query('powersync_checkpoints').first(); - - if (existing) { - await ctx.db.patch(existing._id, { last_updated: Date.now() }); - } else { - await ctx.db.insert('powersync_checkpoints', { last_updated: Date.now() }); - } - } -}); -\`\`\` - `; - try { - // Check if the database is reachable by fetching the schema. - const schema = await connectionManager.client.getJsonSchemas().catch((error) => { - throw new Error('Could not fetch Convex schema for provided connection configuration.', { - cause: error - }); + const { connected, errors } = await checkSourceConfiguration(normalizedConfig); + /** + * The mutation check can report configuration errors even after a successful + * schema fetch, so treat either disconnected or errored states as failures. + * */ + if (!connected || errors.length > 0) { + throw new ServiceError({ + code: ErrorCode.PSYNC_R0001, + description: errors.join('\n') }); - - if (!schema.tables.find((table) => table.tableName == CONVEX_CHECKPOINT_TABLE)) { - throw new Error(` -Could not find the ${CONVEX_CHECKPOINT_TABLE} table in the schema. - -Define the ${CONVEX_CHECKPOINT_TABLE} table in the Convex schema: - -convex/schema.ts -\`\`\`TypeScript -//... - -export default defineSchema({ - // ... your other tables - - powersync_checkpoints: defineTable({ - last_updated: v.float64() - }) -}); -\`\`\` - -${missingMutatorErrorFragment} -`); - } - - // Check that the PowerSync checkpoint table and mutation are deployed. - // It should be safe to update this table at any point. We only use it for emiting a replication event. - await connectionManager.client.createWriteCheckpointMarker().catch((error) => { - throw new Error( - ` -Could not call the createCheckpoint mutator. - -${missingMutatorErrorFragment} - `, - { cause: error } - ); - }); - } finally { - await connectionManager.end(); } - return { connectionDescription: normalizedConfig.deployment_url }; diff --git a/modules/module-convex/src/replication/check-source-configuration.ts b/modules/module-convex/src/replication/check-source-configuration.ts new file mode 100644 index 000000000..7f8ed7ed6 --- /dev/null +++ b/modules/module-convex/src/replication/check-source-configuration.ts @@ -0,0 +1,113 @@ +import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; +import { NormalizedConvexConnectionConfig } from '../types/types.js'; +import { ConvexConnectionManager } from './ConvexConnectionManager.js'; + +const MISSING_MUTATOR_FRAGMENT = ` +Define a mutator for the PowerSync service to use + +convex/powersync_checkpoints.ts +\`\`\`TypeScript +import { mutation } from './_generated/server.js'; + +export const createCheckpoint = mutation({ + args: {}, + handler: async (ctx) => { + const existing = await ctx.db.query('powersync_checkpoints').first(); + + if (existing) { + await ctx.db.patch(existing._id, { last_updated: Date.now() }); + } else { + await ctx.db.insert('powersync_checkpoints', { last_updated: Date.now() }); + } + } +}); +\`\`\``.trim(); + +export type ConvexSourceConfigurationTestResult = { + connected: boolean; + errors: string[]; +}; + +export type CheckSourceConfigurationOptions = { + /** + * Avoid calling Convex mutations while checking the source configuration. + * + * This is used by diagnostics and connection-status routes, which may be + * called often. Creating write checkpoint markers there is safe, but + * unnecessary and can create noisy replication activity on every poll. + */ + readOnly?: boolean; +}; + +export async function checkSourceConfiguration( + normalizedConfig: NormalizedConvexConnectionConfig, + options: CheckSourceConfigurationOptions = {} +): Promise { + const connectionManager = new ConvexConnectionManager(normalizedConfig); + const errors: string[] = []; + let connected = false; + try { + // Check if the database is reachable by fetching the schema. + const schema = await connectionManager.client.getJsonSchemas().catch((error) => { + const message = error instanceof Error ? error.message : `${error}`; + errors.push( + `Could not fetch Convex schema for provided connection configuration. Error: ${message || 'unknown'}` + ); + }); + + // We could connect if we got a schema response + connected = !!schema; + if (!schema) { + return { + connected, + errors + }; + } + + const hasCheckpointTable = schema.tables.some((table) => table.tableName == CONVEX_CHECKPOINT_TABLE); + if (!hasCheckpointTable) { + errors.push( + ` +Could not find the ${CONVEX_CHECKPOINT_TABLE} table in the schema. + +Define the ${CONVEX_CHECKPOINT_TABLE} table in the Convex schema: + +convex/schema.ts +\`\`\`TypeScript +//... + +export default defineSchema({ + // ... your other tables + + powersync_checkpoints: defineTable({ + last_updated: v.float64() + }) +}); +\`\`\` + +${MISSING_MUTATOR_FRAGMENT} + `.trim() + ); + } + + // Check that the PowerSync checkpoint table and mutation are deployed. + // It should be safe to update this table at any point. We only use it for emitting a replication event. + if (hasCheckpointTable && !options.readOnly) { + await connectionManager.client.createWriteCheckpointMarker().catch((error) => { + const message = error instanceof Error ? error.message : `${error}`; + errors.push( + ` +Could not call the createCheckpoint mutator. Error ${message || 'unknown'} + +${MISSING_MUTATOR_FRAGMENT}`.trim() + ); + }); + } + } finally { + await connectionManager.end(); + } + return { + connected, + errors + }; +} From 3d465634fc4af2c1eeb5b2b9a460467c4d9b3814 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 1 Jun 2026 09:31:48 +0200 Subject: [PATCH 85/86] update to HydratedSyncConfig --- .../src/replication/ConvexStream.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/module-convex/src/replication/ConvexStream.ts b/modules/module-convex/src/replication/ConvexStream.ts index 21bd84e48..a33b448b7 100644 --- a/modules/module-convex/src/replication/ConvexStream.ts +++ b/modules/module-convex/src/replication/ConvexStream.ts @@ -16,7 +16,7 @@ import { SourceTable, storage } from '@powersync/service-core'; -import { HydratedSyncRules, TablePattern } from '@powersync/service-sync-rules'; +import { HydratedSyncConfig, TablePattern } from '@powersync/service-sync-rules'; import { ReplicationMetric } from '@powersync/service-types'; import { setTimeout as delay } from 'timers/promises'; import { ConvexListSnapshotResult, ConvexRawDocument } from '../client/ConvexAPITypes.js'; @@ -44,7 +44,7 @@ export class ConvexCursorExpiredError extends DatabaseConnectionError { export class ConvexStream { private readonly storage: storage.SyncRulesBucketStorage; private readonly metrics: MetricsEngine; - private readonly syncRules: HydratedSyncRules; + private readonly syncConfig: HydratedSyncConfig; private readonly logger: Logger; private readonly relationCache = new RelationCache(getCacheIdentifier); @@ -58,7 +58,7 @@ export class ConvexStream { constructor(private readonly options: ConvexStreamOptions) { this.storage = options.storage; this.metrics = options.metrics; - this.syncRules = options.storage.getParsedSyncRules({ defaultSchema: options.connections.schema }); + this.syncConfig = options.storage.getParsedSyncRules({ defaultSchema: options.connections.schema }); this.logger = options.logger ?? defaultLogger; } @@ -162,7 +162,7 @@ export class ConvexStream { throw new ReplicationAssertionError(`No LSN found to resume replication from.`); } - // Resolve source tables up-front to warm table metadata and sync-rule matching. + // Resolve source tables up-front to warm table metadata and sync-config matching. await this.resolveAllSourceTables(batch); let cursor: string = resumeFromLsn; @@ -495,7 +495,7 @@ export class ConvexStream { } private async resolveAllSourceTables(batch: storage.BucketStorageBatch): Promise { - const sourceTablePatterns = this.syncRules.getSourceTables(); + const sourceTablePatterns = this.syncConfig.getSourceTables(); const resolved: SourceTable[] = []; const seenSourceTableIds = new Set(); @@ -552,7 +552,7 @@ export class ConvexStream { tableName: string, snapshotLSN: string ): Promise { - if (!this.isTableSelectedBySyncRules(tableName)) { + if (!this.isTableSelectedBySyncConfig(tableName)) { return []; } @@ -584,8 +584,8 @@ export class ConvexStream { return tables; } - private isTableSelectedBySyncRules(tableName: string): boolean { - for (const sourceTablePattern of this.syncRules.getSourceTables()) { + private isTableSelectedBySyncConfig(tableName: string): boolean { + for (const sourceTablePattern of this.syncConfig.getSourceTables()) { if (sourceTablePattern.connectionTag != this.connections.connectionTag) { continue; } @@ -674,7 +674,7 @@ export class ConvexStream { } private toSqliteRow(change: ConvexRawDocument) { - return this.syncRules.applyRowContext(toSqliteInputRow(change)); + return this.syncConfig.applyRowContext(toSqliteInputRow(change)); } private touch() { From 3f4fb19e44d0b6e7decf515b47673eca5bbe02d0 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Mon, 1 Jun 2026 15:08:09 +0200 Subject: [PATCH 86/86] use http agent for lookup handling --- modules/module-convex/package.json | 1 + .../src/client/ConvexApiClient.ts | 44 +++++++---- .../test/src/ConvexApiClient.test.ts | 76 +++++++++++-------- pnpm-lock.yaml | 3 + 4 files changed, 76 insertions(+), 48 deletions(-) diff --git a/modules/module-convex/package.json b/modules/module-convex/package.json index 6dda9180b..8a8327fe7 100644 --- a/modules/module-convex/package.json +++ b/modules/module-convex/package.json @@ -36,6 +36,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "bson": "^6.10.4", + "node-fetch": "^3.3.2", "ts-codec": "^1.3.0" }, "devDependencies": { diff --git a/modules/module-convex/src/client/ConvexApiClient.ts b/modules/module-convex/src/client/ConvexApiClient.ts index 073013f69..1c22945bb 100644 --- a/modules/module-convex/src/client/ConvexApiClient.ts +++ b/modules/module-convex/src/client/ConvexApiClient.ts @@ -1,4 +1,8 @@ import { JSONBig } from '@powersync/service-jsonbig'; +// Using node-fetch in order to supply a custom agent +import fetch from 'node-fetch'; +import * as http from 'node:http'; +import * as https from 'node:https'; import { setTimeout as delay } from 'timers/promises'; import { CONVEX_CHECKPOINT_TABLE } from '../common/ConvexCheckpoints.js'; import { NormalizedConvexConnectionConfig } from '../types/types.js'; @@ -36,7 +40,11 @@ export class ConvexApiError extends Error { } export class ConvexApiClient { - constructor(private readonly config: NormalizedConvexConnectionConfig) {} + private readonly agent: http.Agent | https.Agent; + + constructor(private readonly config: NormalizedConvexConnectionConfig) { + this.agent = this.resolveAgent(); + } async getJsonSchemas(options?: { signal?: AbortSignal }): Promise { const raw = await this.performTypedGetRequest( @@ -177,9 +185,7 @@ export class ConvexApiClient { const signal = AbortSignal.any(signals); try { - await this.assertHostAllowed(url); - - const response = await fetch(url, { + const response = await fetch(url.toString(), { method, headers: { Authorization: `Convex ${this.config.deploy_key}`, @@ -187,7 +193,8 @@ export class ConvexApiClient { ...extraHeaders }, body: body == null ? undefined : JSON.stringify(options.body), - signal + signal, + agent: this.agent }); const text = await response.text(); @@ -246,19 +253,24 @@ export class ConvexApiClient { } } - private async assertHostAllowed(url: URL): Promise { - if (!this.config.lookup) { - return; + private resolveAgent(): http.Agent | https.Agent { + const deploymentUrl = new URL(this.config.deployment_url); + const options: http.AgentOptions = {}; + + if (this.config.lookup) { + options.lookup = this.config.lookup; } - await new Promise((resolve, reject) => { - this.config.lookup!(url.hostname, {}, (error) => { - if (error) { - reject(error); - } else { - resolve(); - } - }); + switch (deploymentUrl.protocol) { + case 'http:': + return new http.Agent(options); + case 'https:': + return new https.Agent(options); + } + + throw new ConvexApiError({ + message: `Convex deployment_url must use http or https, got ${JSON.stringify(deploymentUrl.protocol)}`, + retryable: false }); } } diff --git a/modules/module-convex/test/src/ConvexApiClient.test.ts b/modules/module-convex/test/src/ConvexApiClient.test.ts index dd0990326..c96266594 100644 --- a/modules/module-convex/test/src/ConvexApiClient.test.ts +++ b/modules/module-convex/test/src/ConvexApiClient.test.ts @@ -3,29 +3,37 @@ import { ConvexListSnapshotResult, RawJsonSchemaResponse } from '@module/client/ import { CONVEX_CHECKPOINT_TABLE } from '@module/common/ConvexCheckpoints.js'; import { normalizeConnectionConfig } from '@module/types/types.js'; import { JSONBig } from '@powersync/service-jsonbig'; +import nodeFetch from 'node-fetch'; +import * as https from 'node:https'; import { afterEach, describe, expect, it, vi } from 'vitest'; +vi.mock('node-fetch', () => ({ + default: vi.fn() +})); + const baseConfig = normalizeConnectionConfig({ type: 'convex', deployment_url: 'https://example.convex.cloud', deploy_key: 'test-key' }); const SNAPSHOT_CURSOR = 1770335566197683000n; +const fetchMock = vi.mocked(nodeFetch); describe('ConvexApiClient', () => { afterEach(() => { vi.useRealTimers(); + fetchMock.mockReset(); vi.restoreAllMocks(); }); it('sends Convex authorization header and format=json', async () => { - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + fetchMock.mockResolvedValue( new Response( JSON.stringify({ users: { type: 'table', properties: { _id: { type: 'string' } } } } satisfies RawJsonSchemaResponse), { status: 200 } - ) + ) as any ); const client = new ConvexApiClient(baseConfig); @@ -33,18 +41,18 @@ describe('ConvexApiClient', () => { expect(result.tables.map((table) => table.tableName)).toEqual(['users']); - const [url, init] = fetchSpy.mock.calls[0]!; + const [url, init] = fetchMock.mock.calls[0]!; expect(String(url)).toContain('/api/json_schemas'); expect(String(url)).toContain('format=json'); expect((init?.headers as Record).Authorization).toBe('Convex test-key'); }); it('preserves high-precision numeric snapshot values', async () => { - vi.spyOn(globalThis, 'fetch').mockResolvedValue( + fetchMock.mockResolvedValue( new Response( '{"values":[],"snapshot":1770335566197682922,"cursor":"{\\"tablet\\":\\"X0yj4Cm7GfuikfsSBm9QCQ\\",\\"id\\":\\"j5700000000000000000000000001qv0\\"}","hasMore":true}', { status: 200 } - ) + ) as any ); const client = new ConvexApiClient(baseConfig); @@ -56,7 +64,7 @@ describe('ConvexApiClient', () => { }); it('sends table_name as snake_case query parameter in list_snapshot', async () => { - const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + fetchMock.mockResolvedValue( new Response( JSONBig.stringify({ snapshot: SNAPSHOT_CURSOR, @@ -65,19 +73,19 @@ describe('ConvexApiClient', () => { values: [] } satisfies ConvexListSnapshotResult), { status: 200 } - ) + ) as any ); const client = new ConvexApiClient(baseConfig); await client.listSnapshot({ tableName: 'lists', snapshot: SNAPSHOT_CURSOR.toString() }); - const url = String(fetchSpy.mock.calls[0]![0]); + const url = String(fetchMock.mock.calls[0]![0]); expect(url).toContain('table_name=lists'); expect(url).not.toContain('tableName=lists'); }); it('marks network failures as retryable', async () => { - vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('fetch failed: ECONNRESET')); + fetchMock.mockRejectedValue(new Error('fetch failed: ECONNRESET')); const client = new ConvexApiClient(baseConfig); @@ -88,13 +96,13 @@ describe('ConvexApiClient', () => { it('uses the configured request timeout', async () => { vi.useFakeTimers(); - vi.spyOn(globalThis, 'fetch').mockImplementation( + fetchMock.mockImplementation( (_url, init) => new Promise((_resolve, reject) => { init?.signal?.addEventListener('abort', () => { - reject(init.signal!.reason); + reject((init.signal as any).reason); }); - }) + }) as any ); const client = new ConvexApiClient({ @@ -112,15 +120,13 @@ describe('ConvexApiClient', () => { }); it('creates write checkpoint markers via mutation', async () => { - const fetchSpy = vi - .spyOn(globalThis, 'fetch') - .mockResolvedValue(new Response(JSON.stringify({ status: 'success' }), { status: 200 })); + fetchMock.mockResolvedValue(new Response(JSON.stringify({ status: 'success' }), { status: 200 }) as any); const client = new ConvexApiClient(baseConfig); await client.createWriteCheckpointMarker(); - expect(fetchSpy).toHaveBeenCalledTimes(1); - const [url, init] = fetchSpy.mock.calls[0]!; + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0]!; expect(String(url)).toContain('/api/mutation'); expect(init?.method).toBe('POST'); expect((init?.headers as Record).Authorization).toBe('Convex test-key'); @@ -132,9 +138,7 @@ describe('ConvexApiClient', () => { }); it('propagates checkpoint write errors directly (no fallback)', async () => { - vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce( - new Response(JSON.stringify({ code: 'SomeError' }), { status: 400 }) - ); + fetchMock.mockResolvedValueOnce(new Response(JSON.stringify({ code: 'SomeError' }), { status: 400 }) as any); const client = new ConvexApiClient(baseConfig); await expect(client.createWriteCheckpointMarker()).rejects.toMatchObject({ @@ -143,10 +147,8 @@ describe('ConvexApiClient', () => { }); }); - it('checks the hostname policy before every Convex API request', async () => { - const fetchSpy = vi - .spyOn(globalThis, 'fetch') - .mockResolvedValue(new Response(JSON.stringify({ status: 'success' }), { status: 200 })); + it('uses an agent with the configured hostname policy for Convex API requests', async () => { + fetchMock.mockResolvedValue(new Response(JSON.stringify({ status: 'success' }), { status: 200 }) as any); const lookup = vi.fn((_hostname: string, _options: any, callback: (error: Error) => void) => { callback(new Error('blocked by reject_ip_ranges')); }) as unknown as import('node:net').LookupFunction; @@ -156,13 +158,23 @@ describe('ConvexApiClient', () => { lookup }); - await expect(client.getJsonSchemas()).rejects.toThrow('blocked by reject_ip_ranges'); - await expect(client.listSnapshot({ tableName: 'lists' })).rejects.toThrow('blocked by reject_ip_ranges'); - await expect(client.documentDeltas({ cursor: SNAPSHOT_CURSOR.toString() })).rejects.toThrow( - 'blocked by reject_ip_ranges' - ); - await expect(client.createWriteCheckpointMarker()).rejects.toThrow('blocked by reject_ip_ranges'); - expect(lookup).toHaveBeenCalledTimes(4); - expect(fetchSpy).not.toHaveBeenCalled(); + await client.createWriteCheckpointMarker(); + + const init = fetchMock.mock.calls[0]![1] as RequestInit & { agent: https.Agent }; + expect(init.agent).toBeInstanceOf(https.Agent); + expect(init.agent.options.lookup).toBe(lookup); + expect(lookup).not.toHaveBeenCalled(); + + await expect( + new Promise((resolve, reject) => { + init.agent.options.lookup!('example.convex.cloud', {}, (error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }) + ).rejects.toThrow('blocked by reject_ip_ranges'); }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 232693f35..4c2d7ad03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -216,6 +216,9 @@ importers: bson: specifier: ^6.10.4 version: 6.10.4 + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 ts-codec: specifier: ^1.3.0 version: 1.3.0