From 4170fa0a233ad1845c8c4c94439c23a3dfb84284 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:12:54 +0800 Subject: [PATCH 01/46] test(gateway): specify nonblocking model refresh triggers Require cold and arbitrarily stale catalog reads to return immediately while background refreshes populate the cache. Cover explicit force-fetch semantics, persistent failure cooldown, concurrent trigger coalescing, generation fencing, and obsolete cache replacement. --- .../data-plane/providers/models-cache_test.ts | 259 ++++++++---------- 1 file changed, 110 insertions(+), 149 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 7cd6d38dc..e5980805f 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -19,9 +19,6 @@ const CACHE_GENERATION = vi.hoisted(() => ({ const aModel = (id: string): ProviderModel => stubProviderModel({ id }); -// The SWR check reads the stored catalog off the provider instance, which the -// registry mirrors from the row that produced it — so a seeded catalog is -// handed to both the repo and the instance here. const stubInstance = ( fetchFn: () => Promise, modelsCache: UpstreamModelsCache | null = null, @@ -40,8 +37,6 @@ const stubInstance = ( modelsFetchIdentity: fetchIdentity, }); -// The cache lives on the upstream row, so every write needs a row to -// land on. const setupRepo = async (): Promise => { const repo = new InMemoryRepo(); initRepo(repo); @@ -78,215 +73,185 @@ const seedCache = async ( const storedCache = async (repo: InMemoryRepo): Promise => (await repo.upstreams.getById(UPSTREAM_ID))?.modelsCache ?? null; +const captureScheduled = () => { + const promises: Promise[] = []; + return { + promises, + scheduler: (promise: Promise): void => { promises.push(promise); }, + }; +}; + beforeEach(() => { + vi.restoreAllMocks(); clearInFlightForTesting(); }); describe('fetchUpstreamModelsCached', () => { - test('cold cache: fetches, stores, returns models', async () => { + test('cold cache returns immediately and refreshes in the background', async () => { const repo = await setupRepo(); - const fetchFn = vi.fn(async () => [aModel('m1')]); + let resolveFetch: ((models: ProviderModel[]) => void) | null = null; + const fetchFn = vi.fn(() => new Promise(resolve => { resolveFetch = resolve; })); + const scheduled = captureScheduled(); const result = await fetchUpstreamModelsCached( stubInstance(fetchFn), - { scheduler: () => {}, fetcher: directFetcher }, + { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); - expect(result.map(m => m.id)).toEqual(['m1']); - expect(fetchFn).toHaveBeenCalledTimes(1); - expect((await storedCache(repo))?.models.map(m => m.id)).toEqual(['m1']); + expect(result).toEqual([]); + expect(scheduled.promises).toHaveLength(1); + await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); + resolveFetch!([aModel('m1')]); + await scheduled.promises[0]; + expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['m1']); }); - test('within SOFT: no fetch, returns stored', async () => { + test('within SOFT returns the stored catalog without scheduling a refresh', async () => { const repo = await setupRepo(); const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 1000, models: [aModel('cached')] }); const fetchFn = vi.fn(async () => [aModel('fresh')]); + const scheduled = captureScheduled(); const result = await fetchUpstreamModelsCached( stubInstance(fetchFn, cache), - { scheduler: () => {}, fetcher: directFetcher }, + { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); - expect(result.map(m => m.id)).toEqual(['cached']); + expect(result.map(model => model.id)).toEqual(['cached']); + expect(scheduled.promises).toEqual([]); expect(fetchFn).not.toHaveBeenCalled(); }); - test('past SOFT within HARD: returns stored + schedules revalidate', async () => { + test('every stale age remains SWR forever', async () => { const repo = await setupRepo(); - const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 20 * 60_000, models: [aModel('stale')] }); - const fetchFn = vi.fn(async () => [aModel('fresh')]); - let scheduled: Promise | null = null; + const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 365 * 24 * 60 * 60_000, models: [aModel('stale')] }); + let resolveFetch: ((models: ProviderModel[]) => void) | null = null; + const fetchFn = vi.fn(() => new Promise(resolve => { resolveFetch = resolve; })); + const scheduled = captureScheduled(); const result = await fetchUpstreamModelsCached( stubInstance(fetchFn, cache), - { scheduler: p => { scheduled = p; }, fetcher: directFetcher }, + { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); - expect(result.map(m => m.id)).toEqual(['stale']); - expect(fetchFn).toHaveBeenCalledTimes(1); - expect(scheduled).not.toBeNull(); - await scheduled!; - expect((await storedCache(repo))?.models.map(m => m.id)).toEqual(['fresh']); + expect(result.map(model => model.id)).toEqual(['stale']); + await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); + resolveFetch!([aModel('fresh')]); + await scheduled.promises[0]; + expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['fresh']); }); - test('past HARD: blocks on fetch', async () => { + test('force is the explicit fetch operation and blocks for a fresh result', async () => { const repo = await setupRepo(); - const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 25 * 60 * 60_000, models: [aModel('stale')] }); + const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 1000, models: [aModel('stored')] }); const fetchFn = vi.fn(async () => [aModel('fresh')]); + const scheduled = captureScheduled(); const result = await fetchUpstreamModelsCached( stubInstance(fetchFn, cache), - { scheduler: () => {}, fetcher: directFetcher }, + { scheduler: scheduled.scheduler, fetcher: directFetcher, force: true }, ); - expect(result.map(m => m.id)).toEqual(['fresh']); - expect(fetchFn).toHaveBeenCalledTimes(1); - expect((await storedCache(repo))?.models.map(m => m.id)).toEqual(['fresh']); + expect(result.map(model => model.id)).toEqual(['fresh']); + expect(scheduled.promises).toEqual([]); + expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['fresh']); }); - // The instance carries the row as it was read at request start, and a - // request reaches this function once per alias target resolved. Without the - // write-back the second resolution would still see the stale snapshot and - // refetch, which the repo read used to prevent. - test('a fetch updates the instance so a later call in the same request is a cache hit', async () => { - const repo = await setupRepo(); - const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 25 * 60 * 60_000, models: [aModel('stale')] }); - const fetchFn = vi.fn(async () => [aModel('fresh')]); - const instance = stubInstance(fetchFn, cache); + test('concurrent cold callers join one background refresh', async () => { + await setupRepo(); + let resolveFetch: ((models: ProviderModel[]) => void) | null = null; + const fetchFn = vi.fn(() => new Promise(resolve => { resolveFetch = resolve; })); + const instance = stubInstance(fetchFn); + const scheduled = captureScheduled(); - const first = await fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher }); - const second = await fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher }); + const [first, second] = await Promise.all([ + fetchUpstreamModelsCached(instance, { scheduler: scheduled.scheduler, fetcher: directFetcher }), + fetchUpstreamModelsCached(instance, { scheduler: scheduled.scheduler, fetcher: directFetcher }), + ]); - expect(first.map(m => m.id)).toEqual(['fresh']); - expect(second.map(m => m.id)).toEqual(['fresh']); - expect(fetchFn).toHaveBeenCalledTimes(1); + expect(first).toEqual([]); + expect(second).toEqual([]); + await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); + resolveFetch!([aModel('m1')]); + await Promise.all(scheduled.promises); }); - test('force=true: bypasses cache and blocks on fetch', async () => { + test('a failed refresh preserves stale data and activates persistent backoff', async () => { const repo = await setupRepo(); - const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 1000, models: [aModel('stored')] }); - const fetchFn = vi.fn(async () => [aModel('fresh')]); + const now = 1_800_000_000_000; + vi.spyOn(Date, 'now').mockReturnValue(now); + const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: now - 20 * 60_000, models: [aModel('stale')] }); + const fetchFn = vi.fn(async () => { throw new Error('boom'); }); + const instance = stubInstance(fetchFn, cache); + const firstScheduled = captureScheduled(); - const result = await fetchUpstreamModelsCached( - stubInstance(fetchFn, cache), - { scheduler: () => {}, fetcher: directFetcher, force: true }, - ); + expect((await fetchUpstreamModelsCached(instance, { scheduler: firstScheduled.scheduler, fetcher: directFetcher })).map(model => model.id)).toEqual(['stale']); + await expect(firstScheduled.promises[0]).rejects.toThrow('boom'); - expect(result.map(m => m.id)).toEqual(['fresh']); + clearInFlightForTesting(); + const secondScheduled = captureScheduled(); + expect((await fetchUpstreamModelsCached(instance, { scheduler: secondScheduled.scheduler, fetcher: directFetcher })).map(model => model.id)).toEqual(['stale']); + await expect(secondScheduled.promises[0]).resolves.toBeUndefined(); expect(fetchFn).toHaveBeenCalledTimes(1); - expect((await storedCache(repo))?.models.map(m => m.id)).toEqual(['fresh']); + expect((await storedCache(repo))?.lastError?.message).toContain('boom'); }); - test('two concurrent cold callers join one fetch', async () => { + test('cold failures return empty and retry after the persisted backoff expires', async () => { await setupRepo(); - let resolveFetch: ((v: ProviderModel[]) => void) | null = null; - const fetchFn = vi.fn(() => new Promise(r => { resolveFetch = r; })); + let now = 1_800_000_000_000; + vi.spyOn(Date, 'now').mockImplementation(() => now); + const fetchFn = vi.fn(async () => { throw new Error('boom'); }); const instance = stubInstance(fetchFn); - const p1 = fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher }); - const p2 = fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher }); - - // Yield once so both calls reach the L1 lookup before we resolve the fetch. - await Promise.resolve(); - resolveFetch!([aModel('m1')]); - const [r1, r2] = await Promise.all([p1, p2]); + const firstScheduled = captureScheduled(); + await expect(fetchUpstreamModelsCached(instance, { scheduler: firstScheduled.scheduler, fetcher: directFetcher })).resolves.toEqual([]); + await expect(firstScheduled.promises[0]).rejects.toThrow('boom'); - expect(r1.map(m => m.id)).toEqual(['m1']); - expect(r2.map(m => m.id)).toEqual(['m1']); + clearInFlightForTesting(); + now += 59_999; + const backedOff = captureScheduled(); + await expect(fetchUpstreamModelsCached(instance, { scheduler: backedOff.scheduler, fetcher: directFetcher })).resolves.toEqual([]); + await expect(backedOff.promises[0]).resolves.toBeUndefined(); expect(fetchFn).toHaveBeenCalledTimes(1); + + clearInFlightForTesting(); + now += 1; + const retry = captureScheduled(); + await expect(fetchUpstreamModelsCached(instance, { scheduler: retry.scheduler, fetcher: directFetcher })).resolves.toEqual([]); + await expect(retry.promises[0]).rejects.toThrow('boom'); + expect(fetchFn).toHaveBeenCalledTimes(2); }); test('a superseded generation neither joins nor overwrites the current catalog', async () => { const repo = await setupRepo(); let resolveOld: ((models: ProviderModel[]) => void) | null = null; const oldFetch = vi.fn(() => new Promise(resolve => { resolveOld = resolve; })); - const oldRequest = fetchUpstreamModelsCached( + const oldScheduled = captureScheduled(); + await fetchUpstreamModelsCached( stubInstance(oldFetch, null, CACHE_GENERATION, 'same-fetch'), - { scheduler: () => {}, fetcher: directFetcher }, + { scheduler: oldScheduled.scheduler, fetcher: directFetcher }, ); - await Promise.resolve(); + await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); const nextGeneration = { updatedAt: CACHE_GENERATION.updatedAt, config: { identity: 'new' } }; const current = await repo.upstreams.getById(UPSTREAM_ID); if (!current) throw new Error('upstream row missing'); await repo.upstreams.saveClearingModelsCache({ ...current, updatedAt: nextGeneration.updatedAt, config: nextGeneration.config }); const newFetch = vi.fn(async () => [aModel('new-tenant-model')]); - const newRequest = fetchUpstreamModelsCached( + const newResult = await fetchUpstreamModelsCached( stubInstance(newFetch, null, nextGeneration, 'same-fetch'), { scheduler: () => {}, fetcher: directFetcher, force: true }, ); - expect((await newRequest).map(model => model.id)).toEqual(['new-tenant-model']); + expect(newResult.map(model => model.id)).toEqual(['new-tenant-model']); resolveOld!([aModel('old-tenant-model')]); - expect((await oldRequest).map(model => model.id)).toEqual(['old-tenant-model']); + await oldScheduled.promises[0]; expect(oldFetch).toHaveBeenCalledTimes(1); expect(newFetch).toHaveBeenCalledTimes(1); expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['new-tenant-model']); }); - test('drafts with one persistence generation but different fetch identities do not join', async () => { - await setupRepo(); - const firstFetch = vi.fn(async () => [aModel('first-draft-model')]); - const secondFetch = vi.fn(async () => [aModel('second-draft-model')]); - - const [first, second] = await Promise.all([ - fetchUpstreamModelsCached(stubInstance(firstFetch, null, CACHE_GENERATION, 'first-draft'), { scheduler: () => {}, fetcher: directFetcher, force: true }), - fetchUpstreamModelsCached(stubInstance(secondFetch, null, CACHE_GENERATION, 'second-draft'), { scheduler: () => {}, fetcher: directFetcher, force: true }), - ]); - - expect(first.map(model => model.id)).toEqual(['first-draft-model']); - expect(second.map(model => model.id)).toEqual(['second-draft-model']); - expect(firstFetch).toHaveBeenCalledTimes(1); - expect(secondFetch).toHaveBeenCalledTimes(1); - }); - - test('background revalidate failure preserves stored row and writes lastError', async () => { - const repo = await setupRepo(); - const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 20 * 60_000, models: [aModel('stale')] }); - const fetchFn = vi.fn(async () => { throw new Error('boom'); }); - let scheduled: Promise | null = null; - - const result = await fetchUpstreamModelsCached( - stubInstance(fetchFn, cache), - { scheduler: p => { scheduled = p; }, fetcher: directFetcher }, - ); - - expect(result.map(m => m.id)).toEqual(['stale']); - expect(scheduled).not.toBeNull(); - await scheduled!; - const stored = await storedCache(repo); - expect(stored?.models.map(m => m.id)).toEqual(['stale']); - expect(stored?.lastError?.message).toContain('boom'); - }); - - test('cold + fetch failure: throws and writes nothing', async () => { - const repo = await setupRepo(); - const fetchFn = vi.fn(async () => { throw new Error('boom'); }); - - await expect(fetchUpstreamModelsCached( - stubInstance(fetchFn), - { scheduler: () => {}, fetcher: directFetcher }, - )).rejects.toThrow('boom'); - - expect(await storedCache(repo)).toBeNull(); - }); - - test('force=true + fetch failure: throws (no fallback) and annotates lastError', async () => { - const repo = await setupRepo(); - const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 1000, models: [aModel('stored')] }); - const fetchFn = vi.fn(async () => { throw new Error('boom'); }); - - await expect(fetchUpstreamModelsCached( - stubInstance(fetchFn, cache), - { scheduler: () => {}, fetcher: directFetcher, force: true }, - )).rejects.toThrow('boom'); - - const stored = await storedCache(repo); - expect(stored?.models.map(m => m.id)).toEqual(['stored']); - expect(stored?.lastError?.message).toContain('boom'); - }); - - test('catalog revision mismatch bypasses a soft-fresh stored row', async () => { + test('catalog revision mismatch is cold and refreshes without blocking', async () => { const repo = await setupRepo(); const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION - 1, @@ -294,18 +259,19 @@ describe('fetchUpstreamModelsCached', () => { models: [aModel('old-catalog')], }); const fetchFn = vi.fn(async () => [aModel('current-catalog')]); + const scheduled = captureScheduled(); const result = await fetchUpstreamModelsCached( stubInstance(fetchFn, cache), - { scheduler: () => {}, fetcher: directFetcher }, + { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); - expect(result.map(model => model.id)).toEqual(['current-catalog']); - expect(fetchFn).toHaveBeenCalledTimes(1); + expect(result).toEqual([]); + await scheduled.promises[0]; expect((await storedCache(repo))?.revision).toBe(MODEL_CATALOG_REVISION); }); - test('an old-shape stale SQL cache hydrates cold and is replaced by a current fetch', async () => { + test('an obsolete SQL cache hydrates cold and is replaced in the background', async () => { const db = await createSqliteTestDb(); const repo = new SqlRepo(db); initRepo(repo); @@ -326,9 +292,6 @@ describe('fetchUpstreamModelsCached', () => { modelPrefix: null, hue: 210, }); - // Deliberately schema-incompatible body: a stale numeric revision is an - // opaque obsolete payload, so hydration must not apply today's model - // schema before deciding the cache is cold. await db.prepare('UPDATE upstreams SET models_cache_json = ? WHERE id = ?').bind(JSON.stringify({ revision: MODEL_CATALOG_REVISION - 1, fetchedAt: Date.now() - 1_000, @@ -338,18 +301,16 @@ describe('fetchUpstreamModelsCached', () => { const hydrated = await repo.upstreams.getById(UPSTREAM_ID); if (!hydrated) throw new Error('upstream row missing'); - expect(hydrated?.modelsCache).toBeNull(); + expect(hydrated.modelsCache).toBeNull(); const fetchFn = vi.fn(async () => [aModel('current-catalog')]); + const scheduled = captureScheduled(); const result = await fetchUpstreamModelsCached( - stubInstance(fetchFn, hydrated.modelsCache, { - updatedAt: hydrated.updatedAt, - config: hydrated.config, - }), - { scheduler: () => {}, fetcher: directFetcher }, + stubInstance(fetchFn, hydrated.modelsCache, { updatedAt: hydrated.updatedAt, config: hydrated.config }), + { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); - expect(result.map(model => model.id)).toEqual(['current-catalog']); - expect(fetchFn).toHaveBeenCalledTimes(1); + expect(result).toEqual([]); + await scheduled.promises[0]; expect((await repo.upstreams.getById(UPSTREAM_ID))?.modelsCache?.revision).toBe(MODEL_CATALOG_REVISION); }); }); From 14f5db092c2ce9de9d56c6dee3dfb8467a1dbdca Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:16:54 +0800 Subject: [PATCH 02/46] feat(gateway): coordinate model refresh triggers persistently Serve cold and stale catalogs without awaiting upstream I/O, retain stale catalogs indefinitely, and move automatic refresh attempts behind an inline per-upstream claim with exponential failure cooldown and lease recovery. Keep explicit model fetches and post-save or OAuth warming synchronous while sharing generation fencing and in-flight coordination. --- packages/gateway/__tests__/repo/memory.ts | 38 ++++++ .../0077_upstream_models_refresh.sql | 5 + .../control-plane/shared/warm-models-cache.ts | 6 +- .../src/data-plane/providers/models-cache.ts | 111 +++++++++++++----- .../src/repo/models-refresh-contract.ts | 10 ++ packages/gateway/src/repo/sql.ts | 56 ++++++++- packages/gateway/src/repo/types.ts | 3 + 7 files changed, 192 insertions(+), 37 deletions(-) create mode 100644 packages/gateway/migrations/0077_upstream_models_refresh.sql create mode 100644 packages/gateway/src/repo/models-refresh-contract.ts diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index 0ffd4f6ad..f5df66963 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -1,6 +1,7 @@ import { normalizeDisabledPublicModelIds } from '../../src/repo/disabled-public-models.ts'; import { normalizeFlagOverrides } from '../../src/repo/flag-overrides.ts'; import { normalizeProxyFallbackList } from '../../src/repo/proxy-fallback-list.ts'; +import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; import { assertSameStoredResponsesItem, cloneStoredResponsesItem, @@ -557,6 +558,7 @@ class MemoryWebSearchConfigRepo implements WebSearchConfigRepo { class MemoryUpstreamRepo implements UpstreamRepo { private store = new Map(); + private modelsRefreshes = new Map(); list(): Promise { return Promise.resolve([...this.store.values()].map(cloneUpstreamRecord).sort((a, b) => a.sortOrder - b.sortOrder || a.createdAt.localeCompare(b.createdAt))); @@ -585,15 +587,18 @@ class MemoryUpstreamRepo implements UpstreamRepo { ? { ...upstream, createdAt: existing.createdAt, modelsCache: null } : { ...upstream, modelsCache: null }; this.store.set(next.id, cloneUpstreamRecord(next)); + this.modelsRefreshes.delete(next.id); return Promise.resolve(); } delete(id: string): Promise { + this.modelsRefreshes.delete(id); return Promise.resolve(this.store.delete(id)); } deleteAll(): Promise { this.store.clear(); + this.modelsRefreshes.clear(); return Promise.resolve(); } @@ -628,6 +633,39 @@ class MemoryUpstreamRepo implements UpstreamRepo { cache.lastError = error; return Promise.resolve(true); } + + claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise { + if (this.store.get(id)?.updatedAt !== generation.updatedAt) return Promise.resolve(false); + const existing = this.modelsRefreshes.get(id); + const eligible = force + || existing === undefined + || (existing.retryAt <= now && (existing.claimToken === null || existing.claimedAt! <= staleClaimedBefore)); + if (!eligible) return Promise.resolve(false); + this.modelsRefreshes.set(id, { + failCount: existing?.failCount ?? 0, + retryAt: existing?.retryAt ?? 0, + claimToken: token, + claimedAt: now, + }); + return Promise.resolve(true); + } + + completeModelsRefreshSuccess(id: string, token: string): Promise { + if (this.modelsRefreshes.get(id)?.claimToken === token) this.modelsRefreshes.delete(id); + return Promise.resolve(); + } + + completeModelsRefreshFailure(id: string, token: string, now: number): Promise { + const existing = this.modelsRefreshes.get(id); + if (existing?.claimToken !== token) return Promise.resolve(); + this.modelsRefreshes.set(id, { + failCount: existing.failCount + 1, + retryAt: modelsRefreshRetryAt(now, existing.failCount), + claimToken: null, + claimedAt: null, + }); + return Promise.resolve(); + } } const cloneUpstreamRecord = (upstream: UpstreamRecord): UpstreamRecord => ({ diff --git a/packages/gateway/migrations/0077_upstream_models_refresh.sql b/packages/gateway/migrations/0077_upstream_models_refresh.sql new file mode 100644 index 000000000..5719349e8 --- /dev/null +++ b/packages/gateway/migrations/0077_upstream_models_refresh.sql @@ -0,0 +1,5 @@ +-- Refresh coordination stays on the upstream row beside the catalog it +-- protects. Stale catalog reads need one atomic claim before upstream I/O; +-- keeping that claim inline avoids restoring the serial side-table read that +-- migration 0072 removed from every catalog access. +ALTER TABLE upstreams ADD COLUMN models_refresh_json TEXT NULL; diff --git a/packages/gateway/src/control-plane/shared/warm-models-cache.ts b/packages/gateway/src/control-plane/shared/warm-models-cache.ts index 7ea12f15a..b426697a4 100644 --- a/packages/gateway/src/control-plane/shared/warm-models-cache.ts +++ b/packages/gateway/src/control-plane/shared/warm-models-cache.ts @@ -1,10 +1,9 @@ import type { Context } from 'hono'; -import { fetchUpstreamModelsCached } from '../../data-plane/providers/models-cache.ts'; +import { fetchUpstreamModels } from '../../data-plane/providers/models-cache.ts'; import { createProvider } from '../../data-plane/providers/registry.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; import { getRepo } from '../../repo/index.ts'; -import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { UpstreamModelsCache, UpstreamRecord } from '@floway-dev/provider'; import { logInfo } from '@floway-dev/provider-claude-code'; @@ -20,11 +19,10 @@ const errorMessage = (error: unknown): string => error instanceof Error ? error. // freshness this warm produced rather than the snapshot it read before saving. // Null when the upstream fetch failed and left the row with nothing to report. export const warmModelsCache = async (record: UpstreamRecord, c: Context): Promise => { - const scheduler = backgroundSchedulerFromContext(c); const provider = createProvider(record); const fetcher = (await createPerRequestFetcher(getRuntimeLocation(c.req.raw)))(record.id); try { - await fetchUpstreamModelsCached(provider, { scheduler, fetcher, force: true }); + await fetchUpstreamModels(provider, fetcher); } catch (error) { logInfo('warm_models_cache_failed', { upstream_id: record.id, error: errorMessage(error) }); } diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index 33d83531e..37029b92d 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -1,28 +1,23 @@ import type { GatewayProvider } from './registry.ts'; import { getRepo } from '../../repo/index.ts'; import { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; +import { MODELS_REFRESH_CLAIM_LEASE_MS } from '../../repo/models-refresh-contract.ts'; import { serializeStoredConfig } from '../../repo/upstream-json.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { Fetcher, ProviderModel } from '@floway-dev/provider'; -// Soft TTL: a fetched row is served verbatim within this window with no -// upstream call. Past SOFT but within HARD, the stored row is still served -// while a background revalidate refreshes it. Past HARD a fresh fetch is -// required and blocks the caller; a failed background revalidate within -// HARD leaves the row in place and only annotates the entry's `lastError`, -// which is also the rationale for treating SOFT/HARD as a single SWR window -// rather than introducing a separate fail-back tier. +// Soft-fresh rows need no refresh. Every older row remains usable forever; +// access only triggers a background attempt guarded by the persisted refresh +// claim/backoff state. const SOFT_MS = 10 * 60 * 1000; -const HARD_MS = 24 * 60 * 60 * 1000; export { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; export interface ModelsCacheFetchOptions { scheduler: BackgroundScheduler; fetcher: Fetcher; - // Skip the SOFT/HARD cache check and always trigger a fresh fetch. The - // call still joins the L1 in-flight map when both fetch identity and cache - // ownership match. Failure throws; no fall-back to the stored row. + // The upstream editor's explicit Fetch Models action is the sole caller of + // this option. It waits for an actual fetch and bypasses refresh backoff. force?: boolean; // Some control-plane callers also need the upstream's raw catalog shape. // Their loader projects that already-fetched response into the exact @@ -36,12 +31,12 @@ export interface ModelsCacheFetchOptions { // and superseded rows remain isolated. Not a TTL cache — the entry is removed // when the promise settles. The conditional delete defends against a stale // removal racing a later replacement. -const inFlight = new Map>(); +const inFlight = new Map>(); const memoInFlight = ( key: string, - fn: () => Promise, -): Promise => { + fn: () => Promise, +): Promise => { const existing = inFlight.get(key); if (existing) return existing; const promise = fn(); @@ -81,39 +76,91 @@ const runFetch = async ( } }; +const runClaimedFetch = async ( + instance: GatewayProvider, + fetcher: Fetcher, + force: boolean, + loadProvidedModels?: () => Promise, +): Promise => { + const repo = getRepo(); + const now = Date.now(); + const token = crypto.randomUUID(); + const claimed = await repo.upstreams.claimModelsRefresh( + instance.upstreamId, + instance.modelsCacheGeneration, + token, + now, + now - MODELS_REFRESH_CLAIM_LEASE_MS, + force, + ); + if (!claimed) return null; + + try { + const models = await runFetch(instance, fetcher, instance.upstreamId, loadProvidedModels); + await repo.upstreams.completeModelsRefreshSuccess(instance.upstreamId, token); + return models; + } catch (error) { + try { + await repo.upstreams.completeModelsRefreshFailure(instance.upstreamId, token, Date.now()); + } catch (backoffError) { + throw new AggregateError([error, backoffError], errorMessage(error)); + } + throw error; + } +}; + +const inFlightKey = (instance: GatewayProvider): string => { + const generation = instance.modelsCacheGeneration; + return `${instance.upstreamId}\0${instance.modelsFetchIdentity}\0${generation.updatedAt}\0${serializeStoredConfig(generation.config)}`; +}; + +export const fetchUpstreamModels = async ( + instance: GatewayProvider, + fetcher: Fetcher, + loadProvidedModels?: () => Promise, +): Promise => { + const key = inFlightKey(instance); + const existing = inFlight.get(key); + if (existing) { + const joined = await existing; + if (joined !== null) return joined; + if (inFlight.get(key) === existing) inFlight.delete(key); + } + + const models = await memoInFlight(key, () => runClaimedFetch(instance, fetcher, true, loadProvidedModels)); + if (models === null) throw new Error(`Failed to force-claim models refresh for ${instance.upstreamId}`); + return models; +}; + +export const triggerUpstreamModelsFetch = ( + instance: GatewayProvider, + scheduler: BackgroundScheduler, + fetcher: Fetcher, + loadProvidedModels?: () => Promise, +): void => { + const key = inFlightKey(instance); + scheduler(memoInFlight(key, () => runClaimedFetch(instance, fetcher, false, loadProvidedModels))); +}; + export const fetchUpstreamModelsCached = async ( instance: GatewayProvider, opts: ModelsCacheFetchOptions, ): Promise => { const { scheduler, fetcher, force, loadProvidedModels } = opts; - const key = instance.upstreamId; - const generation = instance.modelsCacheGeneration; - const inFlightKey = `${key}\0${instance.modelsFetchIdentity}\0${generation.updatedAt}\0${serializeStoredConfig(generation.config)}`; const now = Date.now(); if (force) { - return await memoInFlight(inFlightKey, () => runFetch(instance, fetcher, key, loadProvidedModels)); + return await fetchUpstreamModels(instance, fetcher, loadProvidedModels); } // Read off the instance rather than queried: the row that produced this // provider carried its catalog, so the SWR check costs nothing. const cached = instance.modelsCache?.revision === MODEL_CATALOG_REVISION ? instance.modelsCache : null; - if (cached && now - cached.fetchedAt < SOFT_MS) { - return cached.models; - } - - if (cached && now - cached.fetchedAt < HARD_MS) { - // Joining L1 here means a second request arriving mid-flight does - // not enqueue a second background task. The trailing `.catch` is the - // sink for the background branch only — `runFetch` already persists - // the failure via `saveModelsCacheError` before rethrowing, so the SWR - // caller who got `cached.models` does not need to learn about it. - scheduler(memoInFlight(inFlightKey, () => runFetch(instance, fetcher, key)).catch(() => {})); - return cached.models; - } + if (cached && now - cached.fetchedAt < SOFT_MS) return cached.models; - return await memoInFlight(inFlightKey, () => runFetch(instance, fetcher, key)); + triggerUpstreamModelsFetch(instance, scheduler, fetcher, loadProvidedModels); + return cached?.models ?? []; }; // Test-only: drop the L1 map so a test's setup is independent of any diff --git a/packages/gateway/src/repo/models-refresh-contract.ts b/packages/gateway/src/repo/models-refresh-contract.ts new file mode 100644 index 000000000..001bbbdb5 --- /dev/null +++ b/packages/gateway/src/repo/models-refresh-contract.ts @@ -0,0 +1,10 @@ +export const MODELS_REFRESH_BACKOFF_BASE_MS = 60_000; +export const MODELS_REFRESH_BACKOFF_CAP_MS = 60 * 60_000; +export const MODELS_REFRESH_BACKOFF_EXPONENT_CAP = 6; +export const MODELS_REFRESH_CLAIM_LEASE_MS = 15 * 60_000; + +export const modelsRefreshRetryAt = (now: number, previousFailureCount: number): number => + now + Math.min( + MODELS_REFRESH_BACKOFF_BASE_MS * (2 ** Math.min(previousFailureCount, MODELS_REFRESH_BACKOFF_EXPONENT_CAP)), + MODELS_REFRESH_BACKOFF_CAP_MS, + ); diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index 2ff27ab34..148be5865 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -3,6 +3,7 @@ import { SqlExpirationSweepsRepo } from './expiration-sweeps-sql.ts'; import { normalizeFlagOverrides } from './flag-overrides.ts'; import { decodeAliasTargets, decodeAnnouncedMetadata, encodeAliasTargets, encodeAnnouncedMetadata } from './model-alias-codecs.ts'; import { normalizeProxyFallbackList } from './proxy-fallback-list.ts'; +import { MODELS_REFRESH_BACKOFF_BASE_MS, MODELS_REFRESH_BACKOFF_CAP_MS, MODELS_REFRESH_BACKOFF_EXPONENT_CAP } from './models-refresh-contract.ts'; import { SqlResponsesItemsRepo, SqlResponsesSnapshotsRepo } from './responses-state-sql.ts'; import { generateSessionToken } from './session-tokens.ts'; import { SqlSpilledFilesRepo } from './spilled-files-sql.ts'; @@ -915,7 +916,7 @@ class SqlUpstreamRepo implements UpstreamRepo { disabled_public_model_ids = excluded.disabled_public_model_ids, proxy_fallback_list_json = excluded.proxy_fallback_list_json, model_prefix_json = excluded.model_prefix_json, - hue = excluded.hue${clearModelsCache ? ', models_cache_json = NULL' : ''}`, + hue = excluded.hue${clearModelsCache ? ', models_cache_json = NULL, models_refresh_json = NULL' : ''}`, ) .bind( upstream.id, @@ -973,6 +974,59 @@ class SqlUpstreamRepo implements UpstreamRepo { return (result.meta.changes ?? 0) > 0; } + async claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise { + const result = await this.db + .prepare( + `UPDATE upstreams + SET models_refresh_json = json_object( + 'failCount', coalesce(json_extract(models_refresh_json, '$.failCount'), 0), + 'retryAt', coalesce(json_extract(models_refresh_json, '$.retryAt'), 0), + 'claimToken', ?, + 'claimedAt', ? + ) + WHERE id = ? AND updated_at = ? AND ( + ? = 1 + OR models_refresh_json IS NULL + OR ( + coalesce(json_extract(models_refresh_json, '$.retryAt'), 0) <= ? + AND ( + json_extract(models_refresh_json, '$.claimToken') IS NULL + OR json_extract(models_refresh_json, '$.claimedAt') <= ? + ) + ) + )`, + ) + .bind(token, now, id, generation.updatedAt, sqliteBoolean(force), now, staleClaimedBefore) + .run(); + return (result.meta.changes ?? 0) > 0; + } + + async completeModelsRefreshSuccess(id: string, token: string): Promise { + await this.db + .prepare("UPDATE upstreams SET models_refresh_json = NULL WHERE id = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") + .bind(id, token) + .run(); + } + + async completeModelsRefreshFailure(id: string, token: string, now: number): Promise { + await this.db + .prepare( + `UPDATE upstreams + SET models_refresh_json = json_object( + 'failCount', coalesce(json_extract(models_refresh_json, '$.failCount'), 0) + 1, + 'retryAt', ? + min( + ? * (1 << min(coalesce(json_extract(models_refresh_json, '$.failCount'), 0), ?)), + ? + ), + 'claimToken', NULL, + 'claimedAt', NULL + ) + WHERE id = ? AND json_extract(models_refresh_json, '$.claimToken') = ?`, + ) + .bind(now, MODELS_REFRESH_BACKOFF_BASE_MS, MODELS_REFRESH_BACKOFF_EXPONENT_CAP, MODELS_REFRESH_BACKOFF_CAP_MS, id, token) + .run(); + } + private async modelsCacheWriteConfig(id: string, generation: ModelsCacheGeneration): Promise { const row = await this.db .prepare('SELECT updated_at, config_json FROM upstreams WHERE id = ?') diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index 4acae0e07..87efe89a1 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -267,6 +267,9 @@ export interface UpstreamRepo { // cannot publish models or errors under newer credentials/configuration. saveModelsCache(id: string, generation: ModelsCacheGeneration, cache: Omit): Promise; saveModelsCacheError(id: string, generation: ModelsCacheGeneration, error: NonNullable): Promise; + claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise; + completeModelsRefreshSuccess(id: string, token: string): Promise; + completeModelsRefreshFailure(id: string, token: string, now: number): Promise; } export interface ModelsCacheGeneration { From ea85283391b9a9310bd8b553f2187dbd4d10b76c Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:21:42 +0800 Subject: [PATCH 03/46] fix(gateway): expose model triggers as background work Discard the coordinator's internal fetched-or-skipped result at the scheduler boundary while preserving rejection for runtime observability. --- packages/gateway/src/data-plane/providers/models-cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index 37029b92d..dac4bbacd 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -139,7 +139,7 @@ export const triggerUpstreamModelsFetch = ( loadProvidedModels?: () => Promise, ): void => { const key = inFlightKey(instance); - scheduler(memoInFlight(key, () => runClaimedFetch(instance, fetcher, false, loadProvidedModels))); + scheduler(memoInFlight(key, () => runClaimedFetch(instance, fetcher, false, loadProvidedModels)).then(() => {})); }; export const fetchUpstreamModelsCached = async ( From 411d3aa3547cdb0d73e5f59862547c2de7bca91a Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:23:00 +0800 Subject: [PATCH 04/46] test(gateway): cover persisted model refresh coordination Exercise memory and SQL repositories against the same atomic claim, exponential cooldown, forced bypass, abandoned-lease recovery, generation fence, and stale-completion contract. --- .../__tests__/repo/models-refresh_test.ts | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 packages/gateway/__tests__/repo/models-refresh_test.ts diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts new file mode 100644 index 000000000..f62801ec4 --- /dev/null +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -0,0 +1,69 @@ +import { describe, expect, test } from 'vitest'; + +import { InMemoryRepo } from './memory.ts'; +import { createSqliteTestDb } from './test-sqlite.ts'; +import { SqlRepo } from '../../src/repo/sql.ts'; +import type { ModelsCacheGeneration, Repo } from '../../src/repo/types.ts'; +import type { UpstreamRecord } from '@floway-dev/provider'; + +const record: UpstreamRecord = { + id: 'up_refresh', + kind: 'custom', + name: 'Refresh', + enabled: true, + sortOrder: 0, + createdAt: '2026-08-01T00:00:00.000Z', + updatedAt: '2026-08-01T00:00:00.000Z', + config: {}, + state: null, + modelsCache: null, + flagOverrides: {}, + disabledPublicModelIds: [], + proxyFallbackList: [], + modelPrefix: null, + hue: 210, +}; + +const generation: ModelsCacheGeneration = { updatedAt: record.updatedAt, config: record.config }; + +const factories: [string, () => Promise][] = [ + ['memory', async () => new InMemoryRepo()], + ['SQL', async () => new SqlRepo(await createSqliteTestDb())], +]; + +describe.each(factories)('%s models refresh coordination', (_name, createRepo) => { + test('claims atomically, backs failures off exponentially, and lets force bypass cooldown', async () => { + const repo = await createRepo(); + await repo.upstreams.save(record); + const now = 1_800_000_000_000; + + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'first', now, now - 900_000, false)).resolves.toBe(true); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now, now - 900_000, false)).resolves.toBe(false); + await repo.upstreams.completeModelsRefreshFailure(record.id, 'first', now); + + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'early', now + 59_999, now - 900_000, false)).resolves.toBe(false); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'second', now + 60_000, now - 840_000, false)).resolves.toBe(true); + await repo.upstreams.completeModelsRefreshFailure(record.id, 'second', now + 60_000); + + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'second-early', now + 179_999, now - 720_000, false)).resolves.toBe(false); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'forced', now + 60_001, now - 839_999, true)).resolves.toBe(true); + await repo.upstreams.completeModelsRefreshSuccess(record.id, 'forced'); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'after-success', now + 60_002, now - 839_998, false)).resolves.toBe(true); + }); + + test('recovers abandoned claims and fences stale generations and completions', async () => { + const repo = await createRepo(); + await repo.upstreams.save(record); + const now = 1_800_000_000_000; + + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'abandoned', now, now - 900_000, false)).resolves.toBe(true); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'replacement', now + 900_001, now + 1, false)).resolves.toBe(true); + await repo.upstreams.completeModelsRefreshSuccess(record.id, 'abandoned'); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now + 900_002, now + 2, false)).resolves.toBe(false); + + const next = { ...record, updatedAt: '2026-08-01T00:01:00.000Z' }; + await repo.upstreams.saveClearingModelsCache(next); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'old-generation', now + 900_003, now + 3, false)).resolves.toBe(false); + await expect(repo.upstreams.claimModelsRefresh(record.id, { ...generation, updatedAt: next.updatedAt }, 'current', now + 900_003, now + 3, false)).resolves.toBe(true); + }); +}); From df18f908dd871f474d9a51052e8b99868730271a Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:24:55 +0800 Subject: [PATCH 05/46] feat(gateway): trigger model refreshes during maintenance Walk enabled upstreams on every scheduled maintenance tick, use the same nonblocking cache trigger and persisted cooldown as request access, and await the detached work inside the maintenance lifetime. Use the configured Node runtime location and a dedicated scheduled location tag where the runtime exposes no request colo. --- apps/platform-node/entry.ts | 3 +- .../scheduled/models-refresh_test.ts | 43 +++++++++++++++++++ packages/gateway/src/scheduled.ts | 4 +- .../gateway/src/scheduled/models-refresh.ts | 21 +++++++++ 4 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 packages/gateway/__tests__/scheduled/models-refresh_test.ts create mode 100644 packages/gateway/src/scheduled/models-refresh.ts diff --git a/apps/platform-node/entry.ts b/apps/platform-node/entry.ts index 804270452..7c38b7ec3 100644 --- a/apps/platform-node/entry.ts +++ b/apps/platform-node/entry.ts @@ -48,6 +48,7 @@ initResponsesWebSocketUpgradeResolver((c, events) => const { db } = bootstrapNodePlatform(); const port = Number(getEnvOptional('PORT', '8788')); +const scheduledRuntimeLocation = getEnvOptional('RUNTIME_LOCATION', 'LOCAL').toUpperCase(); // Passwordless admin login is a dev-only shortcut (empty ADMIN_KEY on a // local instance grants seed-admin access). Refuse to boot the Node @@ -72,7 +73,7 @@ initRepo(new SqlRepo(db)); // unref() on both timers lets the process exit cleanly on SIGINT. const STARTUP_DELAY_MS = 30 * 1000; const sweep = (): void => { - runScheduledMaintenance().catch(err => { + runScheduledMaintenance(scheduledRuntimeLocation).catch(err => { console.error('[scheduled-maintenance] sweep failed:', err); }); }; diff --git a/packages/gateway/__tests__/scheduled/models-refresh_test.ts b/packages/gateway/__tests__/scheduled/models-refresh_test.ts new file mode 100644 index 000000000..e135673f6 --- /dev/null +++ b/packages/gateway/__tests__/scheduled/models-refresh_test.ts @@ -0,0 +1,43 @@ +import { expect, test } from 'vitest'; + +import { initRepo } from '../../src/repo/index.ts'; +import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; +import { refreshModelsCaches } from '../../src/scheduled/models-refresh.ts'; +import { InMemoryRepo } from '../repo/memory.ts'; +import type { UpstreamRecord } from '@floway-dev/provider'; + +const azure = (id: string, enabled: boolean): UpstreamRecord => ({ + id, + kind: 'azure', + name: id, + enabled, + sortOrder: 0, + createdAt: '2026-08-01T00:00:00.000Z', + updatedAt: '2026-08-01T00:00:00.000Z', + config: { + endpoint: 'https://example.openai.azure.com/openai/v1', + apiKey: 'azkey', + models: [{ upstreamModelId: `${id}-wire`, publicModelId: `${id}-public`, endpoints: { chatCompletions: {} } }], + }, + state: null, + modelsCache: null, + flagOverrides: {}, + disabledPublicModelIds: [], + proxyFallbackList: [], + modelPrefix: null, + hue: 210, +}); + +test('scheduled maintenance triggers cold enabled upstreams and waits for their background work', async () => { + const repo = new InMemoryRepo(); + initRepo(repo); + await repo.upstreams.save(azure('enabled', true)); + await repo.upstreams.save(azure('disabled', false)); + + await refreshModelsCaches('SCHEDULED'); + + const enabled = await repo.upstreams.getById('enabled'); + expect(enabled?.modelsCache?.revision).toBe(MODEL_CATALOG_REVISION); + expect(enabled?.modelsCache?.models.map(model => model.id)).toEqual(['enabled-public']); + expect((await repo.upstreams.getById('disabled'))?.modelsCache).toBeNull(); +}); diff --git a/packages/gateway/src/scheduled.ts b/packages/gateway/src/scheduled.ts index 54660842a..a847309ae 100644 --- a/packages/gateway/src/scheduled.ts +++ b/packages/gateway/src/scheduled.ts @@ -1,4 +1,5 @@ import { sweepExpirations } from './scheduled/expiration-sweeps.ts'; +import { refreshModelsCaches } from './scheduled/models-refresh.ts'; import { collectSpilledFiles } from './scheduled/spilled-files.ts'; import { getImageCacheStore } from '@floway-dev/platform'; @@ -12,8 +13,9 @@ const runSweep = async (name: string, fn: () => Promise): Promise => { +export const runScheduledMaintenance = async (runtimeLocation = 'SCHEDULED'): Promise => { const nowMs = Date.now(); + await runSweep('models.refresh', () => refreshModelsCaches(runtimeLocation)); await runSweep('expirations.sweep', () => sweepExpirations(nowMs)); await runSweep('spilledFiles.collect', () => collectSpilledFiles(nowMs)); await runSweep('imageCacheStore.sweepExpired', () => getImageCacheStore().sweepExpired(nowMs)); diff --git a/packages/gateway/src/scheduled/models-refresh.ts b/packages/gateway/src/scheduled/models-refresh.ts new file mode 100644 index 000000000..0b0c2e7f0 --- /dev/null +++ b/packages/gateway/src/scheduled/models-refresh.ts @@ -0,0 +1,21 @@ +import { fetchUpstreamModelsCached } from '../data-plane/providers/models-cache.ts'; +import { createProvider } from '../data-plane/providers/registry.ts'; +import { createPerRequestFetcher } from '../dial/per-request.ts'; +import { getRepo } from '../repo/index.ts'; + +export const refreshModelsCaches = async (runtimeLocation: string): Promise => { + const upstreams = (await getRepo().upstreams.list()).filter(upstream => upstream.enabled); + const fetcherForUpstream = await createPerRequestFetcher(runtimeLocation, upstreams); + const pending: Promise[] = []; + + for (const upstream of upstreams) { + await fetchUpstreamModelsCached(createProvider(upstream), { + scheduler: promise => { pending.push(promise); }, + fetcher: fetcherForUpstream(upstream.id), + }); + } + + const settled = await Promise.allSettled(pending); + const errors = settled.flatMap(result => result.status === 'rejected' ? [result.reason] : []); + if (errors.length > 0) throw new AggregateError(errors, `${errors.length} model cache refreshes failed`); +}; From f6176b09bbdc55c942810b3965e3b312a19373b1 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:29:49 +0800 Subject: [PATCH 06/46] fix(gateway): retain cold model refresh failures Persist a failed first refresh as an empty immediately-stale catalog so later requests and the dashboard retain its error while automatic retries remain governed by backoff. Read cached error annotations during catalog assembly now that upstream failures happen outside the requesting lifecycle. --- .../data-plane/providers/models-cache_test.ts | 3 ++- packages/gateway/__tests__/repo/memory.ts | 11 ++++------- .../src/data-plane/providers/catalog.ts | 16 +++++++++------- .../src/data-plane/providers/models-cache.ts | 10 ++++++---- packages/gateway/src/repo/sql.ts | 19 ++++++++++++------- 5 files changed, 33 insertions(+), 26 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index e5980805f..fa6e2e7cf 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -197,7 +197,7 @@ describe('fetchUpstreamModelsCached', () => { }); test('cold failures return empty and retry after the persisted backoff expires', async () => { - await setupRepo(); + const repo = await setupRepo(); let now = 1_800_000_000_000; vi.spyOn(Date, 'now').mockImplementation(() => now); const fetchFn = vi.fn(async () => { throw new Error('boom'); }); @@ -206,6 +206,7 @@ describe('fetchUpstreamModelsCached', () => { const firstScheduled = captureScheduled(); await expect(fetchUpstreamModelsCached(instance, { scheduler: firstScheduled.scheduler, fetcher: directFetcher })).resolves.toEqual([]); await expect(firstScheduled.promises[0]).rejects.toThrow('boom'); + expect(await storedCache(repo)).toMatchObject({ fetchedAt: 0, models: [], lastError: { message: 'boom' } }); clearInFlightForTesting(); now += 59_999; diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index f5df66963..f20353524 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -1,5 +1,6 @@ import { normalizeDisabledPublicModelIds } from '../../src/repo/disabled-public-models.ts'; import { normalizeFlagOverrides } from '../../src/repo/flag-overrides.ts'; +import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; import { normalizeProxyFallbackList } from '../../src/repo/proxy-fallback-list.ts'; import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; import { @@ -622,15 +623,11 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(true); } - // No-op on a row that has never cached a catalog: the annotation belongs to a - // previously-successful fetch. saveModelsCacheError(id: string, generation: ModelsCacheGeneration, error: NonNullable): Promise { const existing = this.store.get(id); - const cache = existing?.updatedAt === generation.updatedAt && serializeStoredConfig(existing.config) === serializeStoredConfig(generation.config) - ? existing.modelsCache - : null; - if (!cache) return Promise.resolve(false); - cache.lastError = error; + if (!existing || existing.updatedAt !== generation.updatedAt || serializeStoredConfig(existing.config) !== serializeStoredConfig(generation.config)) return Promise.resolve(false); + if (existing.modelsCache) existing.modelsCache.lastError = error; + else existing.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }; return Promise.resolve(true); } diff --git a/packages/gateway/src/data-plane/providers/catalog.ts b/packages/gateway/src/data-plane/providers/catalog.ts index 87b36c512..182401409 100644 --- a/packages/gateway/src/data-plane/providers/catalog.ts +++ b/packages/gateway/src/data-plane/providers/catalog.ts @@ -93,16 +93,14 @@ const collectProviderModels = async ( let lastError: unknown = null; const failedUpstreams: string[] = []; - // Fan out per-upstream so a slow provider does not stall the rest. The SWR - // cache layer dedupes concurrent in-flight fetches per upstream and serves - // the SOFT-fresh row without an upstream round trip, so the parallel walk - // is cheap on the warm path and bounded by `max(per-upstream fetch)` on - // the cold path. + // Catalog reads never await upstream I/O. Each result is the persisted + // snapshot carried by the provider; a cold or stale snapshot separately + // triggers background refresh through the supplied scheduler. const fetchOne = (instance: GatewayProvider) => fetchUpstreamModelsCached(instance, { scheduler, fetcher: fetcherForUpstream(instance.upstreamId), - }).then(models => ({ instance, models })); + }).then(models => ({ instance, models, lastError: instance.modelsCache?.lastError ?? null })); const settled = await Promise.allSettled(providers.map(fetchOne)); @@ -121,7 +119,11 @@ const collectProviderModels = async ( continue; } sawSuccess = true; - const { instance, models: providedModels } = result.value; + const { instance, models: providedModels, lastError: cachedError } = result.value; + if (cachedError) { + lastError = new Error(cachedError.message); + failedUpstreams.push(instance.name); + } // Operator-disabled public model ids vanish entirely for this upstream: // dropped before they reach the catalog map, so they appear in no /models // listing and resolve to nothing for routing. The disable is per-upstream, diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index dac4bbacd..431c9d784 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -68,10 +68,12 @@ const runFetch = async ( if (persisted) instance.modelsCache = entry; return models; } catch (err) { - // A no-op on an upstream with no cached catalog: a brand-new upstream that - // fails its first fetch surfaces the error to the caller with nothing - // persisted. - await getRepo().upstreams.saveModelsCacheError(key, generation, { message: errorMessage(err), at: Date.now() }); + const lastError = { message: errorMessage(err), at: Date.now() }; + const persisted = await getRepo().upstreams.saveModelsCacheError(key, generation, lastError); + if (persisted) { + if (instance.modelsCache) instance.modelsCache.lastError = lastError; + else instance.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError }; + } throw err; } }; diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index 148be5865..337c9cc80 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -1,6 +1,7 @@ import { normalizeDisabledPublicModelIds } from './disabled-public-models.ts'; import { SqlExpirationSweepsRepo } from './expiration-sweeps-sql.ts'; import { normalizeFlagOverrides } from './flag-overrides.ts'; +import { MODEL_CATALOG_REVISION } from './models-cache-contract.ts'; import { decodeAliasTargets, decodeAnnouncedMetadata, encodeAliasTargets, encodeAnnouncedMetadata } from './model-alias-codecs.ts'; import { normalizeProxyFallbackList } from './proxy-fallback-list.ts'; import { MODELS_REFRESH_BACKOFF_BASE_MS, MODELS_REFRESH_BACKOFF_CAP_MS, MODELS_REFRESH_BACKOFF_EXPONENT_CAP } from './models-refresh-contract.ts'; @@ -959,17 +960,21 @@ class SqlUpstreamRepo implements UpstreamRepo { return (result.meta.changes ?? 0) > 0; } - // Annotates a previously-successful entry, so an upstream that has never - // cached a catalog has nothing to annotate. Patched in SQL rather than - // read-modify-written: it touches one key of a document whose other keys a - // concurrent refresh may be rewriting, and nothing compares this column's - // text, so the encoding SQLite produces here is immaterial. + // A cold failure persists an empty, immediately-stale catalog so the error + // remains visible without making the failed attempt look soft-fresh. An + // existing last-known-good catalog keeps its models and fetch timestamp. async saveModelsCacheError(id: string, generation: ModelsCacheGeneration, error: NonNullable): Promise { const rawConfig = await this.modelsCacheWriteConfig(id, generation); if (rawConfig === null) return false; + const coldFailure = encodeUpstreamModelsCache({ + revision: MODEL_CATALOG_REVISION, + fetchedAt: 0, + models: [], + lastError: error, + }); const result = await this.db - .prepare("UPDATE upstreams SET models_cache_json = json_set(models_cache_json, '$.lastError', json(?)) WHERE id = ? AND updated_at = ? AND config_json = ? AND models_cache_json IS NOT NULL") - .bind(JSON.stringify(error), id, generation.updatedAt, rawConfig) + .prepare("UPDATE upstreams SET models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END WHERE id = ? AND updated_at = ? AND config_json = ?") + .bind(coldFailure, JSON.stringify(error), id, generation.updatedAt, rawConfig) .run(); return (result.meta.changes ?? 0) > 0; } From 5baff88f04a21baddae76438b00d049a50e7ebad Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:32:40 +0800 Subject: [PATCH 07/46] test(gateway): model production cache warming Warm direct-write app fixtures after fetch mocks are installed and before model-consuming requests, matching the synchronous create, update, and OAuth lifecycle. Keep provider catalog and resolution suites on persisted snapshots so their assertions remain about assembly and routing rather than cold-cache trigger timing. --- .../data-plane/providers/catalog_test.ts | 12 +++++-- .../data-plane/providers/resolution_test.ts | 12 +++++-- packages/gateway/__tests__/test-utils/app.ts | 32 +++++++++++++++++-- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts index 04b8d5bc7..8308ab83b 100644 --- a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts @@ -4,9 +4,17 @@ import { compareModelIds, getModelsFromProviders } from '../../../src/data-plane import { clearInFlightForTesting } from '../../../src/data-plane/providers/models-cache.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; -import { buildCustomUpstreamRecord, copilotModels, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, copilotModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; import { directFetcher, type InternalModel, type ProviderModel } from '@floway-dev/provider'; -import { assertEquals, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { assertEquals, jsonResponse, withMockedFetch as withMockedFetchRaw } from '@floway-dev/test-utils'; + +const withMockedFetch = ( + handler: Parameters[0], + fn: () => Promise, +): Promise => withMockedFetchRaw(handler, async () => { + await warmModelsForTest(); + return await fn(); +}); const realProviderModels = (model: InternalModel | undefined): Record => { if (model?.providerModels === undefined) throw new Error(`expected real InternalModel with providerModels, got ${JSON.stringify(model)}`); diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index dd9c03b67..c74efed6e 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -3,9 +3,17 @@ import { describe, expect, test } from 'vitest'; import { clearInFlightForTesting } from '../../../src/data-plane/providers/models-cache.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; import { enumerateModelCandidates, enumerateRealModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; -import { buildCustomUpstreamRecord, copilotModels, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, copilotModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; import { directFetcher, type InternalModel, type ProviderModel } from '@floway-dev/provider'; -import { assertEquals, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { assertEquals, jsonResponse, withMockedFetch as withMockedFetchRaw } from '@floway-dev/test-utils'; + +const withMockedFetch = ( + handler: Parameters[0], + fn: () => Promise, +): Promise => withMockedFetchRaw(handler, async () => { + await warmModelsForTest(); + return await fn(); +}); const realProviderModels = (model: InternalModel | undefined): Record => { if (model?.providerModels === undefined) throw new Error(`expected real InternalModel with providerModels, got ${JSON.stringify(model)}`); diff --git a/packages/gateway/__tests__/test-utils/app.ts b/packages/gateway/__tests__/test-utils/app.ts index 4a7983f05..c07e076ab 100644 --- a/packages/gateway/__tests__/test-utils/app.ts +++ b/packages/gateway/__tests__/test-utils/app.ts @@ -1,12 +1,15 @@ -import { trackBackground } from './background-tracker.ts'; +import { flushBackground, trackBackground } from './background-tracker.ts'; import { app } from '../../src/app.ts'; -import { clearInFlightForTesting } from '../../src/data-plane/providers/models-cache.ts'; +import { clearInFlightForTesting, fetchUpstreamModelsCached } from '../../src/data-plane/providers/models-cache.ts'; +import { listModelProviders } from '../../src/data-plane/providers/registry.ts'; import type { WebSearchConfig } from '../../src/data-plane/tools/web-search/types.ts'; +import { createPerRequestFetcher } from '../../src/dial/per-request.ts'; import { initRepo } from '../../src/repo/index.ts'; import type { ApiKey } from '../../src/repo/types.ts'; import { initBackgroundSchedulerResolver } from '../../src/runtime/background.ts'; import { InMemoryRepo } from '../repo/memory.ts'; import { createInMemoryImageProcessor, initEnv, initExternalResourceFetcher, initFileStore, initImageProcessor, initSocketDial, MemoryFileStore } from '@floway-dev/platform'; +import { PUBLIC_DATA_PLANE_ROUTES } from '@floway-dev/protocols/common'; import type { ProxyFallbackEntry, UpstreamRecord } from '@floway-dev/provider'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; @@ -297,9 +300,34 @@ export function sseResponsesResponse(response: Record): Respons } export async function requestApp(path: string, init: RequestInit): Promise { + const method = init.method?.toUpperCase() ?? 'GET'; + const pathname = new URL(path, 'http://localhost').pathname; + const isModelConsumer = pathname === '/api/models' || Object.values(PUBLIC_DATA_PLANE_ROUTES).some(route => + route.method === method && route.paths.some(template => { + const parameter = template.indexOf('/:'); + return parameter === -1 ? pathname === template : pathname.startsWith(template.slice(0, parameter + 1)); + })); + if (isModelConsumer) await warmModelsForTest(); return await app.request(path, init); } +// App fixtures write upstream rows directly because their fetch mocks are not +// installed until the test body runs. Production create/update/OAuth flows +// synchronously warm before returning; reproduce that lifecycle immediately +// before a model-consuming request while leaving repository/cache unit tests +// free to exercise genuinely cold reads. +export const warmModelsForTest = async (): Promise => { + const providers = await listModelProviders(null); + const fetcherForUpstream = await createPerRequestFetcher('TEST'); + await Promise.all(providers.map(async provider => { + await fetchUpstreamModelsCached(provider, { + scheduler: trackBackground, + fetcher: fetcherForUpstream(provider.upstreamId), + }); + })); + await flushBackground(); +}; + export function parseSSEText(text: string): Array<{ event: string; data: string }> { const blocks = text .split('\n\n') From 4e75300800a16fd0e1e8a87d8b69577595056663 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:35:23 +0800 Subject: [PATCH 08/46] fix(gateway): report cached model refresh failures Carry persisted catalog errors into request-time failed-upstream metadata without coupling request cancellation to detached refresh I/O. Update catalog and HTTP coverage for empty nonblocking listings when no last-known-good models exist, while retaining secret redaction and healthy sibling results. --- .../__tests__/data-plane/models/http_test.ts | 37 +++++--------- .../data-plane/providers/catalog_test.ts | 1 + .../data-plane/providers/resolution_test.ts | 49 +++++++------------ .../src/data-plane/providers/resolution.ts | 7 +-- 4 files changed, 35 insertions(+), 59 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/models/http_test.ts b/packages/gateway/__tests__/data-plane/models/http_test.ts index 6be546664..8419c70d0 100644 --- a/packages/gateway/__tests__/data-plane/models/http_test.ts +++ b/packages/gateway/__tests__/data-plane/models/http_test.ts @@ -360,9 +360,8 @@ test('/v1/models hides upstream identity when a provider returns an invalid mode headers: { 'x-api-key': apiKey.key }, }); - assertEquals(response.status, 502); - const body = (await response.json()) as { error: { message: string } }; - assertEquals(body.error.message, 'Upstream model listing failed'); + assertEquals(response.status, 200); + assertEquals(await response.json(), { object: 'list', data: [] }); }, ); }); @@ -462,14 +461,11 @@ test('public model list endpoints hide upstream HTTP error bodies and headers', const response = await requestApp(path, { headers: { 'x-api-key': apiKey.key }, }); - assertEquals(response.status, 502); + assertEquals(response.status, 200); assertEquals(response.headers.get('x-upstream-id'), null); - assertEquals(await response.json(), { - error: { - message: 'Upstream model listing failed', - type: 'api_error', - }, - }); + const body = JSON.stringify(await response.json()); + assertEquals(body.includes('secret upstream body'), false); + assertEquals(body.includes('up_http_secret_provider'), false); } }, ); @@ -505,13 +501,9 @@ test('public model list endpoints hide thrown upstream request errors', async () const response = await requestApp(path, { headers: { 'x-api-key': apiKey.key }, }); - assertEquals(response.status, 502); - assertEquals(await response.json(), { - error: { - message: 'Upstream model listing failed', - type: 'api_error', - }, - }); + assertEquals(response.status, 200); + const body = JSON.stringify(await response.json()); + assertEquals(body.includes('throw-secret.example.com'), false); } }, ); @@ -550,13 +542,10 @@ test('public model list endpoints hide malformed upstream response bodies', asyn const response = await requestApp(path, { headers: { 'x-api-key': apiKey.key }, }); - assertEquals(response.status, 502); - assertEquals(await response.json(), { - error: { - message: 'Upstream model listing failed', - type: 'api_error', - }, - }); + assertEquals(response.status, 200); + const body = JSON.stringify(await response.json()); + assertEquals(body.includes('secret malformed body'), false); + assertEquals(body.includes('up_malformed_secret_provider'), false); } }, ); diff --git a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts index 8308ab83b..4d626c5b6 100644 --- a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts @@ -233,6 +233,7 @@ test('disabledPublicModelIds hides models from the catalog and routing, per upst disabledPublicModelIds: [], })); + await warmModelsForTest(); const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; assertEquals([...catalog.map(m => m.id)].sort(), ['gpt-keep', 'gpt-shared']); diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index c74efed6e..f3f61b3d5 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { clearInFlightForTesting } from '../../../src/data-plane/providers/models-cache.ts'; +import { clearInFlightForTesting, fetchUpstreamModels } from '../../../src/data-plane/providers/models-cache.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; import { enumerateModelCandidates, enumerateRealModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { buildCustomUpstreamRecord, copilotModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; @@ -170,7 +170,7 @@ test('enumerateRealModelCandidates only loads the selected providers\' catalogs' const providers = await listModelProviders(null); let secondModelsFetches = 0; - await withMockedFetch( + await withMockedFetchRaw( request => { const url = new URL(request.url); if (url.hostname === 'first.example.com' && url.pathname === '/v1/models') { @@ -183,7 +183,10 @@ test('enumerateRealModelCandidates only loads the selected providers\' catalogs' throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const { candidates } = await enumerateRealModelCandidates('target-model', 'chat', [providers[0]], () => directFetcher, testScheduler); + await fetchUpstreamModels(providers[0], directFetcher); + const warmed = (await listModelProviders(null)).find(provider => provider.upstreamId === 'up_first'); + if (!warmed) throw new Error('warmed provider missing'); + const { candidates } = await enumerateRealModelCandidates('target-model', 'chat', [warmed], () => directFetcher, testScheduler); assertEquals(candidates[0]?.model.id, 'target-model'); assertEquals(candidates[0]?.provider.upstreamId, 'up_first'); @@ -224,6 +227,7 @@ test('enumerateRealModelCandidates rejects a model id disabled on that upstream state: null, }); + await warmModelsForTest(); const providers = await listModelProviders(null); const enabled = await enumerateRealModelCandidates('enabled-model', 'chat', providers, () => directFetcher, testScheduler); const disabled = await enumerateRealModelCandidates('disabled-model', 'chat', providers, () => directFetcher, testScheduler); @@ -364,13 +368,7 @@ test('enumerateModelCandidates deduplicates failedUpstreams across the dated-suf ); }); -// AbortError must propagate end-to-end so the caller's per-request abort -// signal cannot be masked by a slow upstream. Burying it in failedUpstreams -// would let the rest of the data-plane request build a Response against a -// stale catalog. The provider's `fetchUpstreamModels` wraps the upstream -// fetch error in a ProviderModelsUnavailableError with the AbortError as -// its cause, so the resolver's detection walks the cause chain. -test('enumerateModelCandidates rethrows AbortError from a per-upstream catalog fetch', async () => { +test('an AbortError from background catalog refresh does not abort model resolution', async () => { clearInFlightForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); @@ -391,28 +389,15 @@ test('enumerateModelCandidates rethrows AbortError from a per-upstream catalog f throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - let thrown: unknown = null; - try { - await enumerateModelCandidates({ - upstreamIds: null, - model: 'any-model', - kind: 'chat', - scheduler: testScheduler, - runtimeLocation: 'TEST', - }); - } catch (e) { - thrown = e; - } - // The thrown error chains back to our injected AbortError via .cause. - const isAbortInChain = (err: unknown): boolean => { - for (let cur: unknown = err; cur != null; cur = (cur as { cause?: unknown }).cause) { - if (cur instanceof Error && cur.name === 'AbortError') return true; - } - return false; - }; - if (!isAbortInChain(thrown)) { - throw new Error(`expected rejection to carry an AbortError in its cause chain; got: ${thrown instanceof Error ? `${thrown.name}: ${thrown.message}` : String(thrown)}`); - } + const resolved = await enumerateModelCandidates({ + upstreamIds: null, + model: 'any-model', + kind: 'chat', + scheduler: testScheduler, + runtimeLocation: 'TEST', + }); + expect(resolved.candidates).toEqual([]); + expect(resolved.failedUpstreams).toEqual(['Aborting']); }, ); }); diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index d4639a12f..305a500d5 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -30,7 +30,7 @@ const enumerateOneUpstreamCandidates = async ( kind: ModelKind, fetcher: Fetcher, scheduler: BackgroundScheduler, -): Promise<{ candidates: ModelCandidate[]; sawAnyId: boolean }> => { +): Promise<{ candidates: ModelCandidate[]; sawAnyId: boolean; modelsError: boolean }> => { const cfg = provider.modelPrefix; const lookupIds: string[] = []; if (cfg === null) { @@ -41,7 +41,7 @@ const enumerateOneUpstreamCandidates = async ( else if (form === 'prefixed' && modelId.startsWith(cfg.prefix)) lookupIds.push(modelId.slice(cfg.prefix.length)); } } - if (lookupIds.length === 0) return { candidates: [], sawAnyId: false }; + if (lookupIds.length === 0) return { candidates: [], sawAnyId: false, modelsError: provider.modelsCache?.lastError != null }; const providedModels = await fetchUpstreamModelsCached(provider, { scheduler, fetcher }); const disabled = new Set(provider.disabledPublicModelIds); @@ -55,7 +55,7 @@ const enumerateOneUpstreamCandidates = async ( candidates.push({ provider, model: internalModelFromProviderModel(match, provider.upstreamId), fetcher }); } } - return { candidates, sawAnyId }; + return { candidates, sawAnyId, modelsError: provider.modelsCache?.lastError != null }; }; // Walk every visible upstream, in configured order, and collect every @@ -96,6 +96,7 @@ export const enumerateRealModelCandidates = async ( } candidates.push(...result.value.candidates); sawAnyId = sawAnyId || result.value.sawAnyId; + if (result.value.modelsError) failedUpstreams.push(providers[index].name); } return { candidates, sawAnyId, failedUpstreams }; }; From bec453e87b5e5b70b046e371e27abfc91617fdbf Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:36:38 +0800 Subject: [PATCH 09/46] test(gateway): assert empty model-list envelope Preserve the public list pagination fields when a cold catalog refresh fails and the request returns the currently empty snapshot. --- packages/gateway/__tests__/data-plane/models/http_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gateway/__tests__/data-plane/models/http_test.ts b/packages/gateway/__tests__/data-plane/models/http_test.ts index 8419c70d0..9dec995f6 100644 --- a/packages/gateway/__tests__/data-plane/models/http_test.ts +++ b/packages/gateway/__tests__/data-plane/models/http_test.ts @@ -361,7 +361,7 @@ test('/v1/models hides upstream identity when a provider returns an invalid mode }); assertEquals(response.status, 200); - assertEquals(await response.json(), { object: 'list', data: [] }); + assertEquals(await response.json(), { object: 'list', data: [], has_more: false, first_id: null, last_id: null }); }, ); }); From 3bdb6b9f5c84c443a5c4bdcf7eeca6a59c74a407 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:38:09 +0800 Subject: [PATCH 10/46] test(gateway): scope fixture catalog warming Warm model-consuming app fixtures only while their upstream fetch mock is installed, avoiding real network work before validation-only requests. Keep unrelated scheduled-maintenance tests catalog-free and explicitly warm the standalone alpha-search app fixture. --- .../gateway/__tests__/data-plane/alpha-search/routes_test.ts | 3 ++- packages/gateway/__tests__/scheduled_test.ts | 2 ++ packages/gateway/__tests__/test-utils/app.ts | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts b/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts index 1f15cb457..f834c0d47 100644 --- a/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts +++ b/packages/gateway/__tests__/data-plane/alpha-search/routes_test.ts @@ -6,7 +6,7 @@ import { resolveConfiguredWebSearchProvider } from '../../../src/data-plane/tool import type { WebSearchConfig, WebSearchFetchPageRequest, WebSearchFetchPageResult, WebSearchProvider, WebSearchProviderRequest, WebSearchProviderResult } from '../../../src/data-plane/tools/web-search/types.ts'; import { type AuthVars, authMiddleware } from '../../../src/middleware/auth.ts'; import { internalErrorResponse } from '../../../src/middleware/internal-error-response.ts'; -import { buildCustomUpstreamRecord, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; import { withMockedFetch } from '@floway-dev/test-utils'; // Real provider construction (`createTavilyWebSearchProvider` etc.) hits the @@ -192,6 +192,7 @@ describe('/alpha/search data plane', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { + await warmModelsForTest(); const response = await postSearch(buildAlphaSearchApp(), apiKey.key, { id: 'session-search', model: 'caller-model', diff --git a/packages/gateway/__tests__/scheduled_test.ts b/packages/gateway/__tests__/scheduled_test.ts index 4ba4a56f3..03fcd0735 100644 --- a/packages/gateway/__tests__/scheduled_test.ts +++ b/packages/gateway/__tests__/scheduled_test.ts @@ -6,6 +6,7 @@ import { initFileStore, initImageCacheStore, MemoryFileStore } from '@floway-dev test('scheduled maintenance isolates the shared expiration driver from later collectors', async () => { const { repo } = await setupAppTest(); + await repo.upstreams.deleteAll(); initFileStore(new MemoryFileStore()); let imageSwept = false; initImageCacheStore({ @@ -27,6 +28,7 @@ test('scheduled maintenance isolates the shared expiration driver from later col test('scheduled maintenance collects exact spilled files after expiration work', async () => { const { repo } = await setupAppTest(); + await repo.upstreams.deleteAll(); const files = new MemoryFileStore(); initFileStore(files); initImageCacheStore({ async get() { return null; }, async put() {}, async sweepExpired() {} }); diff --git a/packages/gateway/__tests__/test-utils/app.ts b/packages/gateway/__tests__/test-utils/app.ts index c07e076ab..2e47ba678 100644 --- a/packages/gateway/__tests__/test-utils/app.ts +++ b/packages/gateway/__tests__/test-utils/app.ts @@ -50,6 +50,8 @@ interface SSEChunk { data: string | Record; } +const processFetch = globalThis.fetch; + const TEST_UPSTREAM_TIMESTAMP = '2026-03-15T00:00:00.000Z'; // The gateway's default egress is direct_connect, whose seam is a SocketDial. @@ -307,7 +309,7 @@ export async function requestApp(path: string, init: RequestInit): Promise Date: Thu, 6 Aug 2026 02:46:06 +0800 Subject: [PATCH 11/46] test(gateway): warm direct model consumers Bring local Codex, WebSocket, addressable-listing, target-picker, and endpoint fixtures through the same persisted warm lifecycle before exercising their actual subject. Update SQL and Gemini expectations for persisted cold errors and immediate empty listings. --- .../chat/responses/websocket_test.ts | 3 +- .../chat/shared/target-picker_test.ts | 4 +- .../__tests__/data-plane/codex/routes_test.ts | 25 +++++------ .../data-plane/codex/routes_websocket_test.ts | 3 +- .../data-plane/completions/http_test.ts | 3 +- .../data-plane/models/gemini_test.ts | 44 +++++-------------- .../shared/listing/addressable_test.ts | 4 +- packages/gateway/__tests__/repo/sql_test.ts | 9 +++- 8 files changed, 43 insertions(+), 52 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts index bd1534815..5828445c8 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/websocket_test.ts @@ -9,7 +9,7 @@ import { DOWNSTREAM_KEEP_ALIVE_INTERVAL_MS } from '../../../../src/data-plane/sh import { initDumpBroker, initDumpStore } from '../../../../src/dump/registry.ts'; import { installDumpStubs } from '../../../dump/test-fixtures.ts'; import { FakeTime } from '../../../test-time.ts'; -import { copilotModels, flushAsyncWork, setupAppTest, sseResponse, sseResponsesResponse } from '../../../test-utils/app.ts'; +import { copilotModels, flushAsyncWork, setupAppTest, sseResponse, sseResponsesResponse, warmModelsForTest } from '../../../test-utils/app.ts'; import { installWorkerWebSocketRuntime, type TestWorkerWebSocket } from '../../../test-utils/worker-websocket.ts'; import { assert, assertEquals, assertExists, assertStringIncludes, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -66,6 +66,7 @@ const terminalResponseId = (messages: readonly Record[]): strin }; const connectResponsesWebSocket = async (apiKey: string): Promise => { + await warmModelsForTest(); const executionCtx = { waitUntil: () => {}, passThroughOnException: () => {}, diff --git a/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts index bbfea5073..4283690a7 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'vitest'; import { chatTargetPicker } from '../../../../src/data-plane/chat/shared/target-picker.ts'; import { enumerateModelCandidates } from '../../../../src/data-plane/providers/resolution.ts'; -import { setupAppTest } from '../../../test-utils/app.ts'; +import { setupAppTest, warmModelsForTest } from '../../../test-utils/app.ts'; import type { ModelEndpoints } from '@floway-dev/protocols/common'; import type { UpstreamRecord } from '@floway-dev/provider'; import { assertEquals } from '@floway-dev/test-utils'; @@ -78,6 +78,7 @@ describe('enumerateModelCandidates + chatTargetPicker', () => { const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(azureUpstream('up_multi', 10, ['test-model'], { messages: {}, responses: {} })); + await warmModelsForTest(); const { candidates } = await enumerateModelCandidates({ upstreamIds: null, @@ -100,6 +101,7 @@ describe('enumerateModelCandidates + chatTargetPicker', () => { const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(azureUpstream('up_chat', 10, ['test-model'], { chatCompletions: {} })); + await warmModelsForTest(); const { candidates } = await enumerateModelCandidates({ upstreamIds: null, diff --git a/packages/gateway/__tests__/data-plane/codex/routes_test.ts b/packages/gateway/__tests__/data-plane/codex/routes_test.ts index 973432cbe..d3a32830b 100644 --- a/packages/gateway/__tests__/data-plane/codex/routes_test.ts +++ b/packages/gateway/__tests__/data-plane/codex/routes_test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest'; import { mountCodexRoutes } from '../../../src/data-plane/codex/routes.ts'; import { type AuthVars, authMiddleware } from '../../../src/middleware/auth.ts'; -import { copilotModels, setupAppTest } from '../../test-utils/app.ts'; +import { copilotModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; const buildCodexApp = () => { @@ -40,6 +40,13 @@ interface CodexModelsResponse { }>; } +const requestCodexModels = async (apiKey: string): Promise => { + await warmModelsForTest(); + return await buildCodexApp().request('/azure-api.codex/models', { + headers: { authorization: `Bearer ${apiKey}`, 'user-agent': CODEX_USER_AGENT }, + }); +}; + describe('Codex model-provider routes', () => { it('owns the namespaced alpha-search path', async () => { const { apiKey } = await setupAppTest(); @@ -97,9 +104,7 @@ describe('Codex model-provider routes', () => { const body = await withMockedFetch( copilotFetch([{ id: 'gpt-5.5', maxContextWindowTokens: 1050000 }]), async () => { - const response = await buildCodexApp().request('/azure-api.codex/models', { - headers: { authorization: `Bearer ${apiKey.key}`, 'user-agent': CODEX_USER_AGENT }, - }); + const response = await requestCodexModels(apiKey.key); expect(response.status).toBe(200); return await response.json() as CodexModelsResponse; }, @@ -117,9 +122,7 @@ describe('Codex model-provider routes', () => { const body = await withMockedFetch( copilotFetch([{ id: 'gpt-5.4', maxContextWindowTokens: 272000 }]), async () => { - const response = await buildCodexApp().request('/azure-api.codex/models', { - headers: { authorization: `Bearer ${apiKey.key}`, 'user-agent': CODEX_USER_AGENT }, - }); + const response = await requestCodexModels(apiKey.key); expect(response.status).toBe(200); return await response.json() as CodexModelsResponse; }, @@ -134,9 +137,7 @@ describe('Codex model-provider routes', () => { const { apiKey } = await setupAppTest(); const body = await withMockedFetch( copilotFetch([{ id: 'gpt-5.5', maxContextWindowTokens: 1050000 }]), - async () => await (await buildCodexApp().request('/azure-api.codex/models', { - headers: { authorization: `Bearer ${apiKey.key}`, 'user-agent': CODEX_USER_AGENT }, - })).json() as CodexModelsResponse, + async () => await (await requestCodexModels(apiKey.key)).json() as CodexModelsResponse, ); expect(body.models.map(model => model.slug)).toEqual(['gpt-5.5']); @@ -146,9 +147,7 @@ describe('Codex model-provider routes', () => { const { apiKey } = await setupAppTest(); const body = await withMockedFetch( copilotFetch([{ id: 'claude-sonnet-4', supported_endpoints: ['/v1/messages'] }]), - async () => await (await buildCodexApp().request('/azure-api.codex/models', { - headers: { authorization: `Bearer ${apiKey.key}`, 'user-agent': CODEX_USER_AGENT }, - })).json() as CodexModelsResponse, + async () => await (await requestCodexModels(apiKey.key)).json() as CodexModelsResponse, ); expect(body.models).toHaveLength(1); diff --git a/packages/gateway/__tests__/data-plane/codex/routes_websocket_test.ts b/packages/gateway/__tests__/data-plane/codex/routes_websocket_test.ts index 6be42f645..17a7c0695 100644 --- a/packages/gateway/__tests__/data-plane/codex/routes_websocket_test.ts +++ b/packages/gateway/__tests__/data-plane/codex/routes_websocket_test.ts @@ -2,7 +2,7 @@ import type { ExecutionContext } from 'hono'; import { expect, it } from 'vitest'; import { app as gatewayApp } from '../../../src/app.ts'; -import { copilotModels, setupAppTest, sseResponsesResponse } from '../../test-utils/app.ts'; +import { copilotModels, setupAppTest, sseResponsesResponse, warmModelsForTest } from '../../test-utils/app.ts'; import { installWorkerWebSocketRuntime, type TestWorkerWebSocket } from '../../test-utils/worker-websocket.ts'; import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -42,6 +42,7 @@ const connectCodexResponsesWebSocket = async ( runtime: ReturnType, apiKey: string, ): Promise => { + await warmModelsForTest(); const executionCtx = { waitUntil: () => {}, passThroughOnException: () => {}, diff --git a/packages/gateway/__tests__/data-plane/completions/http_test.ts b/packages/gateway/__tests__/data-plane/completions/http_test.ts index eacb12aa4..a8a766310 100644 --- a/packages/gateway/__tests__/data-plane/completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/completions/http_test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; import { installDumpStubs } from '../../dump/test-fixtures.ts'; -import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -223,6 +223,7 @@ test('/v1/completions rejects a model without the completions endpoint with the }], }, })); + await warmModelsForTest(); const response = await requestApp('/v1/completions', { method: 'POST', diff --git a/packages/gateway/__tests__/data-plane/models/gemini_test.ts b/packages/gateway/__tests__/data-plane/models/gemini_test.ts index f793cd05f..e13dfaad6 100644 --- a/packages/gateway/__tests__/data-plane/models/gemini_test.ts +++ b/packages/gateway/__tests__/data-plane/models/gemini_test.ts @@ -262,14 +262,8 @@ test('/v1beta/models hides upstream identity when a provider returns an invalid const response = await requestApp('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); - assertEquals(response.status, 502); - assertEquals(await response.json(), { - error: { - code: 502, - message: 'Upstream model listing failed', - status: 'UNAVAILABLE', - }, - }); + assertEquals(response.status, 200); + assertEquals(await response.json(), { models: [] }); }, ); }); @@ -310,14 +304,9 @@ test('/v1beta/models hides upstream HTTP error bodies', async () => { const response = await requestApp('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); - assertEquals(response.status, 502); - assertEquals(await response.json(), { - error: { - code: 502, - message: 'Upstream model listing failed', - status: 'UNAVAILABLE', - }, - }); + assertEquals(response.status, 200); + const body = JSON.stringify(await response.json()); + assertEquals(body, '{"models":[]}'); }, ); }); @@ -355,14 +344,9 @@ test('/v1beta/models hides thrown upstream request errors', async () => { const response = await requestApp('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); - assertEquals(response.status, 502); - assertEquals(await response.json(), { - error: { - code: 502, - message: 'Upstream model listing failed', - status: 'UNAVAILABLE', - }, - }); + assertEquals(response.status, 200); + const body = JSON.stringify(await response.json()); + assertEquals(body.includes('gemini-throw-secret.example.com'), false); }, ); }); @@ -403,14 +387,10 @@ test('/v1beta/models hides malformed upstream response bodies', async () => { const response = await requestApp('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); - assertEquals(response.status, 502); - assertEquals(await response.json(), { - error: { - code: 502, - message: 'Upstream model listing failed', - status: 'UNAVAILABLE', - }, - }); + assertEquals(response.status, 200); + const body = JSON.stringify(await response.json()); + assertEquals(body.includes('secret malformed body'), false); + assertEquals(body.includes('up_malformed_secret_gemini'), false); }, ); }); diff --git a/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts b/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts index e2b99dae0..4f9e92b1d 100644 --- a/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'vitest'; import { clearInFlightForTesting } from '../../../../src/data-plane/providers/models-cache.ts'; import { enumerateAddressableModelIds } from '../../../../src/data-plane/shared/listing/addressable.ts'; -import { buildCustomUpstreamRecord, setupAppTest } from '../../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, setupAppTest, warmModelsForTest } from '../../../test-utils/app.ts'; import { directFetcher } from '@floway-dev/provider'; import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -26,6 +26,7 @@ describe('enumerateAddressableModelIds', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { + await warmModelsForTest(); const surface = await enumerateAddressableModelIds(null, () => directFetcher, noBackground); expect(surface.map(e => ({ id: e.id, unlisted: e.unlisted }))).toEqual([ { id: 'shared-model', unlisted: undefined }, @@ -55,6 +56,7 @@ describe('enumerateAddressableModelIds', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { + await warmModelsForTest(); const surface = await enumerateAddressableModelIds(null, () => directFetcher, noBackground); const byId = new Map(surface.map(e => [e.id, e])); expect(byId.get('cust/gpt-5.4')?.unlisted).toBeUndefined(); diff --git a/packages/gateway/__tests__/repo/sql_test.ts b/packages/gateway/__tests__/repo/sql_test.ts index 29068f837..d1f8f5475 100644 --- a/packages/gateway/__tests__/repo/sql_test.ts +++ b/packages/gateway/__tests__/repo/sql_test.ts @@ -138,13 +138,18 @@ test('SQL upstream repo saveModelsCacheError annotates a cached catalog and save assertEquals((await repo.getById('up_test'))?.modelsCache?.lastError, null); }); -test('SQL upstream repo saveModelsCacheError is a no-op on a row that never cached a catalog', async () => { +test('SQL upstream repo saveModelsCacheError persists an immediately-stale empty catalog on first failure', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); await repo.saveModelsCacheError('up_test', generationFor(baseRecord()), { message: 'boom', at: 1_700_000_500_000 }); - assertEquals((await repo.getById('up_test'))?.modelsCache, null); + assertEquals((await repo.getById('up_test'))?.modelsCache, { + revision: MODEL_CATALOG_REVISION, + fetchedAt: 0, + models: [], + lastError: { message: 'boom', at: 1_700_000_500_000 }, + }); }); test('SQL upstream repo saveClearingModelsCache updates the row and removes the cached catalog atomically', async () => { From 274e963575e083c74e9da0b0cbe6d5b3aac12a86 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 02:56:42 +0800 Subject: [PATCH 12/46] test(gateway): isolate model fixture warming Await only catalog refreshes started by the fixture warm, so unrelated WebSocket and dump background work cannot hold the next test's setup open. --- packages/gateway/__tests__/test-utils/app.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/gateway/__tests__/test-utils/app.ts b/packages/gateway/__tests__/test-utils/app.ts index 2e47ba678..40159bc02 100644 --- a/packages/gateway/__tests__/test-utils/app.ts +++ b/packages/gateway/__tests__/test-utils/app.ts @@ -1,4 +1,4 @@ -import { flushBackground, trackBackground } from './background-tracker.ts'; +import { trackBackground } from './background-tracker.ts'; import { app } from '../../src/app.ts'; import { clearInFlightForTesting, fetchUpstreamModelsCached } from '../../src/data-plane/providers/models-cache.ts'; import { listModelProviders } from '../../src/data-plane/providers/registry.ts'; @@ -321,13 +321,14 @@ export async function requestApp(path: string, init: RequestInit): Promise => { const providers = await listModelProviders(null); const fetcherForUpstream = await createPerRequestFetcher('TEST'); + const pending: Promise[] = []; await Promise.all(providers.map(async provider => { await fetchUpstreamModelsCached(provider, { - scheduler: trackBackground, + scheduler: promise => { pending.push(promise); }, fetcher: fetcherForUpstream(provider.upstreamId), }); })); - await flushBackground(); + await Promise.allSettled(pending); }; export function parseSSEText(text: string): Array<{ event: string; data: string }> { From 602d13f5a39b9894cf5329aa71703b791412ff88 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 03:02:38 +0800 Subject: [PATCH 13/46] docs: define persistent infinite model SWR Document synchronous control-plane warming, nonblocking request and scheduled triggers, indefinite stale retention, inline claims, leases, failure cooldown, cold-error snapshots, and generation fencing. Align resolution comments and make the refresh fan-out test measure the actual background work. --- AGENTS.md | 5 ++-- docs/RESOLUTION.md | 28 +++++++++++++++++-- .../data-plane/providers/catalog_test.ts | 7 +++-- .../data-plane/providers/resolution_test.ts | 7 ++--- .../src/data-plane/providers/resolution.ts | 9 +++--- 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 394c8602b..4a59c8a87 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -295,8 +295,9 @@ repositories, model catalog/resolution, proxy-bound fetchers, protocol routes, Stateful Responses, affinity, telemetry, and scheduled work. Shared data-plane request context and candidate iteration live under `data-plane/shared/`; provider composition, catalog assembly, and request-time resolution live under -`data-plane/providers/{registry,catalog,resolution}.ts`; scheduled expiration -and spilled-file workers live under `scheduled/`. The package exports the +`data-plane/providers/{registry,catalog,resolution}.ts`; scheduled model-cache +refresh, expiration, and spilled-file workers live under `scheduled/`. The +package exports the migration corpus location through `@floway-dev/gateway/migrations-dir` and the dashboard's dump contracts through the types-only `./dump-types` subpath. diff --git a/docs/RESOLUTION.md b/docs/RESOLUTION.md index a9ae91863..2d576cdd2 100644 --- a/docs/RESOLUTION.md +++ b/docs/RESOLUTION.md @@ -21,8 +21,29 @@ candidate result returned to the client. The stages are deliberately separate: `data-plane/providers/registry.ts` constructs enabled provider instances. `catalog.ts` assembles their models, while `resolution.ts` performs request-time -matching. Both paths use each upstream's SWR-cached `getProvidedModels` result -and an upstream-scoped proxy-aware fetcher. +matching. Both paths read each upstream's persisted `getProvidedModels` +projection and carry an upstream-scoped proxy-aware fetcher for background +refresh and eventual inference. + +Catalog access never waits for an upstream model-list request. A row is +soft-fresh for ten minutes. After that point the last-known-good catalog remains +usable without any hard expiration; every access returns it immediately and +triggers a background refresh. A cold row returns an empty catalog immediately +and triggers the same work. The hourly maintenance driver triggers every +enabled upstream as well. + +Every automatic trigger enters one persisted per-upstream coordinator before +upstream I/O. Its atomic claim coalesces Workers, Node processes, and restarts; +a fifteen-minute lease recovers an abandoned attempt. Failures advance a +one-minute exponential backoff capped at one hour, while success clears the +coordinator and replaces the catalog under the upstream generation fence. A +first failure persists an empty catalog with `fetchedAt: 0` and `lastError`, so +the error stays observable without making the empty result soft-fresh. + +Two control-plane paths deliberately perform synchronous fetches: the explicit +**Fetch models** action, and the warm after create, update, import, or OAuth +credential changes. They share cache persistence and in-flight coordination +with automatic refreshes but bypass automatic-trigger backoff. For every provider model, `modelPrefix.listed` determines its public catalog surface: @@ -408,7 +429,8 @@ coordinate boundary. - Disabling an id on one upstream does not hide the same id on another. - The `-\d{8}` retry is the only request-time model-id normalization. -- Catalogs are SWR-cached per upstream. Soft-fresh reads do not block on refresh. +- Catalogs are persisted per upstream. Soft-fresh reads return directly; every + older read remains SWR forever and schedules a backoff-governed refresh. - Dual-addressable forms intentionally remain separate candidates. Their order follows the configured `addressable` array. - A listing row's unioned endpoint map must never be used for dispatch; attempt diff --git a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts index 4d626c5b6..f08baf87c 100644 --- a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts @@ -250,11 +250,11 @@ test('disabledPublicModelIds hides models from the catalog and routing, per upst assertEquals(keep.candidates.map(m => m.provider.upstreamId), ['up_a']); }); -// Per-upstream catalog fetches fan out in parallel: total wall-clock time +// Per-upstream catalog refresh triggers fan out in parallel: total wall-clock time // tracks the slowest upstream, not the sum. The bound is loose because CI // timer noise eats into a tight `< sum` comparison; what matters is the // ratio. -test('catalog assembly fans out per-upstream catalog fetches in parallel', async () => { +test('catalog refresh triggers fan out per upstream in parallel', async () => { clearInFlightForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); @@ -274,7 +274,7 @@ test('catalog assembly fans out per-upstream catalog fetches in parallel', async })); } - await withMockedFetch( + await withMockedFetchRaw( async request => { const url = new URL(request.url); const match = upstreams.find(u => url.hostname === u.host); @@ -286,6 +286,7 @@ test('catalog assembly fans out per-upstream catalog fetches in parallel', async }, async () => { const start = Date.now(); + await warmModelsForTest(); const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; const elapsed = Date.now() - start; diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index f3f61b3d5..942d4c12b 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -235,10 +235,9 @@ test('enumerateRealModelCandidates rejects a model id disabled on that upstream assertEquals(disabled.candidates.length, 0); }); -// Regression: when an upstream's force re-fetch rejects past HARD, the call -// site asking for a model belonging to one of the *healthy* upstreams must -// still resolve. The broken upstream's display name flows back via -// `failedUpstreams` so the eventual error renderer can mention it. +// A persisted refresh error must not hide healthy siblings. The broken +// upstream's display name flows back via `failedUpstreams` while its empty +// or last-known-good snapshot stays independent of the current request. test('enumerateModelCandidates: healthy upstream still resolves alongside a rejecting one, with failedUpstreams reported', async () => { clearInFlightForTesting(); const { repo } = await setupAppTest(); diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index 305a500d5..68f043b51 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -59,11 +59,10 @@ const enumerateOneUpstreamCandidates = async ( }; // Walk every visible upstream, in configured order, and collect every -// (provider, model, fetcher) candidate the inbound id resolves against -// at the requested kind. Per-upstream catalog fetches fan out concurrently -// so a slow upstream cannot stall the rest. Cancellation (`AbortError`) -// propagates so the per-request abort signal cannot be masked by a slow -// upstream's rejection. +// (provider, model, fetcher) candidate the inbound id resolves against at the +// requested kind. Each lookup reads only the provider's persisted catalog; +// cold and stale rows schedule refresh separately, so upstream model-list I/O +// cannot delay resolution. // // `sawAnyId` aggregates the per-upstream signal: true when at least one // upstream's catalog carried the inbound id under any kind. The caller From 194e00fa43cd9526cb11fa34a16b2da85c49fa82 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 03:31:00 +0800 Subject: [PATCH 14/46] style(gateway): order model refresh imports --- packages/gateway/__tests__/repo/memory.ts | 2 +- packages/gateway/src/repo/sql.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index f20353524..e7249d6f7 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -1,8 +1,8 @@ import { normalizeDisabledPublicModelIds } from '../../src/repo/disabled-public-models.ts'; import { normalizeFlagOverrides } from '../../src/repo/flag-overrides.ts'; import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; -import { normalizeProxyFallbackList } from '../../src/repo/proxy-fallback-list.ts'; import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; +import { normalizeProxyFallbackList } from '../../src/repo/proxy-fallback-list.ts'; import { assertSameStoredResponsesItem, cloneStoredResponsesItem, diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index 337c9cc80..9a6e894b3 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -1,10 +1,10 @@ import { normalizeDisabledPublicModelIds } from './disabled-public-models.ts'; import { SqlExpirationSweepsRepo } from './expiration-sweeps-sql.ts'; import { normalizeFlagOverrides } from './flag-overrides.ts'; -import { MODEL_CATALOG_REVISION } from './models-cache-contract.ts'; import { decodeAliasTargets, decodeAnnouncedMetadata, encodeAliasTargets, encodeAnnouncedMetadata } from './model-alias-codecs.ts'; -import { normalizeProxyFallbackList } from './proxy-fallback-list.ts'; +import { MODEL_CATALOG_REVISION } from './models-cache-contract.ts'; import { MODELS_REFRESH_BACKOFF_BASE_MS, MODELS_REFRESH_BACKOFF_CAP_MS, MODELS_REFRESH_BACKOFF_EXPONENT_CAP } from './models-refresh-contract.ts'; +import { normalizeProxyFallbackList } from './proxy-fallback-list.ts'; import { SqlResponsesItemsRepo, SqlResponsesSnapshotsRepo } from './responses-state-sql.ts'; import { generateSessionToken } from './session-tokens.ts'; import { SqlSpilledFilesRepo } from './spilled-files-sql.ts'; From 0958867dc81f056ff604ac74b31dfceb5fce8bd5 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 03:52:29 +0800 Subject: [PATCH 15/46] fix(gateway): fence model catalogs by refresh owner Require the active persisted claim token to publish catalog or error results, and reject claims from either half of a stale timestamp/config generation. Compute the capped exponential retry schedule once in the coordinator, persist explicit outcomes in both repositories, and cover force/lease supersession plus the complete one-hour cap. --- .../data-plane/providers/models-cache_test.ts | 46 ++++++++++++++++ packages/gateway/__tests__/repo/memory.ts | 26 +++++++--- .../__tests__/repo/models-refresh_test.ts | 52 ++++++++++++------- .../src/data-plane/providers/models-cache.ts | 15 +++--- packages/gateway/src/repo/sql.ts | 48 ++++++++++++----- packages/gateway/src/repo/types.ts | 10 +++- 6 files changed, 148 insertions(+), 49 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index fa6e2e7cf..5410a0fb6 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -252,6 +252,52 @@ describe('fetchUpstreamModelsCached', () => { expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['new-tenant-model']); }); + test('a forced fetch prevents an older claim from publishing a late success', async () => { + const repo = await setupRepo(); + let resolveOld: ((models: ProviderModel[]) => void) | null = null; + const oldFetch = vi.fn(() => new Promise(resolve => { resolveOld = resolve; })); + const oldScheduled = captureScheduled(); + await fetchUpstreamModelsCached( + stubInstance(oldFetch, null, CACHE_GENERATION, 'old-claim'), + { scheduler: oldScheduled.scheduler, fetcher: directFetcher }, + ); + await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); + + const forced = await fetchUpstreamModelsCached( + stubInstance(async () => [aModel('forced-model')], null, CACHE_GENERATION, 'forced-claim'), + { scheduler: () => {}, fetcher: directFetcher, force: true }, + ); + expect(forced.map(model => model.id)).toEqual(['forced-model']); + + resolveOld!([aModel('late-old-model')]); + await oldScheduled.promises[0]; + expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['forced-model']); + }); + + test('a forced fetch prevents an older claim from publishing a late error', async () => { + const repo = await setupRepo(); + let rejectOld: ((error: Error) => void) | null = null; + const oldFetch = vi.fn(() => new Promise((_resolve, reject) => { rejectOld = reject; })); + const oldScheduled = captureScheduled(); + await fetchUpstreamModelsCached( + stubInstance(oldFetch, null, CACHE_GENERATION, 'old-error-claim'), + { scheduler: oldScheduled.scheduler, fetcher: directFetcher }, + ); + await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); + + await fetchUpstreamModelsCached( + stubInstance(async () => [aModel('forced-model')], null, CACHE_GENERATION, 'forced-error-claim'), + { scheduler: () => {}, fetcher: directFetcher, force: true }, + ); + rejectOld!(new Error('late old failure')); + await expect(oldScheduled.promises[0]).rejects.toThrow('late old failure'); + + expect(await storedCache(repo)).toMatchObject({ + models: [{ id: 'forced-model' }], + lastError: null, + }); + }); + test('catalog revision mismatch is cold and refreshes without blocking', async () => { const repo = await setupRepo(); const cache = await seedCache(repo, { diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index e7249d6f7..e14eba89f 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -1,7 +1,6 @@ import { normalizeDisabledPublicModelIds } from '../../src/repo/disabled-public-models.ts'; import { normalizeFlagOverrides } from '../../src/repo/flag-overrides.ts'; import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; -import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; import { normalizeProxyFallbackList } from '../../src/repo/proxy-fallback-list.ts'; import { assertSameStoredResponsesItem, @@ -631,20 +630,31 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(true); } - claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise { - if (this.store.get(id)?.updatedAt !== generation.updatedAt) return Promise.resolve(false); + saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { + if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); + return this.saveModelsCache(id, generation, cache); + } + + saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise { + if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); + return this.saveModelsCacheError(id, generation, error); + } + + claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise<{ failureCount: number } | null> { + const stored = this.store.get(id); + if (!stored || stored.updatedAt !== generation.updatedAt || serializeStoredConfig(stored.config) !== serializeStoredConfig(generation.config)) return Promise.resolve(null); const existing = this.modelsRefreshes.get(id); const eligible = force || existing === undefined || (existing.retryAt <= now && (existing.claimToken === null || existing.claimedAt! <= staleClaimedBefore)); - if (!eligible) return Promise.resolve(false); + if (!eligible) return Promise.resolve(null); this.modelsRefreshes.set(id, { failCount: existing?.failCount ?? 0, retryAt: existing?.retryAt ?? 0, claimToken: token, claimedAt: now, }); - return Promise.resolve(true); + return Promise.resolve({ failureCount: existing?.failCount ?? 0 }); } completeModelsRefreshSuccess(id: string, token: string): Promise { @@ -652,12 +662,12 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(); } - completeModelsRefreshFailure(id: string, token: string, now: number): Promise { + completeModelsRefreshFailure(id: string, token: string, failureCount: number, retryAt: number): Promise { const existing = this.modelsRefreshes.get(id); if (existing?.claimToken !== token) return Promise.resolve(); this.modelsRefreshes.set(id, { - failCount: existing.failCount + 1, - retryAt: modelsRefreshRetryAt(now, existing.failCount), + failCount: failureCount, + retryAt, claimToken: null, claimedAt: null, }); diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index f62801ec4..8f54338fa 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'vitest'; import { InMemoryRepo } from './memory.ts'; import { createSqliteTestDb } from './test-sqlite.ts'; +import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import type { ModelsCacheGeneration, Repo } from '../../src/repo/types.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; @@ -14,7 +15,7 @@ const record: UpstreamRecord = { sortOrder: 0, createdAt: '2026-08-01T00:00:00.000Z', updatedAt: '2026-08-01T00:00:00.000Z', - config: {}, + config: { tenant: 'current' }, state: null, modelsCache: null, flagOverrides: {}, @@ -32,38 +33,51 @@ const factories: [string, () => Promise][] = [ ]; describe.each(factories)('%s models refresh coordination', (_name, createRepo) => { - test('claims atomically, backs failures off exponentially, and lets force bypass cooldown', async () => { + test('claims atomically, applies one backoff schedule, and lets force bypass cooldown', async () => { const repo = await createRepo(); await repo.upstreams.save(record); - const now = 1_800_000_000_000; + let now = 1_800_000_000_000; - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'first', now, now - 900_000, false)).resolves.toBe(true); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now, now - 900_000, false)).resolves.toBe(false); - await repo.upstreams.completeModelsRefreshFailure(record.id, 'first', now); + const first = await repo.upstreams.claimModelsRefresh(record.id, generation, 'claim-0', now, now - 900_000, false); + expect(first).toEqual({ failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now, now - 900_000, false)).resolves.toBeNull(); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'early', now + 59_999, now - 900_000, false)).resolves.toBe(false); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'second', now + 60_000, now - 840_000, false)).resolves.toBe(true); - await repo.upstreams.completeModelsRefreshFailure(record.id, 'second', now + 60_000); + const delays = [1, 2, 4, 8, 16, 32, 60, 60].map(minutes => minutes * 60_000); + let claim = first!; + for (const [index, delay] of delays.entries()) { + const retryAt = modelsRefreshRetryAt(now, claim.failureCount); + expect(retryAt - now).toBe(delay); + await repo.upstreams.completeModelsRefreshFailure(record.id, `claim-${index}`, claim.failureCount + 1, retryAt); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, `early-${index}`, retryAt - 1, retryAt - 900_001, false)).resolves.toBeNull(); + now = retryAt; + claim = (await repo.upstreams.claimModelsRefresh(record.id, generation, `claim-${index + 1}`, now, now - 900_000, false))!; + expect(claim.failureCount).toBe(index + 1); + } - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'second-early', now + 179_999, now - 720_000, false)).resolves.toBe(false); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'forced', now + 60_001, now - 839_999, true)).resolves.toBe(true); + const blockedUntil = modelsRefreshRetryAt(now, claim.failureCount); + await repo.upstreams.completeModelsRefreshFailure(record.id, `claim-${delays.length}`, claim.failureCount + 1, blockedUntil); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'forced', now + 1, now - 899_999, true)).resolves.toEqual({ failureCount: delays.length + 1 }); await repo.upstreams.completeModelsRefreshSuccess(record.id, 'forced'); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'after-success', now + 60_002, now - 839_998, false)).resolves.toBe(true); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'after-success', now + 2, now - 899_998, false)).resolves.toEqual({ failureCount: 0 }); }); - test('recovers abandoned claims and fences stale generations and completions', async () => { + test('recovers abandoned claims and fences tokens, timestamps, and config', async () => { const repo = await createRepo(); await repo.upstreams.save(record); const now = 1_800_000_000_000; - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'abandoned', now, now - 900_000, false)).resolves.toBe(true); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'replacement', now + 900_001, now + 1, false)).resolves.toBe(true); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'abandoned', now, now - 900_000, false)).resolves.toEqual({ failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'replacement', now + 900_001, now + 1, false)).resolves.toEqual({ failureCount: 0 }); await repo.upstreams.completeModelsRefreshSuccess(record.id, 'abandoned'); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now + 900_002, now + 2, false)).resolves.toBe(false); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now + 900_002, now + 2, false)).resolves.toBeNull(); - const next = { ...record, updatedAt: '2026-08-01T00:01:00.000Z' }; + const next = { ...record, config: { tenant: 'next' } }; await repo.upstreams.saveClearingModelsCache(next); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'old-generation', now + 900_003, now + 3, false)).resolves.toBe(false); - await expect(repo.upstreams.claimModelsRefresh(record.id, { ...generation, updatedAt: next.updatedAt }, 'current', now + 900_003, now + 3, false)).resolves.toBe(true); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'old-config', now + 900_003, now + 3, false)).resolves.toBeNull(); + await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'current', now + 900_003, now + 3, false)).resolves.toEqual({ failureCount: 0 }); + + const newer = { ...next, updatedAt: '2026-08-01T00:01:00.000Z' }; + await repo.upstreams.saveClearingModelsCache(newer); + await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'old-time', now + 900_004, now + 4, false)).resolves.toBeNull(); }); }); diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index 431c9d784..f034ae90d 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -1,7 +1,7 @@ import type { GatewayProvider } from './registry.ts'; import { getRepo } from '../../repo/index.ts'; import { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; -import { MODELS_REFRESH_CLAIM_LEASE_MS } from '../../repo/models-refresh-contract.ts'; +import { MODELS_REFRESH_CLAIM_LEASE_MS, modelsRefreshRetryAt } from '../../repo/models-refresh-contract.ts'; import { serializeStoredConfig } from '../../repo/upstream-json.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { Fetcher, ProviderModel } from '@floway-dev/provider'; @@ -53,13 +53,14 @@ const runFetch = async ( instance: GatewayProvider, fetcher: Fetcher, key: string, + token: string, loadProvidedModels?: () => Promise, ): Promise => { const generation = instance.modelsCacheGeneration; try { const models = [...await (loadProvidedModels?.() ?? instance.instance.getProvidedModels(fetcher))]; const entry = { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models, lastError: null }; - const persisted = await getRepo().upstreams.saveModelsCache(key, generation, entry); + const persisted = await getRepo().upstreams.saveClaimedModelsCache(key, generation, token, entry); // The instance carries the row as it was read at request start, and a // request reaches this function more than once -- once per alias target // resolved. Writing the entry back keeps every later read in the request @@ -69,7 +70,7 @@ const runFetch = async ( return models; } catch (err) { const lastError = { message: errorMessage(err), at: Date.now() }; - const persisted = await getRepo().upstreams.saveModelsCacheError(key, generation, lastError); + const persisted = await getRepo().upstreams.saveClaimedModelsCacheError(key, generation, token, lastError); if (persisted) { if (instance.modelsCache) instance.modelsCache.lastError = lastError; else instance.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError }; @@ -95,15 +96,17 @@ const runClaimedFetch = async ( now - MODELS_REFRESH_CLAIM_LEASE_MS, force, ); - if (!claimed) return null; + if (claimed === null) return null; try { - const models = await runFetch(instance, fetcher, instance.upstreamId, loadProvidedModels); + const models = await runFetch(instance, fetcher, instance.upstreamId, token, loadProvidedModels); await repo.upstreams.completeModelsRefreshSuccess(instance.upstreamId, token); return models; } catch (error) { try { - await repo.upstreams.completeModelsRefreshFailure(instance.upstreamId, token, Date.now()); + const failureCount = claimed.failureCount + 1; + const now = Date.now(); + await repo.upstreams.completeModelsRefreshFailure(instance.upstreamId, token, failureCount, modelsRefreshRetryAt(now, claimed.failureCount)); } catch (backoffError) { throw new AggregateError([error, backoffError], errorMessage(error)); } diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index 9a6e894b3..a5ca0734c 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -3,7 +3,6 @@ import { SqlExpirationSweepsRepo } from './expiration-sweeps-sql.ts'; import { normalizeFlagOverrides } from './flag-overrides.ts'; import { decodeAliasTargets, decodeAnnouncedMetadata, encodeAliasTargets, encodeAnnouncedMetadata } from './model-alias-codecs.ts'; import { MODEL_CATALOG_REVISION } from './models-cache-contract.ts'; -import { MODELS_REFRESH_BACKOFF_BASE_MS, MODELS_REFRESH_BACKOFF_CAP_MS, MODELS_REFRESH_BACKOFF_EXPONENT_CAP } from './models-refresh-contract.ts'; import { normalizeProxyFallbackList } from './proxy-fallback-list.ts'; import { SqlResponsesItemsRepo, SqlResponsesSnapshotsRepo } from './responses-state-sql.ts'; import { generateSessionToken } from './session-tokens.ts'; @@ -979,8 +978,31 @@ class SqlUpstreamRepo implements UpstreamRepo { return (result.meta.changes ?? 0) > 0; } - async claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise { + async saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { + const rawConfig = await this.modelsCacheWriteConfig(id, generation); + if (rawConfig === null) return false; + const result = await this.db + .prepare("UPDATE upstreams SET models_cache_json = ? WHERE id = ? AND updated_at = ? AND config_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") + .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, generation.updatedAt, rawConfig, token) + .run(); + return (result.meta.changes ?? 0) > 0; + } + + async saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise { + const rawConfig = await this.modelsCacheWriteConfig(id, generation); + if (rawConfig === null) return false; + const coldFailure = encodeUpstreamModelsCache({ revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }); const result = await this.db + .prepare("UPDATE upstreams SET models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END WHERE id = ? AND updated_at = ? AND config_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") + .bind(coldFailure, JSON.stringify(error), id, generation.updatedAt, rawConfig, token) + .run(); + return (result.meta.changes ?? 0) > 0; + } + + async claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise<{ failureCount: number } | null> { + const rawConfig = await this.modelsCacheWriteConfig(id, generation); + if (rawConfig === null) return null; + const row = await this.db .prepare( `UPDATE upstreams SET models_refresh_json = json_object( @@ -989,7 +1011,7 @@ class SqlUpstreamRepo implements UpstreamRepo { 'claimToken', ?, 'claimedAt', ? ) - WHERE id = ? AND updated_at = ? AND ( + WHERE id = ? AND updated_at = ? AND config_json = ? AND ( ? = 1 OR models_refresh_json IS NULL OR ( @@ -999,11 +1021,12 @@ class SqlUpstreamRepo implements UpstreamRepo { OR json_extract(models_refresh_json, '$.claimedAt') <= ? ) ) - )`, + ) + RETURNING json_extract(models_refresh_json, '$.failCount') AS fail_count`, ) - .bind(token, now, id, generation.updatedAt, sqliteBoolean(force), now, staleClaimedBefore) - .run(); - return (result.meta.changes ?? 0) > 0; + .bind(token, now, id, generation.updatedAt, rawConfig, sqliteBoolean(force), now, staleClaimedBefore) + .first<{ fail_count: number }>(); + return row === null ? null : { failureCount: row.fail_count }; } async completeModelsRefreshSuccess(id: string, token: string): Promise { @@ -1013,22 +1036,19 @@ class SqlUpstreamRepo implements UpstreamRepo { .run(); } - async completeModelsRefreshFailure(id: string, token: string, now: number): Promise { + async completeModelsRefreshFailure(id: string, token: string, failureCount: number, retryAt: number): Promise { await this.db .prepare( `UPDATE upstreams SET models_refresh_json = json_object( - 'failCount', coalesce(json_extract(models_refresh_json, '$.failCount'), 0) + 1, - 'retryAt', ? + min( - ? * (1 << min(coalesce(json_extract(models_refresh_json, '$.failCount'), 0), ?)), - ? - ), + 'failCount', ?, + 'retryAt', ?, 'claimToken', NULL, 'claimedAt', NULL ) WHERE id = ? AND json_extract(models_refresh_json, '$.claimToken') = ?`, ) - .bind(now, MODELS_REFRESH_BACKOFF_BASE_MS, MODELS_REFRESH_BACKOFF_EXPONENT_CAP, MODELS_REFRESH_BACKOFF_CAP_MS, id, token) + .bind(failureCount, retryAt, id, token) .run(); } diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index 87efe89a1..d88a5123d 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -267,9 +267,15 @@ export interface UpstreamRepo { // cannot publish models or errors under newer credentials/configuration. saveModelsCache(id: string, generation: ModelsCacheGeneration, cache: Omit): Promise; saveModelsCacheError(id: string, generation: ModelsCacheGeneration, error: NonNullable): Promise; - claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise; + saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise; + saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise; + claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise; completeModelsRefreshSuccess(id: string, token: string): Promise; - completeModelsRefreshFailure(id: string, token: string, now: number): Promise; + completeModelsRefreshFailure(id: string, token: string, failureCount: number, retryAt: number): Promise; +} + +export interface ModelsRefreshClaim { + failureCount: number; } export interface ModelsCacheGeneration { From 6109ee0fa1f8ed0f360fe07682d9230ec67fbe38 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 03:55:07 +0800 Subject: [PATCH 16/46] fix(gateway): keep synchronous warms within backoff Add a blocking non-forced warm operation that joins in-flight work and respects persisted cooldown, while reserving forced claim bypass for the explicit Fetch Models action. Repair control-plane mocks and prove OAuth responses wait for their post-credential warm. --- .../data-transfer/routes_test.ts | 2 +- .../upstreams/copilot-device-login_test.ts | 49 +++++++++++++++++-- .../data-plane/providers/models-cache_test.ts | 23 ++++++++- .../control-plane/shared/warm-models-cache.ts | 4 +- .../src/data-plane/providers/models-cache.ts | 16 ++++++ 5 files changed, 87 insertions(+), 7 deletions(-) diff --git a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts index 54b12f373..9b8e36cf7 100644 --- a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts @@ -8,7 +8,7 @@ import { expect, test, vi } from 'vitest'; // path's own behavior (upserts, identity validation, etc.) is what the tests // exercise — the warm itself has dedicated coverage in models-cache_test.ts. vi.mock('../../../src/data-plane/providers/models-cache.ts', () => ({ - fetchUpstreamModelsCached: () => Promise.resolve([]), + warmUpstreamModels: () => Promise.resolve([]), })); import { exportData, importData } from '../../../src/control-plane/data-transfer/routes.ts'; diff --git a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts index 400119e54..1222689ca 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts @@ -1,12 +1,17 @@ -import { afterEach, test, vi } from 'vitest'; +import { afterEach, expect, test, vi } from 'vitest'; // Copilot OAuth poll handlers warm the model cache after rotating the PAT. The // cache behavior has dedicated coverage; these route tests isolate credential // exchange and persistence. -const modelsCacheMock = vi.hoisted<{ error: Error | null }>(() => ({ error: null })); +const modelsCacheMock = vi.hoisted<{ calls: number; error: Error | null; pending: Promise | null }>(() => ({ calls: 0, error: null, pending: null })); vi.mock('../../../src/data-plane/providers/models-cache.ts', () => ({ - fetchUpstreamModelsCached: () => modelsCacheMock.error ? Promise.reject(modelsCacheMock.error) : Promise.resolve([]), + warmUpstreamModels: async () => { + modelsCacheMock.calls++; + if (modelsCacheMock.pending) await modelsCacheMock.pending; + if (modelsCacheMock.error) throw modelsCacheMock.error; + return []; + }, clearInFlightForTesting: () => {}, })); @@ -27,7 +32,45 @@ const githubAccessToken = (accessToken: string) => ({ }); afterEach(() => { + modelsCacheMock.calls = 0; modelsCacheMock.error = null; + modelsCacheMock.pending = null; +}); + +test('/api/upstreams/copilot/oauth/device-login/poll waits for the post-OAuth model warm', async () => { + const { adminSession } = await setupAppTest(); + let releaseWarm: (() => void) | null = null; + modelsCacheMock.pending = new Promise(resolve => { releaseWarm = resolve; }); + + await withMockedFetch( + request => { + const url = new URL(request.url); + if (url.hostname === 'github.com' && url.pathname === '/login/oauth/access_token') return jsonResponse(githubAccessToken('ghu_blocking_warm')); + if (url.hostname === 'api.github.com' && url.pathname === '/user') return jsonResponse(githubUser); + if (url.hostname === 'api.github.com' && url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ + token: 'ct_blocking_warm', + expires_at: Math.floor(Date.now() / 1000) + 1500, + refresh_in: 1200, + endpoints: { api: 'https://api.business.githubcopilot.com' }, + }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + let settled = false; + const responsePromise = requestApp('/api/upstreams/copilot/oauth/device-login/poll', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-floway-session': adminSession }, + body: JSON.stringify({ record: copilotBlueprintEnvelope, deviceCode: 'device' }), + }).finally(() => { settled = true; }); + + await vi.waitFor(() => expect(modelsCacheMock.calls).toBe(1)); + expect(settled).toBe(false); + releaseWarm!(); + expect((await responsePromise).status).toBe(200); + }, + ); }); test('/api/upstreams/copilot/oauth/device-login/start starts GitHub device flow', async () => { diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 5410a0fb6..3098d50f0 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { clearInFlightForTesting, fetchUpstreamModelsCached, MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; +import { clearInFlightForTesting, fetchUpstreamModelsCached, MODEL_CATALOG_REVISION, warmUpstreamModels } from '../../../src/data-plane/providers/models-cache.ts'; import type { GatewayProvider } from '../../../src/data-plane/providers/registry.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { SqlRepo } from '../../../src/repo/sql.ts'; @@ -223,6 +223,27 @@ describe('fetchUpstreamModelsCached', () => { expect(fetchFn).toHaveBeenCalledTimes(2); }); + test('synchronous warm respects backoff while explicit force bypasses it', async () => { + const repo = await setupRepo(); + const now = 1_800_000_000_000; + vi.spyOn(Date, 'now').mockReturnValue(now); + const failing = stubInstance(async () => { throw new Error('boom'); }, null, CACHE_GENERATION, 'backoff-source'); + const scheduled = captureScheduled(); + await fetchUpstreamModelsCached(failing, { scheduler: scheduled.scheduler, fetcher: directFetcher }); + await expect(scheduled.promises[0]).rejects.toThrow('boom'); + clearInFlightForTesting(); + + const fetchFn = vi.fn(async () => [aModel('recovered')]); + const cache = await storedCache(repo); + const warming = stubInstance(fetchFn, cache, CACHE_GENERATION, 'warm-during-backoff'); + await expect(warmUpstreamModels(warming, directFetcher)).resolves.toEqual([]); + expect(fetchFn).not.toHaveBeenCalled(); + + await expect(fetchUpstreamModelsCached(warming, { scheduler: () => {}, fetcher: directFetcher, force: true })) + .resolves.toEqual([aModel('recovered')]); + expect(fetchFn).toHaveBeenCalledTimes(1); + }); + test('a superseded generation neither joins nor overwrites the current catalog', async () => { const repo = await setupRepo(); let resolveOld: ((models: ProviderModel[]) => void) | null = null; diff --git a/packages/gateway/src/control-plane/shared/warm-models-cache.ts b/packages/gateway/src/control-plane/shared/warm-models-cache.ts index b426697a4..ea562bbde 100644 --- a/packages/gateway/src/control-plane/shared/warm-models-cache.ts +++ b/packages/gateway/src/control-plane/shared/warm-models-cache.ts @@ -1,6 +1,6 @@ import type { Context } from 'hono'; -import { fetchUpstreamModels } from '../../data-plane/providers/models-cache.ts'; +import { warmUpstreamModels } from '../../data-plane/providers/models-cache.ts'; import { createProvider } from '../../data-plane/providers/registry.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; import { getRepo } from '../../repo/index.ts'; @@ -22,7 +22,7 @@ export const warmModelsCache = async (record: UpstreamRecord, c: Context): Promi const provider = createProvider(record); const fetcher = (await createPerRequestFetcher(getRuntimeLocation(c.req.raw)))(record.id); try { - await fetchUpstreamModels(provider, fetcher); + await warmUpstreamModels(provider, fetcher); } catch (error) { logInfo('warm_models_cache_failed', { upstream_id: record.id, error: errorMessage(error) }); } diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index f034ae90d..71f7a00a4 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -137,6 +137,22 @@ export const fetchUpstreamModels = async ( return models; }; +export const warmUpstreamModels = async ( + instance: GatewayProvider, + fetcher: Fetcher, + loadProvidedModels?: () => Promise, +): Promise => { + const key = inFlightKey(instance); + const existing = inFlight.get(key); + if (existing) { + const joined = await existing; + return joined ?? instance.modelsCache?.models ?? []; + } + + const models = await memoInFlight(key, () => runClaimedFetch(instance, fetcher, false, loadProvidedModels)); + return models ?? instance.modelsCache?.models ?? []; +}; + export const triggerUpstreamModelsFetch = ( instance: GatewayProvider, scheduler: BackgroundScheduler, From 4f50912b86a1f3fed09a0ad28866bfe1b2ea5fff Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 03:58:32 +0800 Subject: [PATCH 17/46] test(gateway): exercise existing-upstream OAuth warm Use the credential-update path that actually persists and warms an existing upstream before asserting the OAuth response remains pending. --- .../control-plane/upstreams/copilot-device-login_test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts index 1222689ca..1a78b5e2b 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts @@ -38,7 +38,10 @@ afterEach(() => { }); test('/api/upstreams/copilot/oauth/device-login/poll waits for the post-OAuth model warm', async () => { - const { adminSession } = await setupAppTest(); + const { adminSession, githubAccount, repo } = await setupAppTest(); + const existing = buildCopilotUpstreamRecord(githubAccount, { id: 'up_blocking_warm' }); + await repo.upstreams.deleteAll(); + await repo.upstreams.save(existing); let releaseWarm: (() => void) | null = null; modelsCacheMock.pending = new Promise(resolve => { releaseWarm = resolve; }); @@ -62,7 +65,7 @@ test('/api/upstreams/copilot/oauth/device-login/poll waits for the post-OAuth mo const responsePromise = requestApp('/api/upstreams/copilot/oauth/device-login/poll', { method: 'POST', headers: { 'content-type': 'application/json', 'x-floway-session': adminSession }, - body: JSON.stringify({ record: copilotBlueprintEnvelope, deviceCode: 'device' }), + body: JSON.stringify({ record: { ...copilotBlueprintEnvelope, id: existing.id }, deviceCode: 'device' }), }).finally(() => { settled = true; }); await vi.waitFor(() => expect(modelsCacheMock.calls).toBe(1)); From b84880a15ebe79a67d3b82ea8f04c822e0c3fa1c Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:00:27 +0800 Subject: [PATCH 18/46] fix(gateway): detach scheduled model refresh I/O Submit scheduled model refresh promises to each runtime background scheduler without awaiting upstream model-list work, and run independent maintenance sweeps concurrently. Prove the scheduled trigger returns while a deferred upstream fetch remains owned by the runtime lifecycle. --- apps/platform-cloudflare/entry.ts | 2 +- apps/platform-node/entry.ts | 5 +- .../scheduled/models-refresh_test.ts | 51 +++++++++++++------ packages/gateway/src/scheduled.ts | 21 +++++--- .../gateway/src/scheduled/models-refresh.ts | 10 ++-- 5 files changed, 59 insertions(+), 30 deletions(-) diff --git a/apps/platform-cloudflare/entry.ts b/apps/platform-cloudflare/entry.ts index 3eca4b32e..00da2c4b6 100644 --- a/apps/platform-cloudflare/entry.ts +++ b/apps/platform-cloudflare/entry.ts @@ -25,6 +25,6 @@ export default { scheduled(_controller: unknown, env: CloudflareEnv, ctx: ExecutionContext) { const { db } = bootstrapCloudflarePlatform(env); initRepo(new SqlRepo(db)); - ctx.waitUntil(runScheduledMaintenance()); + ctx.waitUntil(runScheduledMaintenance('SCHEDULED', promise => ctx.waitUntil(promise))); }, }; diff --git a/apps/platform-node/entry.ts b/apps/platform-node/entry.ts index 7c38b7ec3..c9734d88d 100644 --- a/apps/platform-node/entry.ts +++ b/apps/platform-node/entry.ts @@ -72,8 +72,11 @@ initRepo(new SqlRepo(db)); // 30s delay keeps the very first request after boot from racing the sweep. // unref() on both timers lets the process exit cleanly on SIGINT. const STARTUP_DELAY_MS = 30 * 1000; +const scheduleBackground = (promise: Promise): void => { + promise.catch(err => console.error('[scheduled-maintenance background]', err)); +}; const sweep = (): void => { - runScheduledMaintenance(scheduledRuntimeLocation).catch(err => { + runScheduledMaintenance(scheduledRuntimeLocation, scheduleBackground).catch(err => { console.error('[scheduled-maintenance] sweep failed:', err); }); }; diff --git a/packages/gateway/__tests__/scheduled/models-refresh_test.ts b/packages/gateway/__tests__/scheduled/models-refresh_test.ts index e135673f6..66027c87a 100644 --- a/packages/gateway/__tests__/scheduled/models-refresh_test.ts +++ b/packages/gateway/__tests__/scheduled/models-refresh_test.ts @@ -1,43 +1,64 @@ -import { expect, test } from 'vitest'; +import { expect, test, vi } from 'vitest'; import { initRepo } from '../../src/repo/index.ts'; import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; import { refreshModelsCaches } from '../../src/scheduled/models-refresh.ts'; import { InMemoryRepo } from '../repo/memory.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; +import { withMockedFetch } from '@floway-dev/test-utils'; -const azure = (id: string, enabled: boolean): UpstreamRecord => ({ +const custom = (id: string, enabled: boolean): UpstreamRecord => ({ id, - kind: 'azure', + kind: 'custom', name: id, enabled, sortOrder: 0, createdAt: '2026-08-01T00:00:00.000Z', updatedAt: '2026-08-01T00:00:00.000Z', config: { - endpoint: 'https://example.openai.azure.com/openai/v1', - apiKey: 'azkey', - models: [{ upstreamModelId: `${id}-wire`, publicModelId: `${id}-public`, endpoints: { chatCompletions: {} } }], + baseUrl: `https://${id}.example.com`, + authStyle: 'bearer', + apiKey: 'key', + endpoints: { chatCompletions: {} }, + ingressHeadersRules: [], }, state: null, modelsCache: null, flagOverrides: {}, disabledPublicModelIds: [], - proxyFallbackList: [], + proxyFallbackList: [{ id: 'direct_fetch' }], modelPrefix: null, hue: 210, }); -test('scheduled maintenance triggers cold enabled upstreams and waits for their background work', async () => { +test('scheduled maintenance submits enabled refreshes without waiting for model I/O', async () => { const repo = new InMemoryRepo(); initRepo(repo); - await repo.upstreams.save(azure('enabled', true)); - await repo.upstreams.save(azure('disabled', false)); + await repo.upstreams.save(custom('enabled', true)); + await repo.upstreams.save(custom('disabled', false)); + let resolveFetch: ((response: Response) => void) | null = null; + const requested: string[] = []; - await refreshModelsCaches('SCHEDULED'); + await withMockedFetch( + request => { + requested.push(new URL(request.url).hostname); + return new Promise(resolve => { resolveFetch = resolve; }); + }, + async () => { + const background: Promise[] = []; + await refreshModelsCaches('SCHEDULED', promise => { background.push(promise); }); - const enabled = await repo.upstreams.getById('enabled'); - expect(enabled?.modelsCache?.revision).toBe(MODEL_CATALOG_REVISION); - expect(enabled?.modelsCache?.models.map(model => model.id)).toEqual(['enabled-public']); - expect((await repo.upstreams.getById('disabled'))?.modelsCache).toBeNull(); + expect(background).toHaveLength(1); + await vi.waitFor(() => expect(requested).toEqual(['enabled.example.com'])); + expect((await repo.upstreams.getById('enabled'))?.modelsCache).toBeNull(); + + resolveFetch!(Response.json({ data: [{ id: 'enabled-public' }] })); + await background[0]; + expect((await repo.upstreams.getById('enabled'))?.modelsCache).toMatchObject({ + revision: MODEL_CATALOG_REVISION, + models: [{ id: 'enabled-public' }], + }); + expect((await repo.upstreams.getById('disabled'))?.modelsCache).toBeNull(); + }, + ); }); diff --git a/packages/gateway/src/scheduled.ts b/packages/gateway/src/scheduled.ts index a847309ae..0e9e32aa2 100644 --- a/packages/gateway/src/scheduled.ts +++ b/packages/gateway/src/scheduled.ts @@ -1,7 +1,7 @@ import { sweepExpirations } from './scheduled/expiration-sweeps.ts'; import { refreshModelsCaches } from './scheduled/models-refresh.ts'; import { collectSpilledFiles } from './scheduled/spilled-files.ts'; -import { getImageCacheStore } from '@floway-dev/platform'; +import { getImageCacheStore, type BackgroundScheduler } from '@floway-dev/platform'; const runSweep = async (name: string, fn: () => Promise): Promise => { try { @@ -13,10 +13,19 @@ const runSweep = async (name: string, fn: () => Promise): Promise => { +const defaultBackgroundScheduler: BackgroundScheduler = promise => { + promise.catch(error => console.error('[scheduled] background task failed', error)); +}; + +export const runScheduledMaintenance = async ( + runtimeLocation = 'SCHEDULED', + backgroundScheduler: BackgroundScheduler = defaultBackgroundScheduler, +): Promise => { const nowMs = Date.now(); - await runSweep('models.refresh', () => refreshModelsCaches(runtimeLocation)); - await runSweep('expirations.sweep', () => sweepExpirations(nowMs)); - await runSweep('spilledFiles.collect', () => collectSpilledFiles(nowMs)); - await runSweep('imageCacheStore.sweepExpired', () => getImageCacheStore().sweepExpired(nowMs)); + await Promise.all([ + runSweep('models.refresh', () => refreshModelsCaches(runtimeLocation, backgroundScheduler)), + runSweep('expirations.sweep', () => sweepExpirations(nowMs)), + runSweep('spilledFiles.collect', () => collectSpilledFiles(nowMs)), + runSweep('imageCacheStore.sweepExpired', () => getImageCacheStore().sweepExpired(nowMs)), + ]); }; diff --git a/packages/gateway/src/scheduled/models-refresh.ts b/packages/gateway/src/scheduled/models-refresh.ts index 0b0c2e7f0..c441f175b 100644 --- a/packages/gateway/src/scheduled/models-refresh.ts +++ b/packages/gateway/src/scheduled/models-refresh.ts @@ -2,20 +2,16 @@ import { fetchUpstreamModelsCached } from '../data-plane/providers/models-cache. import { createProvider } from '../data-plane/providers/registry.ts'; import { createPerRequestFetcher } from '../dial/per-request.ts'; import { getRepo } from '../repo/index.ts'; +import type { BackgroundScheduler } from '@floway-dev/platform'; -export const refreshModelsCaches = async (runtimeLocation: string): Promise => { +export const refreshModelsCaches = async (runtimeLocation: string, scheduler: BackgroundScheduler): Promise => { const upstreams = (await getRepo().upstreams.list()).filter(upstream => upstream.enabled); const fetcherForUpstream = await createPerRequestFetcher(runtimeLocation, upstreams); - const pending: Promise[] = []; for (const upstream of upstreams) { await fetchUpstreamModelsCached(createProvider(upstream), { - scheduler: promise => { pending.push(promise); }, + scheduler, fetcher: fetcherForUpstream(upstream.id), }); } - - const settled = await Promise.allSettled(pending); - const errors = settled.flatMap(result => result.status === 'rejected' ? [result.reason] : []); - if (errors.length > 0) throw new AggregateError(errors, `${errors.length} model cache refreshes failed`); }; From 6fbc0399802afb4ad714b7e359944d5c93afe98e Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:02:12 +0800 Subject: [PATCH 19/46] fix(gateway): scope cached errors to addressable models Do not attribute a persisted refresh failure to a model id that the upstream prefix policy cannot address. --- .../data-plane/providers/resolution_test.ts | 28 +++++++++++++++++++ .../src/data-plane/providers/resolution.ts | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index 942d4c12b..3768d947b 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -235,6 +235,34 @@ test('enumerateRealModelCandidates rejects a model id disabled on that upstream assertEquals(disabled.candidates.length, 0); }); +test('a recorded refresh failure is irrelevant when the prefix policy cannot address the model id', async () => { + const { repo } = await setupAppTest(); + await repo.upstreams.deleteAll(); + await repo.upstreams.save(buildCustomUpstreamRecord({ + id: 'up_prefixed_failure', + name: 'Prefixed failure', + modelPrefix: { prefix: 'tenant/', addressable: ['prefixed'], listed: ['prefixed'] }, + config: { baseUrl: 'https://prefixed-failure.example.com', authStyle: 'bearer', apiKey: 'sk-x', endpoints: { chatCompletions: {} }, ingressHeadersRules: [] }, + })); + + await withMockedFetch( + request => { + if (new URL(request.url).hostname === 'prefixed-failure.example.com') return jsonResponse({ error: 'down' }, 502); + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const resolved = await enumerateModelCandidates({ + upstreamIds: null, + model: 'unprefixed-model', + kind: 'chat', + scheduler: testScheduler, + runtimeLocation: 'TEST', + }); + expect(resolved.failedUpstreams).toEqual([]); + }, + ); +}); + // A persisted refresh error must not hide healthy siblings. The broken // upstream's display name flows back via `failedUpstreams` while its empty // or last-known-good snapshot stays independent of the current request. diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index 68f043b51..d0142c3e8 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -41,7 +41,7 @@ const enumerateOneUpstreamCandidates = async ( else if (form === 'prefixed' && modelId.startsWith(cfg.prefix)) lookupIds.push(modelId.slice(cfg.prefix.length)); } } - if (lookupIds.length === 0) return { candidates: [], sawAnyId: false, modelsError: provider.modelsCache?.lastError != null }; + if (lookupIds.length === 0) return { candidates: [], sawAnyId: false, modelsError: false }; const providedModels = await fetchUpstreamModelsCached(provider, { scheduler, fetcher }); const disabled = new Set(provider.disabledPublicModelIds); From 3d192e3bbe7e273ff910d010f870d668026fdb98 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:03:57 +0800 Subject: [PATCH 20/46] test(gateway): make fixture model warming explicit Keep requestApp transparent, move production-equivalent blocking warm behavior behind an explicitly named helper, and select it at model-dependent test imports. Add an HTTP regression proving a cold model listing returns before its triggered upstream fetch settles and later persists the result. --- .../control-plane/models/routes_test.ts | 2 +- .../__tests__/data-plane/audio/http_test.ts | 2 +- .../data-plane/codex/routes_images_test.ts | 2 +- .../data-plane/completions/http_test.ts | 2 +- .../data-plane/embeddings/http_test.ts | 2 +- .../__tests__/data-plane/images/http_test.ts | 2 +- .../data-plane/models/gemini_test.ts | 2 +- .../__tests__/data-plane/models/http_test.ts | 30 +++++++++++++++++-- .../__tests__/data-plane/rerank/serve_test.ts | 2 +- .../shared/passthrough-serve_test.ts | 2 +- packages/gateway/__tests__/test-utils/app.ts | 26 +++++----------- 11 files changed, 45 insertions(+), 29 deletions(-) diff --git a/packages/gateway/__tests__/control-plane/models/routes_test.ts b/packages/gateway/__tests__/control-plane/models/routes_test.ts index 4c70d446a..7fa8423d2 100644 --- a/packages/gateway/__tests__/control-plane/models/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/models/routes_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { buildCustomUpstreamRecord, copilotModels, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, copilotModels, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; import { assert, assertEquals, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/audio/http_test.ts b/packages/gateway/__tests__/data-plane/audio/http_test.ts index 7e0117177..3db7f001a 100644 --- a/packages/gateway/__tests__/data-plane/audio/http_test.ts +++ b/packages/gateway/__tests__/data-plane/audio/http_test.ts @@ -1,7 +1,7 @@ import { test, vi } from 'vitest'; import type { InMemoryRepo } from '../../repo/memory.ts'; -import { flushAsyncWork, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { flushAsyncWork, MOCKED_FETCH_EGRESS, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; import type { ModelPricing } from '@floway-dev/protocols/common'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts b/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts index 044552214..12bb2ddfc 100644 --- a/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts +++ b/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import type { InMemoryRepo } from '../../repo/memory.ts'; -import { copilotModels, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { copilotModels, MOCKED_FETCH_EGRESS, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; const PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/wEAAAAASUVORK5CYII='; diff --git a/packages/gateway/__tests__/data-plane/completions/http_test.ts b/packages/gateway/__tests__/data-plane/completions/http_test.ts index a8a766310..21ff80222 100644 --- a/packages/gateway/__tests__/data-plane/completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/completions/http_test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; import { installDumpStubs } from '../../dump/test-fixtures.ts'; -import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/embeddings/http_test.ts b/packages/gateway/__tests__/data-plane/embeddings/http_test.ts index cb8631568..cde73f4a1 100644 --- a/packages/gateway/__tests__/data-plane/embeddings/http_test.ts +++ b/packages/gateway/__tests__/data-plane/embeddings/http_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; -import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/images/http_test.ts b/packages/gateway/__tests__/data-plane/images/http_test.ts index 61355433b..ad28eb3ca 100644 --- a/packages/gateway/__tests__/data-plane/images/http_test.ts +++ b/packages/gateway/__tests__/data-plane/images/http_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; -import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, MOCKED_FETCH_EGRESS, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/models/gemini_test.ts b/packages/gateway/__tests__/data-plane/models/gemini_test.ts index e13dfaad6..40389ac4c 100644 --- a/packages/gateway/__tests__/data-plane/models/gemini_test.ts +++ b/packages/gateway/__tests__/data-plane/models/gemini_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { buildCustomUpstreamRecord, copilotModels, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, copilotModels, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/models/http_test.ts b/packages/gateway/__tests__/data-plane/models/http_test.ts index 9dec995f6..940e9c207 100644 --- a/packages/gateway/__tests__/data-plane/models/http_test.ts +++ b/packages/gateway/__tests__/data-plane/models/http_test.ts @@ -1,6 +1,6 @@ -import { test } from 'vitest'; +import { expect, test, vi } from 'vitest'; -import { buildCopilotUpstreamRecord, buildCustomUpstreamRecord, copilotModels, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCopilotUpstreamRecord, buildCustomUpstreamRecord, copilotModels, flushAsyncWork, requestApp as requestAppCold, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; import type { ModelKind } from '@floway-dev/protocols/common'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals } from '@floway-dev/test-utils'; @@ -15,6 +15,32 @@ const SECOND_ACCOUNT = { }, }; +test('/v1/models returns a cold snapshot before its triggered upstream fetch settles', async () => { + const { apiKey, repo } = await setupAppTest(); + await repo.upstreams.deleteAll(); + await repo.upstreams.save(buildCustomUpstreamRecord()); + let resolveFetch: ((response: Response) => void) | null = null; + + await withMockedFetch( + () => new Promise(resolve => { resolveFetch = resolve; }), + async () => { + let responseSettled = false; + const responsePromise = requestAppCold('/v1/models', { headers: { 'x-api-key': apiKey.key } }) + .then(response => { responseSettled = true; return response; }); + + await vi.waitFor(() => expect(resolveFetch).not.toBeNull()); + await vi.waitFor(() => expect(responseSettled).toBe(true)); + const response = await responsePromise; + expect(response.status).toBe(200); + expect((await response.json() as { data: unknown[] }).data).toEqual([]); + + resolveFetch!(jsonResponse({ data: [{ id: 'eventual-model' }] })); + await flushAsyncWork(); + expect((await repo.upstreams.getById('up_custom'))?.modelsCache?.models.map(model => model.id)).toEqual(['eventual-model']); + }, + ); +}); + test('/v1/models returns merged model list from Copilot and custom upstreams', async () => { const { repo, apiKey } = await setupAppTest(); diff --git a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts index ef4e067cc..88e62dca8 100644 --- a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import type { Repo } from '../../../src/repo/types.ts'; -import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; import type { ModelPricing, RerankTarget } from '@floway-dev/protocols/common'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts b/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts index fa8e4bec0..36731db65 100644 --- a/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts @@ -17,7 +17,7 @@ import { test, vi } from 'vitest'; -import { buildCustomUpstreamRecord, flushAsyncWork, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/test-utils/app.ts b/packages/gateway/__tests__/test-utils/app.ts index 40159bc02..9a74fa8b0 100644 --- a/packages/gateway/__tests__/test-utils/app.ts +++ b/packages/gateway/__tests__/test-utils/app.ts @@ -1,6 +1,6 @@ import { trackBackground } from './background-tracker.ts'; import { app } from '../../src/app.ts'; -import { clearInFlightForTesting, fetchUpstreamModelsCached } from '../../src/data-plane/providers/models-cache.ts'; +import { clearInFlightForTesting, warmUpstreamModels } from '../../src/data-plane/providers/models-cache.ts'; import { listModelProviders } from '../../src/data-plane/providers/registry.ts'; import type { WebSearchConfig } from '../../src/data-plane/tools/web-search/types.ts'; import { createPerRequestFetcher } from '../../src/dial/per-request.ts'; @@ -9,7 +9,6 @@ import type { ApiKey } from '../../src/repo/types.ts'; import { initBackgroundSchedulerResolver } from '../../src/runtime/background.ts'; import { InMemoryRepo } from '../repo/memory.ts'; import { createInMemoryImageProcessor, initEnv, initExternalResourceFetcher, initFileStore, initImageProcessor, initSocketDial, MemoryFileStore } from '@floway-dev/platform'; -import { PUBLIC_DATA_PLANE_ROUTES } from '@floway-dev/protocols/common'; import type { ProxyFallbackEntry, UpstreamRecord } from '@floway-dev/provider'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; @@ -302,17 +301,14 @@ export function sseResponsesResponse(response: Record): Respons } export async function requestApp(path: string, init: RequestInit): Promise { - const method = init.method?.toUpperCase() ?? 'GET'; - const pathname = new URL(path, 'http://localhost').pathname; - const isModelConsumer = pathname === '/api/models' || Object.values(PUBLIC_DATA_PLANE_ROUTES).some(route => - route.method === method && route.paths.some(template => { - const parameter = template.indexOf('/:'); - return parameter === -1 ? pathname === template : pathname.startsWith(template.slice(0, parameter + 1)); - })); - if (isModelConsumer && globalThis.fetch !== processFetch) await warmModelsForTest(); return await app.request(path, init); } +export const requestAppWithWarmModels = async (path: string, init: RequestInit): Promise => { + if (globalThis.fetch !== processFetch) await warmModelsForTest(); + return await requestApp(path, init); +}; + // App fixtures write upstream rows directly because their fetch mocks are not // installed until the test body runs. Production create/update/OAuth flows // synchronously warm before returning; reproduce that lifecycle immediately @@ -321,14 +317,8 @@ export async function requestApp(path: string, init: RequestInit): Promise => { const providers = await listModelProviders(null); const fetcherForUpstream = await createPerRequestFetcher('TEST'); - const pending: Promise[] = []; - await Promise.all(providers.map(async provider => { - await fetchUpstreamModelsCached(provider, { - scheduler: promise => { pending.push(promise); }, - fetcher: fetcherForUpstream(provider.upstreamId), - }); - })); - await Promise.allSettled(pending); + await Promise.allSettled(providers.map(async provider => + await warmUpstreamModels(provider, fetcherForUpstream(provider.upstreamId)))); }; export function parseSSEText(text: string): Array<{ event: string; data: string }> { From 51ade131bf6b985338bbc87a13c54111723fea42 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:07:53 +0800 Subject: [PATCH 21/46] docs(gateway): align catalog snapshot semantics Remove stale request-time fetch and AbortError narratives, document durable refresh-error reporting and backoff-respecting warm behavior, and delete unreachable ProviderModelsUnavailableError listing branches. --- docs/RESOLUTION.md | 24 ++++++++++--------- .../src/control-plane/models/routes.ts | 7 ------ .../control-plane/shared/warm-models-cache.ts | 3 ++- .../src/data-plane/chat/shared/errors.ts | 8 +++---- .../gateway/src/data-plane/models/gemini.ts | 10 ++------ .../gateway/src/data-plane/models/http.ts | 10 +------- .../gateway/src/data-plane/models/shared.ts | 11 ++++----- .../src/data-plane/providers/catalog.ts | 13 ++++------ .../src/data-plane/providers/resolution.ts | 4 ++-- .../src/data-plane/shared/failed-upstreams.ts | 10 ++++---- .../data-plane/shared/listing/addressable.ts | 20 ++++------------ 11 files changed, 41 insertions(+), 79 deletions(-) diff --git a/docs/RESOLUTION.md b/docs/RESOLUTION.md index 2d576cdd2..5e94755a9 100644 --- a/docs/RESOLUTION.md +++ b/docs/RESOLUTION.md @@ -40,10 +40,11 @@ coordinator and replaces the catalog under the upstream generation fence. A first failure persists an empty catalog with `fetchedAt: 0` and `lastError`, so the error stays observable without making the empty result soft-fresh. -Two control-plane paths deliberately perform synchronous fetches: the explicit -**Fetch models** action, and the warm after create, update, import, or OAuth -credential changes. They share cache persistence and in-flight coordination -with automatic refreshes but bypass automatic-trigger backoff. +Two control-plane paths deliberately wait for refresh coordination: the +explicit **Fetch models** action and the warm after create, update, import, or +OAuth credential changes. Only the explicit action force-claims through an +active cooldown. A warm joins an existing refresh or respects its persisted +backoff, returning the current snapshot when no new attempt is eligible. For every provider model, `modelPrefix.listed` determines its public catalog surface: @@ -75,10 +76,11 @@ upstream id. Consequently: `getModelsFromProviders` returns the merged `InternalModel[]` and `upstreamsByPublicId`, preserving provider enumeration order in the reverse -index. Per-upstream fetches fan out concurrently. `AbortError` propagates; -other failures are collected while healthy upstreams still contribute rows. If -all catalog fetches fail, the last error surfaces. Listing currently does not -expose the partial-failure names. +index. Assembly reads persisted snapshots immediately. Cold or stale reads +submit background refresh work through the runtime scheduler; upstream HTTP, +parse, and transport failures therefore cannot reject the listing request. +Persisted `lastError` values identify affected upstreams while healthy and +last-known-good snapshots continue to contribute rows. ### Listing surfaces @@ -98,9 +100,9 @@ catalog feeds: `toPublicModel` projects an `InternalModel` onto the public DTO. Gemini uses its own projection, Codex synthesizes its client-catalog shape, and the control plane -adds dashboard-only fields. The listing paths and request resolver are separate -consumers of the same SWR cache; listing failures do not feed state into -resolution. +adds dashboard-only fields. Listing and request resolution are separate +consumers of the same persisted snapshots and both report recorded refresh +failures from `modelsCache.lastError`. ## Addressable surfaces diff --git a/packages/gateway/src/control-plane/models/routes.ts b/packages/gateway/src/control-plane/models/routes.ts index d917f629a..2a14675a7 100644 --- a/packages/gateway/src/control-plane/models/routes.ts +++ b/packages/gateway/src/control-plane/models/routes.ts @@ -1,5 +1,4 @@ import { toPublicModel } from '../../data-plane/models/load.ts'; -import { MODEL_LISTING_FAILURE_MESSAGE } from '../../data-plane/models/shared.ts'; import { type AddressableIdEntry, enumerateAddressableModelIds, listedRealModels } from '../../data-plane/shared/listing/addressable.ts'; import { mergeAliasesIntoModels } from '../../data-plane/shared/listing/alias.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; @@ -10,7 +9,6 @@ import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { modelsQuery } from '../schemas.ts'; import type { PublicModel, PublicModelsResponse } from '@floway-dev/protocols/common'; -import { ProviderModelsUnavailableError } from '@floway-dev/provider'; import type { InternalModel, Provider, UpstreamProviderKind } from '@floway-dev/provider'; // Same DTO as the public /models endpoint, plus one dashboard-only field: @@ -153,11 +151,6 @@ export const controlPlaneModels = async (c: CtxWithQuery) => if (e instanceof Error && e.message.startsWith('No upstream provider configured')) { return c.json({ object: 'list', has_more: false, first_id: null, last_id: null, data: [] }); } - // Genuine upstream HTTP/parse failures are squashed to a generic 502 so - // the control plane does not leak provider identity. - if (e instanceof ProviderModelsUnavailableError) { - return c.json({ error: { message: MODEL_LISTING_FAILURE_MESSAGE, type: 'api_error' } }, 502); - } return c.json({ error: { message: e instanceof Error ? e.message : String(e), type: 'api_error' } }, 502); } }; diff --git a/packages/gateway/src/control-plane/shared/warm-models-cache.ts b/packages/gateway/src/control-plane/shared/warm-models-cache.ts index ea562bbde..c680566eb 100644 --- a/packages/gateway/src/control-plane/shared/warm-models-cache.ts +++ b/packages/gateway/src/control-plane/shared/warm-models-cache.ts @@ -17,7 +17,8 @@ const errorMessage = (error: unknown): string => error instanceof Error ? error. // // Returns what the row holds afterwards so the caller can answer with the // freshness this warm produced rather than the snapshot it read before saving. -// Null when the upstream fetch failed and left the row with nothing to report. +// A persisted cold failure is an empty error-bearing cache; null means the row +// disappeared or its generation was superseded before readback. export const warmModelsCache = async (record: UpstreamRecord, c: Context): Promise => { const provider = createProvider(record); const fetcher = (await createPerRequestFetcher(getRuntimeLocation(c.req.raw)))(record.id); diff --git a/packages/gateway/src/data-plane/chat/shared/errors.ts b/packages/gateway/src/data-plane/chat/shared/errors.ts index 65bb54c9f..51629a9f8 100644 --- a/packages/gateway/src/data-plane/chat/shared/errors.ts +++ b/packages/gateway/src/data-plane/chat/shared/errors.ts @@ -2,11 +2,9 @@ import type { ApiErrorResult, PerformanceTelemetryContext } from '@floway-dev/pr // Failures a chat protocol can render before reaching an upstream; unexpected // throws bubble as-is. `failedUpstreams` on model-{missing,unsupported} -// carries the upstream names whose catalog fetch threw during this -// resolution — surfaced parenthetically so the caller can tell a genuine -// "no upstream has this model" miss from a transient outage where the -// upstream that owns the model is currently unreachable. Empty means -// every consulted upstream returned a catalog. +// carries upstream names with a recorded catalog-refresh failure. The error +// may predate this resolution and may accompany a usable last-known-good +// catalog; empty means no consulted snapshot records such a failure. export type ChatServeFailure = | { readonly kind: 'model-missing'; readonly model: string; readonly failedUpstreams: readonly string[] } | { readonly kind: 'model-unsupported'; readonly model: string; readonly failedUpstreams: readonly string[] } diff --git a/packages/gateway/src/data-plane/models/gemini.ts b/packages/gateway/src/data-plane/models/gemini.ts index 6f5f3bf44..353e7787d 100644 --- a/packages/gateway/src/data-plane/models/gemini.ts +++ b/packages/gateway/src/data-plane/models/gemini.ts @@ -1,6 +1,5 @@ import type { Context } from 'hono'; -import { MODEL_LISTING_FAILURE_MESSAGE } from './shared.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; import { effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts'; import { getRepo } from '../../repo/index.ts'; @@ -12,7 +11,6 @@ import { enumerateAddressableModelIds, listedRealModels } from '../shared/listin import { mergeAliasesIntoModels } from '../shared/listing/alias.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ModelPricing } from '@floway-dev/protocols/common'; -import { ProviderModelsUnavailableError } from '@floway-dev/provider'; import type { InternalModel, Fetcher } from '@floway-dev/provider'; type GeminiGenerationMethod = 'generateContent' | 'streamGenerateContent' | 'countTokens'; @@ -58,12 +56,8 @@ const geminiError = (status: number, message: string): Response => { status: status as 400 | 404 | 500 | 502 }, ); -const geminiModelLoadError = (error: unknown): Response => { - if (error instanceof ProviderModelsUnavailableError) { - return geminiError(502, MODEL_LISTING_FAILURE_MESSAGE); - } - return geminiError(502, error instanceof Error ? error.message : String(error)); -}; +const geminiModelLoadError = (error: unknown): Response => + geminiError(502, error instanceof Error ? error.message : String(error)); // Real chat models plus chat-kind alias entries; collision and dedupe ride // on the shared `mergeAliasesIntoModels` helper so /v1beta/models stays in diff --git a/packages/gateway/src/data-plane/models/http.ts b/packages/gateway/src/data-plane/models/http.ts index d1d8fa6e8..8aa06391a 100644 --- a/packages/gateway/src/data-plane/models/http.ts +++ b/packages/gateway/src/data-plane/models/http.ts @@ -6,7 +6,6 @@ import type { Context } from 'hono'; import { loadModels } from './load.ts'; -import { MODEL_LISTING_FAILURE_MESSAGE } from './shared.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; import { effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts'; import { getRepo } from '../../repo/index.ts'; @@ -15,7 +14,6 @@ import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import { isCodexUserAgent } from '../codex/catalog.ts'; import { loadCodexCatalog } from '../codex/models.ts'; import type { PublicModelsResponse } from '@floway-dev/protocols/common'; -import { ProviderModelsUnavailableError } from '@floway-dev/provider'; // Anthropic's official /v1/models shape — `{data, first_id, has_more, // last_id}` with `ModelInfo` rows — served to Claude Code CLI's `/model` @@ -95,13 +93,7 @@ export const serveModels = async (c: Context): Promise => { ? toClaudeCodeCatalog(publicCatalog) : publicCatalog); } catch (e) { - // Upstream HTTP/parse failures squash to a generic message so we do not - // leak upstream identity. Other registry-thrown errors (e.g. the "no - // upstream configured" hint) carry actionable operator guidance and - // surface verbatim with the same 502. - const message = e instanceof ProviderModelsUnavailableError - ? MODEL_LISTING_FAILURE_MESSAGE - : (e instanceof Error ? e.message : String(e)); + const message = e instanceof Error ? e.message : String(e); return Response.json({ error: { message, type: 'api_error' } }, { status: 502 }); } }; diff --git a/packages/gateway/src/data-plane/models/shared.ts b/packages/gateway/src/data-plane/models/shared.ts index ff08ecf18..f415d9b50 100644 --- a/packages/gateway/src/data-plane/models/shared.ts +++ b/packages/gateway/src/data-plane/models/shared.ts @@ -1,12 +1,11 @@ -// Squash genuine upstream HTTP/parse failures (ProviderModelsUnavailableError) -// to a generic 502 so we do not leak upstream identity. Other errors (e.g. -// the registry's "no upstream configured" hint) carry actionable operator -// guidance and surface verbatim. +// The synchronous control-plane Fetch Models action squashes upstream +// HTTP/parse failures to this generic message so provider identity stays +// private. Ordinary listing routes read persisted snapshots and never observe +// the triggered upstream failure in their request lifecycle. export const MODEL_LISTING_FAILURE_MESSAGE = 'Upstream model listing failed'; // The message says nothing about the upstream and is prose, so the upstream // list-models route pairs it with this code and the dashboard tells that // failure apart from an arbitrary one without matching English. The model-list -// endpoints stay message-only: /v1/models, /models and /api/models answer a -// listing failure identically. +// snapshot listing routes do not use this action-specific discriminator. export const MODEL_LISTING_FAILURE_CODE = 'upstream_model_listing_failed'; diff --git a/packages/gateway/src/data-plane/providers/catalog.ts b/packages/gateway/src/data-plane/providers/catalog.ts index 182401409..83578f1ee 100644 --- a/packages/gateway/src/data-plane/providers/catalog.ts +++ b/packages/gateway/src/data-plane/providers/catalog.ts @@ -14,9 +14,8 @@ interface ProviderModelsResult { upstreamsByPublicId: Map; sawSuccess: boolean; lastError: unknown; - // Upstream names whose catalog fetch rejected this round, in the same - // order as the input `providers` list so the model-missing renderer can - // surface a stable, dashboard-aligned list. + // Upstreams carrying a persisted catalog-refresh error, plus any provider + // whose snapshot access failed synchronously, in provider order. failedUpstreams: string[]; } @@ -106,12 +105,8 @@ const collectProviderModels = async ( for (const [index, result] of settled.entries()) { if (result.status === 'rejected') { - // Caller-driven cancellation must propagate. Burying it in lastError - // and letting an earlier sawSuccess return a partially-populated - // model list would mask the abort and let the rest of the data-plane - // request build a Response against a stale catalog. `isAbortError` - // walks the cause chain so an AbortError wrapped inside - // ProviderModelsUnavailableError still surfaces here. + // Snapshot setup failures stay isolated per provider. Cancellation is + // the exception because the caller has withdrawn the whole operation. const error = result.reason; if (isAbortError(error)) throw error; lastError = error; diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index d0142c3e8..1db8f825f 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -104,8 +104,8 @@ export const enumerateRealModelCandidates = async ( // (`claude-sonnet-4-5-20250929`) even though the gateway's merged catalog // only carries the undated alias. When the inbound id matches no catalog // entry, strip an 8-digit `-YYYYMMDD` suffix and try once more — failed -// catalog fetches across the two attempts dedupe into a single -// `failedUpstreams` list for the caller's renderer. +// providers carrying a recorded catalog-refresh error across the two snapshot +// lookups dedupe into one `failedUpstreams` list. const DATED_SUFFIX = /-\d{8}$/; // Real-catalog resolution with the dated-suffix retry baked in. Used both diff --git a/packages/gateway/src/data-plane/shared/failed-upstreams.ts b/packages/gateway/src/data-plane/shared/failed-upstreams.ts index 9d78377d4..b9a66b957 100644 --- a/packages/gateway/src/data-plane/shared/failed-upstreams.ts +++ b/packages/gateway/src/data-plane/shared/failed-upstreams.ts @@ -1,10 +1,8 @@ // Append a parenthetical clause to a "model not found / unsupported" -// error body when one or more upstreams' catalog fetches rejected -// during the request. Surfaced inline alongside the per-request 4xx -// so a client can tell a genuine miss from a transient outage where -// the upstream that owns the model is currently unreachable. The same -// data is independently visible to operators on the dashboard via -// `modelsCache.lastError`. +// error body when one or more consulted upstreams record a catalog-refresh +// failure. The same durable state is visible to operators through +// `modelsCache.lastError`; it does not imply that this request performed a +// refresh or that the upstream remains unreachable. // // The suffix is inserted *before* a trailing `.` so the final message // reads "Model X is not available on any configured upstream (models diff --git a/packages/gateway/src/data-plane/shared/listing/addressable.ts b/packages/gateway/src/data-plane/shared/listing/addressable.ts index 6c9e62216..aa1ca2895 100644 --- a/packages/gateway/src/data-plane/shared/listing/addressable.ts +++ b/packages/gateway/src/data-plane/shared/listing/addressable.ts @@ -79,16 +79,9 @@ export const enumerateAddressableModelIds = async ( push({ id: model.id, unlisted: undefined, model, upstreams: upstreamsByPublicId.get(model.id) ?? [] }); } - // Per-upstream walk for the prefix-addressable alternates the listed - // surface chose not to publish. The catalog round-trip is the same SWR - // cache the listed surface just consumed, so this loop never pays a - // second upstream hit. - // - // A rejected per-upstream catalog refresh collapses to no addressable- - // only contribution from THAT upstream — its listed rows already came - // (or were dropped) through `getModelsFromProviders`. Mirrors the `Promise.allSettled` - // tolerance there so a transiently-down upstream cannot tank /v1/models - // on a cold-start gateway. + // Prefix alternates reuse the same persisted provider snapshots as the + // listed surface. Repeated access may join the same L1 refresh trigger, but + // never performs upstream model-list I/O in this request. const perUpstream = await Promise.allSettled(providers.map(async provider => { const cfg = provider.modelPrefix; const addressableOnly = cfg !== null ? cfg.addressable.filter(form => !cfg.listed.includes(form)) : []; @@ -122,11 +115,8 @@ export const enumerateAddressableModelIds = async ( for (const result of perUpstream) { if (result.status === 'rejected') { - // Cancellation must propagate even from this tolerant fanout — the - // per-request abort signal cannot be masked by an upstream's slow - // rejection. Other failures (catalog 5xx, parse, transport) collapse - // to no addressable-only contribution from that upstream per the - // contract above. + // Snapshot setup failures omit only that provider; cancellation still + // withdraws the whole caller operation. if (isAbortError(result.reason)) throw result.reason; continue; } From 93390d043f8db0ae7013d4e278525f1d4f8ffc09 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:14:46 +0800 Subject: [PATCH 22/46] fix(gateway): reset refresh state across generations Clear claim and backoff state whenever an upstream row is saved into a new generation, while preserving its last-known-good catalog unless the caller explicitly clears it. --- packages/gateway/__tests__/repo/memory.ts | 1 + .../__tests__/repo/models-refresh_test.ts | 20 +++++++++++++++++++ packages/gateway/src/repo/sql.ts | 3 ++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index e14eba89f..b93d52715 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -578,6 +578,7 @@ class MemoryUpstreamRepo implements UpstreamRepo { ? { ...upstream, createdAt: existing.createdAt, modelsCache: existing.modelsCache } : { ...upstream, modelsCache: null }; this.store.set(preserved.id, cloneUpstreamRecord(preserved)); + this.modelsRefreshes.delete(preserved.id); return Promise.resolve(); } diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index 8f54338fa..266e8fb74 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -80,4 +80,24 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.saveClearingModelsCache(newer); await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'old-time', now + 900_004, now + 4, false)).resolves.toBeNull(); }); + + test('saving a new upstream generation clears its predecessor refresh state', async () => { + const repo = await createRepo(); + await repo.upstreams.save(record); + const now = 1_800_000_000_000; + const claim = await repo.upstreams.claimModelsRefresh(record.id, generation, 'failed', now, now - 900_000, false); + if (!claim) throw new Error('expected refresh claim'); + await repo.upstreams.completeModelsRefreshFailure(record.id, 'failed', 1, modelsRefreshRetryAt(now, 0)); + + const next = { ...record, name: 'Renamed', updatedAt: '2026-08-01T00:01:00.000Z' }; + await repo.upstreams.save(next); + await expect(repo.upstreams.claimModelsRefresh( + record.id, + { updatedAt: next.updatedAt, config: next.config }, + 'next-generation', + now + 1, + now - 899_999, + false, + )).resolves.toEqual({ failureCount: 0 }); + }); }); diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index a5ca0734c..9712fd671 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -916,7 +916,8 @@ class SqlUpstreamRepo implements UpstreamRepo { disabled_public_model_ids = excluded.disabled_public_model_ids, proxy_fallback_list_json = excluded.proxy_fallback_list_json, model_prefix_json = excluded.model_prefix_json, - hue = excluded.hue${clearModelsCache ? ', models_cache_json = NULL, models_refresh_json = NULL' : ''}`, + hue = excluded.hue, + models_refresh_json = NULL${clearModelsCache ? ', models_cache_json = NULL' : ''}`, ) .bind( upstream.id, From 72bd4138948d0edccc1bc686762eb7d2e2c73295 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:15:54 +0800 Subject: [PATCH 23/46] test(gateway): observe refresh fan-out directly Gate every mocked provider response until all sibling refreshes have started, replacing a load-sensitive elapsed-time assertion with an ordering proof. --- .../data-plane/providers/catalog_test.ts | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts index f08baf87c..dbf00111e 100644 --- a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts @@ -1,4 +1,4 @@ -import { describe, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; import { compareModelIds, getModelsFromProviders } from '../../../src/data-plane/providers/catalog.ts'; import { clearInFlightForTesting } from '../../../src/data-plane/providers/models-cache.ts'; @@ -250,16 +250,14 @@ test('disabledPublicModelIds hides models from the catalog and routing, per upst assertEquals(keep.candidates.map(m => m.provider.upstreamId), ['up_a']); }); -// Per-upstream catalog refresh triggers fan out in parallel: total wall-clock time -// tracks the slowest upstream, not the sum. The bound is loose because CI -// timer noise eats into a tight `< sum` comparison; what matters is the -// ratio. +// Every upstream request must start before any sibling is released. This +// directly observes concurrency without a wall-clock threshold that load can +// satisfy or violate independently of execution order. test('catalog refresh triggers fan out per upstream in parallel', async () => { clearInFlightForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); - const FETCH_DELAY_MS = 60; const upstreams = [ { id: 'up_p1', host: 'p1.example.com', model: 'p1-model' }, { id: 'up_p2', host: 'p2.example.com', model: 'p2-model' }, @@ -274,30 +272,28 @@ test('catalog refresh triggers fan out per upstream in parallel', async () => { })); } + const started: string[] = []; + const releases = new Map void>(); await withMockedFetchRaw( - async request => { + request => { const url = new URL(request.url); const match = upstreams.find(u => url.hostname === u.host); if (match && url.pathname === '/v1/models') { - await new Promise(resolve => setTimeout(resolve, FETCH_DELAY_MS)); - return jsonResponse({ object: 'list', data: [{ id: match.model, supported_endpoints: ['/chat/completions'] }] }); + started.push(match.host); + return new Promise(resolve => { + releases.set(match.host, () => resolve(jsonResponse({ object: 'list', data: [{ id: match.model, supported_endpoints: ['/chat/completions'] }] }))); + }); } throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const start = Date.now(); - await warmModelsForTest(); + const warming = warmModelsForTest(); + await vi.waitFor(() => expect(started.toSorted()).toEqual(upstreams.map(upstream => upstream.host).toSorted())); + for (const release of releases.values()) release(); + await warming; const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; - const elapsed = Date.now() - start; assertEquals([...catalog.map(m => m.id)].sort(), ['p1-model', 'p2-model', 'p3-model']); - // A serial walk would take >= 3 * FETCH_DELAY_MS; parallel is bounded by - // ~FETCH_DELAY_MS plus per-test overhead. Half the serial budget is the - // loosest threshold that still excludes any serial regression. - const serialBudget = upstreams.length * FETCH_DELAY_MS; - if (elapsed >= serialBudget / 2) { - throw new Error(`expected parallel walk (~${FETCH_DELAY_MS}ms) but took ${elapsed}ms (serial would be ${serialBudget}ms)`); - } }, ); }); From d9eecbbab7a54a946f045755972945dda2745552 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:19:26 +0800 Subject: [PATCH 24/46] docs: name background catalog refresh precisely --- docs/RESOLUTION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/RESOLUTION.md b/docs/RESOLUTION.md index 5e94755a9..946b10015 100644 --- a/docs/RESOLUTION.md +++ b/docs/RESOLUTION.md @@ -131,7 +131,7 @@ pointing them back to their canonical listed row. unrestricted, an empty list means no provider is visible); - `kind`, derived from the source route: `chat`, `embedding`, `image`, `rerank`, or `transcription`; -- the background scheduler and runtime-location tag needed by catalog fetch and +- the background scheduler and runtime-location tag needed by catalog refresh and proxy selection. `/v1/completions` and `/completions` deliberately use `kind: 'chat'`, then From 788d22ec7a0a7272bdfffdaa0b3fedbc2622019b Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:24:36 +0800 Subject: [PATCH 25/46] fix(gateway): preserve scheduled storage ordering Run model and image maintenance alongside the existing storage pipeline while keeping expiration processing ahead of spilled-file collection. --- packages/gateway/__tests__/scheduled_test.ts | 20 ++++++++++++++++++++ packages/gateway/src/scheduled.ts | 7 +++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/gateway/__tests__/scheduled_test.ts b/packages/gateway/__tests__/scheduled_test.ts index 03fcd0735..0e56554f4 100644 --- a/packages/gateway/__tests__/scheduled_test.ts +++ b/packages/gateway/__tests__/scheduled_test.ts @@ -42,3 +42,23 @@ test('scheduled maintenance collects exact spilled files after expiration work', expect(await files.get(key)).toBeNull(); }); + +test('scheduled maintenance does not collect spilled files before expiration work finishes', async () => { + const { repo } = await setupAppTest(); + await repo.upstreams.deleteAll(); + initFileStore(new MemoryFileStore()); + initImageCacheStore({ async get() { return null; }, async put() {}, async sweepExpired() {} }); + let releaseExpiration: (() => void) | null = null; + vi.spyOn(repo.expirationSweeps, 'claim').mockImplementation(async () => { + await new Promise(resolve => { releaseExpiration = resolve; }); + return null; + }); + const collect = vi.spyOn(repo.spilledFiles, 'claimCollectible').mockResolvedValue([]); + + const maintenance = runScheduledMaintenance(); + await vi.waitFor(() => expect(releaseExpiration).not.toBeNull()); + expect(collect).not.toHaveBeenCalled(); + releaseExpiration!(); + await maintenance; + expect(collect).toHaveBeenCalledOnce(); +}); diff --git a/packages/gateway/src/scheduled.ts b/packages/gateway/src/scheduled.ts index 0e9e32aa2..f366bb6cd 100644 --- a/packages/gateway/src/scheduled.ts +++ b/packages/gateway/src/scheduled.ts @@ -22,10 +22,13 @@ export const runScheduledMaintenance = async ( backgroundScheduler: BackgroundScheduler = defaultBackgroundScheduler, ): Promise => { const nowMs = Date.now(); + const storageMaintenance = async (): Promise => { + await runSweep('expirations.sweep', () => sweepExpirations(nowMs)); + await runSweep('spilledFiles.collect', () => collectSpilledFiles(nowMs)); + }; await Promise.all([ runSweep('models.refresh', () => refreshModelsCaches(runtimeLocation, backgroundScheduler)), - runSweep('expirations.sweep', () => sweepExpirations(nowMs)), - runSweep('spilledFiles.collect', () => collectSpilledFiles(nowMs)), + storageMaintenance(), runSweep('imageCacheStore.sweepExpired', () => getImageCacheStore().sweepExpired(nowMs)), ]); }; From 1f6ecc29b08610102584ec37027485663bf8b0ca Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:28:36 +0800 Subject: [PATCH 26/46] fix(gateway): wait for cross-runtime refresh owners Classify persisted claim denials as active, backoff, or generation mismatch. Blocking warm operations now poll an active cross-runtime owner until it publishes or yields, while automatic triggers still return immediately. Cover durable warm joining and prove stale-token failure completion cannot reinstall cooldown. --- .../data-plane/providers/models-cache_test.ts | 29 ++++++++ packages/gateway/__tests__/repo/memory.ts | 15 ++-- .../__tests__/repo/models-refresh_test.ts | 33 +++++---- .../src/data-plane/providers/models-cache.ts | 48 +++++++++---- packages/gateway/src/repo/sql.ts | 69 ++++++++++++------- packages/gateway/src/repo/types.ts | 8 ++- 6 files changed, 140 insertions(+), 62 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 3098d50f0..a5400c3f7 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -244,6 +244,30 @@ describe('fetchUpstreamModelsCached', () => { expect(fetchFn).toHaveBeenCalledTimes(1); }); + test('synchronous warm waits for a refresh owned by another runtime', async () => { + const repo = await setupRepo(); + const now = Date.now(); + await expect(repo.upstreams.claimModelsRefresh(UPSTREAM_ID, CACHE_GENERATION, 'remote-owner', now, now - 900_000, false)) + .resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + const localFetch = vi.fn(async () => [aModel('duplicate-local-model')]); + const warming = warmUpstreamModels(stubInstance(localFetch), directFetcher); + + let settled = false; + void warming.finally(() => { settled = true; }); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(settled).toBe(false); + + await repo.upstreams.saveClaimedModelsCache(UPSTREAM_ID, CACHE_GENERATION, 'remote-owner', { + revision: MODEL_CATALOG_REVISION, + fetchedAt: now + 1, + models: [aModel('remote-model')], + }); + await repo.upstreams.completeModelsRefreshSuccess(UPSTREAM_ID, 'remote-owner'); + + expect((await warming).map(model => model.id)).toEqual(['remote-model']); + expect(localFetch).not.toHaveBeenCalled(); + }); + test('a superseded generation neither joins nor overwrites the current catalog', async () => { const repo = await setupRepo(); let resolveOld: ((models: ProviderModel[]) => void) | null = null; @@ -317,6 +341,11 @@ describe('fetchUpstreamModelsCached', () => { models: [{ id: 'forced-model' }], lastError: null, }); + + clearInFlightForTesting(); + const recovery = vi.fn(async () => [aModel('post-race-model')]); + await warmUpstreamModels(stubInstance(recovery, await storedCache(repo), CACHE_GENERATION, 'post-race'), directFetcher); + expect(recovery).toHaveBeenCalledTimes(1); }); test('catalog revision mismatch is cold and refreshes without blocking', async () => { diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index b93d52715..8a465fbe1 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -26,6 +26,7 @@ import type { AgentSetupRepository, BackoffRow, ModelsCacheGeneration, + ModelsRefreshClaimResult, ModelAliasesRepo, ModelAliasRecord, PerformanceDimensions, @@ -641,21 +642,21 @@ class MemoryUpstreamRepo implements UpstreamRepo { return this.saveModelsCacheError(id, generation, error); } - claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise<{ failureCount: number } | null> { + claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise { const stored = this.store.get(id); - if (!stored || stored.updatedAt !== generation.updatedAt || serializeStoredConfig(stored.config) !== serializeStoredConfig(generation.config)) return Promise.resolve(null); + if (!stored || stored.updatedAt !== generation.updatedAt || serializeStoredConfig(stored.config) !== serializeStoredConfig(generation.config)) return Promise.resolve({ kind: 'generation-mismatch' }); const existing = this.modelsRefreshes.get(id); - const eligible = force - || existing === undefined - || (existing.retryAt <= now && (existing.claimToken === null || existing.claimedAt! <= staleClaimedBefore)); - if (!eligible) return Promise.resolve(null); + if (!force && existing !== undefined) { + if (existing.claimToken !== null && existing.claimedAt! > staleClaimedBefore) return Promise.resolve({ kind: 'active' }); + if (existing.retryAt > now) return Promise.resolve({ kind: 'backoff' }); + } this.modelsRefreshes.set(id, { failCount: existing?.failCount ?? 0, retryAt: existing?.retryAt ?? 0, claimToken: token, claimedAt: now, }); - return Promise.resolve({ failureCount: existing?.failCount ?? 0 }); + return Promise.resolve({ kind: 'claimed', failureCount: existing?.failCount ?? 0 }); } completeModelsRefreshSuccess(id: string, token: string): Promise { diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index 266e8fb74..92f51055d 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -39,26 +39,29 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = let now = 1_800_000_000_000; const first = await repo.upstreams.claimModelsRefresh(record.id, generation, 'claim-0', now, now - 900_000, false); - expect(first).toEqual({ failureCount: 0 }); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now, now - 900_000, false)).resolves.toBeNull(); + expect(first).toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now, now - 900_000, false)).resolves.toEqual({ kind: 'active' }); const delays = [1, 2, 4, 8, 16, 32, 60, 60].map(minutes => minutes * 60_000); - let claim = first!; + if (first.kind !== 'claimed') throw new Error('expected refresh claim'); + let claim = first; for (const [index, delay] of delays.entries()) { const retryAt = modelsRefreshRetryAt(now, claim.failureCount); expect(retryAt - now).toBe(delay); await repo.upstreams.completeModelsRefreshFailure(record.id, `claim-${index}`, claim.failureCount + 1, retryAt); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, `early-${index}`, retryAt - 1, retryAt - 900_001, false)).resolves.toBeNull(); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, `early-${index}`, retryAt - 1, retryAt - 900_001, false)).resolves.toEqual({ kind: 'backoff' }); now = retryAt; - claim = (await repo.upstreams.claimModelsRefresh(record.id, generation, `claim-${index + 1}`, now, now - 900_000, false))!; + const nextClaim = await repo.upstreams.claimModelsRefresh(record.id, generation, `claim-${index + 1}`, now, now - 900_000, false); + if (nextClaim.kind !== 'claimed') throw new Error('expected refresh claim'); + claim = nextClaim; expect(claim.failureCount).toBe(index + 1); } const blockedUntil = modelsRefreshRetryAt(now, claim.failureCount); await repo.upstreams.completeModelsRefreshFailure(record.id, `claim-${delays.length}`, claim.failureCount + 1, blockedUntil); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'forced', now + 1, now - 899_999, true)).resolves.toEqual({ failureCount: delays.length + 1 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'forced', now + 1, now - 899_999, true)).resolves.toEqual({ kind: 'claimed', failureCount: delays.length + 1 }); await repo.upstreams.completeModelsRefreshSuccess(record.id, 'forced'); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'after-success', now + 2, now - 899_998, false)).resolves.toEqual({ failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'after-success', now + 2, now - 899_998, false)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); }); test('recovers abandoned claims and fences tokens, timestamps, and config', async () => { @@ -66,19 +69,19 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.save(record); const now = 1_800_000_000_000; - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'abandoned', now, now - 900_000, false)).resolves.toEqual({ failureCount: 0 }); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'replacement', now + 900_001, now + 1, false)).resolves.toEqual({ failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'abandoned', now, now - 900_000, false)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'replacement', now + 900_001, now + 1, false)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); await repo.upstreams.completeModelsRefreshSuccess(record.id, 'abandoned'); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now + 900_002, now + 2, false)).resolves.toBeNull(); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now + 900_002, now + 2, false)).resolves.toEqual({ kind: 'active' }); const next = { ...record, config: { tenant: 'next' } }; await repo.upstreams.saveClearingModelsCache(next); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'old-config', now + 900_003, now + 3, false)).resolves.toBeNull(); - await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'current', now + 900_003, now + 3, false)).resolves.toEqual({ failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'old-config', now + 900_003, now + 3, false)).resolves.toEqual({ kind: 'generation-mismatch' }); + await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'current', now + 900_003, now + 3, false)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); const newer = { ...next, updatedAt: '2026-08-01T00:01:00.000Z' }; await repo.upstreams.saveClearingModelsCache(newer); - await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'old-time', now + 900_004, now + 4, false)).resolves.toBeNull(); + await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'old-time', now + 900_004, now + 4, false)).resolves.toEqual({ kind: 'generation-mismatch' }); }); test('saving a new upstream generation clears its predecessor refresh state', async () => { @@ -86,7 +89,7 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.save(record); const now = 1_800_000_000_000; const claim = await repo.upstreams.claimModelsRefresh(record.id, generation, 'failed', now, now - 900_000, false); - if (!claim) throw new Error('expected refresh claim'); + if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); await repo.upstreams.completeModelsRefreshFailure(record.id, 'failed', 1, modelsRefreshRetryAt(now, 0)); const next = { ...record, name: 'Renamed', updatedAt: '2026-08-01T00:01:00.000Z' }; @@ -98,6 +101,6 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = now + 1, now - 899_999, false, - )).resolves.toEqual({ failureCount: 0 }); + )).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); }); }); diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index 71f7a00a4..5f1440577 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -10,6 +10,7 @@ import type { Fetcher, ProviderModel } from '@floway-dev/provider'; // access only triggers a background attempt guarded by the persisted refresh // claim/backoff state. const SOFT_MS = 10 * 60 * 1000; +const ACTIVE_REFRESH_POLL_MS = 100; export { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; @@ -83,20 +84,38 @@ const runClaimedFetch = async ( instance: GatewayProvider, fetcher: Fetcher, force: boolean, + waitForActive: boolean, loadProvidedModels?: () => Promise, ): Promise => { const repo = getRepo(); - const now = Date.now(); const token = crypto.randomUUID(); - const claimed = await repo.upstreams.claimModelsRefresh( - instance.upstreamId, - instance.modelsCacheGeneration, - token, - now, - now - MODELS_REFRESH_CLAIM_LEASE_MS, - force, - ); - if (claimed === null) return null; + const initialFetchedAt = instance.modelsCache?.fetchedAt ?? null; + const initialErrorAt = instance.modelsCache?.lastError?.at ?? null; + let claimed: Extract>, { kind: 'claimed' }>; + while (true) { + const now = Date.now(); + const outcome = await repo.upstreams.claimModelsRefresh( + instance.upstreamId, + instance.modelsCacheGeneration, + token, + now, + now - MODELS_REFRESH_CLAIM_LEASE_MS, + force, + ); + if (outcome.kind === 'claimed') { + claimed = outcome; + break; + } + if (outcome.kind !== 'active' || !waitForActive) return null; + await new Promise(resolve => setTimeout(resolve, ACTIVE_REFRESH_POLL_MS)); + const current = await repo.upstreams.getById(instance.upstreamId); + if (current === null + || current.updatedAt !== instance.modelsCacheGeneration.updatedAt + || serializeStoredConfig(current.config) !== serializeStoredConfig(instance.modelsCacheGeneration.config)) return null; + instance.modelsCache = current.modelsCache; + if ((current.modelsCache?.fetchedAt ?? null) !== initialFetchedAt + || (current.modelsCache?.lastError?.at ?? null) !== initialErrorAt) return null; + } try { const models = await runFetch(instance, fetcher, instance.upstreamId, token, loadProvidedModels); @@ -132,7 +151,7 @@ export const fetchUpstreamModels = async ( if (inFlight.get(key) === existing) inFlight.delete(key); } - const models = await memoInFlight(key, () => runClaimedFetch(instance, fetcher, true, loadProvidedModels)); + const models = await memoInFlight(key, () => runClaimedFetch(instance, fetcher, true, false, loadProvidedModels)); if (models === null) throw new Error(`Failed to force-claim models refresh for ${instance.upstreamId}`); return models; }; @@ -146,10 +165,11 @@ export const warmUpstreamModels = async ( const existing = inFlight.get(key); if (existing) { const joined = await existing; - return joined ?? instance.modelsCache?.models ?? []; + if (joined !== null) return joined; + if (inFlight.get(key) === existing) inFlight.delete(key); } - const models = await memoInFlight(key, () => runClaimedFetch(instance, fetcher, false, loadProvidedModels)); + const models = await memoInFlight(key, () => runClaimedFetch(instance, fetcher, false, true, loadProvidedModels)); return models ?? instance.modelsCache?.models ?? []; }; @@ -160,7 +180,7 @@ export const triggerUpstreamModelsFetch = ( loadProvidedModels?: () => Promise, ): void => { const key = inFlightKey(instance); - scheduler(memoInFlight(key, () => runClaimedFetch(instance, fetcher, false, loadProvidedModels)).then(() => {})); + scheduler(memoInFlight(key, () => runClaimedFetch(instance, fetcher, false, false, loadProvidedModels)).then(() => {})); }; export const fetchUpstreamModelsCached = async ( diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index 9712fd671..a21816b43 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -19,6 +19,7 @@ import type { AgentSetupRepository, BackoffRow, ModelsCacheGeneration, + ModelsRefreshClaimResult, ModelAliasesRepo, ModelAliasRecord, PerformanceBucketRow, @@ -1000,34 +1001,52 @@ class SqlUpstreamRepo implements UpstreamRepo { return (result.meta.changes ?? 0) > 0; } - async claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise<{ failureCount: number } | null> { + async claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise { const rawConfig = await this.modelsCacheWriteConfig(id, generation); - if (rawConfig === null) return null; - const row = await this.db - .prepare( - `UPDATE upstreams - SET models_refresh_json = json_object( - 'failCount', coalesce(json_extract(models_refresh_json, '$.failCount'), 0), - 'retryAt', coalesce(json_extract(models_refresh_json, '$.retryAt'), 0), - 'claimToken', ?, - 'claimedAt', ? - ) - WHERE id = ? AND updated_at = ? AND config_json = ? AND ( - ? = 1 - OR models_refresh_json IS NULL - OR ( - coalesce(json_extract(models_refresh_json, '$.retryAt'), 0) <= ? - AND ( - json_extract(models_refresh_json, '$.claimToken') IS NULL - OR json_extract(models_refresh_json, '$.claimedAt') <= ? + if (rawConfig === null) return { kind: 'generation-mismatch' }; + for (let attempt = 0; attempt < 3; attempt += 1) { + const row = await this.db + .prepare( + `UPDATE upstreams + SET models_refresh_json = json_object( + 'failCount', coalesce(json_extract(models_refresh_json, '$.failCount'), 0), + 'retryAt', coalesce(json_extract(models_refresh_json, '$.retryAt'), 0), + 'claimToken', ?, + 'claimedAt', ? + ) + WHERE id = ? AND updated_at = ? AND config_json = ? AND ( + ? = 1 + OR models_refresh_json IS NULL + OR ( + coalesce(json_extract(models_refresh_json, '$.retryAt'), 0) <= ? + AND ( + json_extract(models_refresh_json, '$.claimToken') IS NULL + OR json_extract(models_refresh_json, '$.claimedAt') <= ? + ) ) ) - ) - RETURNING json_extract(models_refresh_json, '$.failCount') AS fail_count`, - ) - .bind(token, now, id, generation.updatedAt, rawConfig, sqliteBoolean(force), now, staleClaimedBefore) - .first<{ fail_count: number }>(); - return row === null ? null : { failureCount: row.fail_count }; + RETURNING json_extract(models_refresh_json, '$.failCount') AS fail_count`, + ) + .bind(token, now, id, generation.updatedAt, rawConfig, sqliteBoolean(force), now, staleClaimedBefore) + .first<{ fail_count: number }>(); + if (row !== null) return { kind: 'claimed', failureCount: row.fail_count }; + + const state = await this.db + .prepare( + `SELECT models_refresh_json, + json_extract(models_refresh_json, '$.retryAt') AS retry_at, + json_extract(models_refresh_json, '$.claimToken') AS claim_token, + json_extract(models_refresh_json, '$.claimedAt') AS claimed_at + FROM upstreams WHERE id = ? AND updated_at = ? AND config_json = ?`, + ) + .bind(id, generation.updatedAt, rawConfig) + .first<{ models_refresh_json: string | null; retry_at: number | null; claim_token: string | null; claimed_at: number | null }>(); + if (state === null) return { kind: 'generation-mismatch' }; + if (state.models_refresh_json === null) continue; + if (state.claim_token !== null && state.claimed_at !== null && state.claimed_at > staleClaimedBefore) return { kind: 'active' }; + if (state.retry_at !== null && state.retry_at > now) return { kind: 'backoff' }; + } + throw new Error(`Failed to classify models refresh claim contention for ${id}`); } async completeModelsRefreshSuccess(id: string, token: string): Promise { diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index d88a5123d..f04663a15 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -269,15 +269,21 @@ export interface UpstreamRepo { saveModelsCacheError(id: string, generation: ModelsCacheGeneration, error: NonNullable): Promise; saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise; saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise; - claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise; + claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise; completeModelsRefreshSuccess(id: string, token: string): Promise; completeModelsRefreshFailure(id: string, token: string, failureCount: number, retryAt: number): Promise; } export interface ModelsRefreshClaim { + kind: 'claimed'; failureCount: number; } +export type ModelsRefreshClaimResult = ModelsRefreshClaim + | { kind: 'active' } + | { kind: 'backoff' } + | { kind: 'generation-mismatch' }; + export interface ModelsCacheGeneration { updatedAt: string; config: unknown; From 4522b503fd1dd0a9017d6a0ea8805f48d29335d0 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:36:13 +0800 Subject: [PATCH 27/46] refactor(gateway): require claims for every catalog write Remove unowned model-cache repository writes so production and fixtures cannot bypass the persisted refresh protocol. Test seeding now acquires an explicit forced claim and publishes through the same owner-fenced methods. --- .../__tests__/node-sqlite-repo_test.ts | 9 +++-- .../upstreams/copilot-device-login_test.ts | 3 +- .../control-plane/upstreams/routes_test.ts | 17 +++++----- .../data-plane/providers/models-cache_test.ts | 3 +- .../data-plane/providers/registry_test.ts | 3 +- packages/gateway/__tests__/repo/memory.ts | 16 +++------ .../__tests__/repo/models-cache-fixture.ts | 30 ++++++++++++++++ packages/gateway/__tests__/repo/sql_test.ts | 27 ++++++++------- packages/gateway/src/repo/sql.ts | 34 ++----------------- packages/gateway/src/repo/types.ts | 2 -- 10 files changed, 72 insertions(+), 72 deletions(-) create mode 100644 packages/gateway/__tests__/repo/models-cache-fixture.ts diff --git a/apps/platform-node/__tests__/node-sqlite-repo_test.ts b/apps/platform-node/__tests__/node-sqlite-repo_test.ts index 011fe18cd..248e3ff95 100644 --- a/apps/platform-node/__tests__/node-sqlite-repo_test.ts +++ b/apps/platform-node/__tests__/node-sqlite-repo_test.ts @@ -94,14 +94,19 @@ test('repository JSON codecs round-trip upstream, alias, and Responses state thr modelPrefix: null, hue: 210, }); - await repo.upstreams.saveModelsCache('up_node', { + const cacheGeneration = { updatedAt: '2026-08-05T00:00:00.000Z', config: { opaque: { value: true } }, - }, { + }; + const cacheToken = 'node-cache-fixture'; + const cacheClaim = await repo.upstreams.claimModelsRefresh('up_node', cacheGeneration, cacheToken, Date.now(), Number.MIN_SAFE_INTEGER, true); + if (cacheClaim.kind !== 'claimed') throw new Error('expected model-cache fixture claim'); + await repo.upstreams.saveClaimedModelsCache('up_node', cacheGeneration, cacheToken, { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_786_000_000_000, models: [stubProviderModel({ id: 'node-model', enabledFlags: new Set(['vendor-kimi'] as const) })], }); + await repo.upstreams.completeModelsRefreshSuccess('up_node', cacheToken); await repo.modelAliases.insert({ id: 'alias_node', name: 'node-alias', diff --git a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts index 1a78b5e2b..89bcd563d 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts @@ -16,6 +16,7 @@ vi.mock('../../../src/data-plane/providers/models-cache.ts', () => ({ })); import { buildCopilotUpstreamRecord, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; import { assertEquals, assertStringIncludes, jsonResponse, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; const githubUser = { @@ -369,7 +370,7 @@ test('/api/upstreams/copilot/oauth/device-login/poll clears the previous identit const existing = buildCopilotUpstreamRecord(githubAccount, { id: 'up_switch_identity' }); await repo.upstreams.deleteAll(); await repo.upstreams.save(existing); - await repo.upstreams.saveModelsCache(existing.id, { updatedAt: existing.updatedAt, config: existing.config }, { + await seedModelsCache(repo.upstreams, existing.id, { updatedAt: existing.updatedAt, config: existing.config }, { revision: 1, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'old-tenant-model' })], diff --git a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts index 99e2ff95f..a932b0fb9 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts @@ -3,6 +3,7 @@ import { test } from 'vitest'; import { blueprintUpstreamRecord, upstreamRecordToFullJson } from '../../../src/control-plane/upstreams/serialize.ts'; import { MODEL_LISTING_FAILURE_CODE } from '../../../src/data-plane/models/shared.ts'; import { MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; +import { seedModelsCache, seedModelsCacheError } from '../../repo/models-cache-fixture.ts'; import { MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import type { UpstreamProviderKind, UpstreamRecord } from '@floway-dev/provider'; import { assertEquals, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -282,7 +283,7 @@ test('PATCH /api/upstreams preserves omitted secrets and re-warms the models cac // Plant a stale row so the post-PATCH read can verify the warm overwrote // it with the new upstream-supplied catalog rather than leaving the old // models in place. - await repo.upstreams.saveModelsCache(created.id, await getCacheGeneration(repo, created.id), { + await seedModelsCache(repo.upstreams, created.id, await getCacheGeneration(repo, created.id), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1, models: [{ id: 'stale-model', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], @@ -420,7 +421,7 @@ test('GET /api/upstreams attaches models-cache freshness to every row', async () await repo.upstreams.deleteAll(); // Three upstreams cover the three cache states: no row, warm row, warm row - // with a follow-up failure annotated via saveModelsCacheError. + // with a follow-up failure annotation. const baseRow = { kind: 'custom' as const, enabled: true, @@ -440,17 +441,17 @@ test('GET /api/upstreams attaches models-cache freshness to every row', async () await repo.upstreams.save({ ...baseRow, id: 'up_warm', name: 'Warm', sortOrder: 1 }); await repo.upstreams.save({ ...baseRow, id: 'up_failed', name: 'Failed', sortOrder: 2 }); - await repo.upstreams.saveModelsCache('up_warm', { updatedAt: baseRow.updatedAt, config: baseRow.config }, { + await seedModelsCache(repo.upstreams, 'up_warm', { updatedAt: baseRow.updatedAt, config: baseRow.config }, { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [{ id: 'm1', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], }); - await repo.upstreams.saveModelsCache('up_failed', { updatedAt: baseRow.updatedAt, config: baseRow.config }, { + await seedModelsCache(repo.upstreams, 'up_failed', { updatedAt: baseRow.updatedAt, config: baseRow.config }, { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [{ id: 'm1', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], }); - await repo.upstreams.saveModelsCacheError('up_failed', { updatedAt: baseRow.updatedAt, config: baseRow.config }, { message: 'boom', at: 1_700_000_500_000 }); + await seedModelsCacheError(repo.upstreams, 'up_failed', { updatedAt: baseRow.updatedAt, config: baseRow.config }, { message: 'boom', at: 1_700_000_500_000 }); const list = await requestApp('/api/upstreams', { headers: { 'x-floway-session': adminSession } }); assertEquals(list.status, 200); @@ -487,7 +488,7 @@ test('GET /api/upstream-options returns the minimal picker shape to admin and no }); // A disabled upstream is absent from the live catalog, so the picker's count // comes from the catalog it stored while it was on. - await repo.upstreams.saveModelsCache('up_disabled_custom', await getCacheGeneration(repo, 'up_disabled_custom'), { + await seedModelsCache(repo.upstreams, 'up_disabled_custom', await getCacheGeneration(repo, 'up_disabled_custom'), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [ @@ -744,14 +745,14 @@ test('PATCH /api/upstreams warms the models cache before responding', async () = // Overwrite whatever the create-time warm landed on the row with a marker // catalog, so the assertion below can only pass if the PATCH-time warm wrote // over it. - await repo.upstreams.saveModelsCache(created.id, await getCacheGeneration(repo, created.id), { + await seedModelsCache(repo.upstreams, created.id, await getCacheGeneration(repo, created.id), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1, models: [{ id: 'warmed-on-create', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], }); // …and annotate it with an error the successful PATCH-time warm must clear, // so the response body cannot pass by echoing the pre-warm row. - await repo.upstreams.saveModelsCacheError(created.id, await getCacheGeneration(repo, created.id), { message: 'stale failure', at: 1 }); + await seedModelsCacheError(repo.upstreams, created.id, await getCacheGeneration(repo, created.id), { message: 'stale failure', at: 1 }); const patched = await withMockedFetch( async request => { diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index a5400c3f7..44bb5b94a 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -7,6 +7,7 @@ import { SqlRepo } from '../../../src/repo/sql.ts'; import type { ModelsCacheGeneration } from '../../../src/repo/types.ts'; import { serializeStoredConfig } from '../../../src/repo/upstream-json.ts'; import { InMemoryRepo } from '../../repo/memory.ts'; +import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; import { createSqliteTestDb } from '../../repo/test-sqlite.ts'; import { directFetcher, type ProviderModel, type UpstreamModelsCache } from '@floway-dev/provider'; import { stubProvider, stubProviderModel } from '@floway-dev/test-utils'; @@ -64,7 +65,7 @@ const seedCache = async ( repo: InMemoryRepo, cache: { revision: number; fetchedAt: number; models: ProviderModel[] }, ): Promise => { - await repo.upstreams.saveModelsCache(UPSTREAM_ID, CACHE_GENERATION, cache); + await seedModelsCache(repo.upstreams, UPSTREAM_ID, CACHE_GENERATION, cache); const stored = (await repo.upstreams.getById(UPSTREAM_ID))?.modelsCache; if (!stored) throw new Error('the seeded catalog did not land on the upstream row'); return stored; diff --git a/packages/gateway/__tests__/data-plane/providers/registry_test.ts b/packages/gateway/__tests__/data-plane/providers/registry_test.ts index d7fcfab77..6dd1be7cf 100644 --- a/packages/gateway/__tests__/data-plane/providers/registry_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/registry_test.ts @@ -2,6 +2,7 @@ import { test } from 'vitest'; import { MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; +import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; import { buildCopilotUpstreamRecord, buildCustomUpstreamRecord, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; @@ -95,7 +96,7 @@ test('listModelProviders carries each row cached catalog onto its instance', asy const cachedRecord = buildCustomUpstreamRecord({ id: 'up_cached', name: 'Cached', sortOrder: 10 }); await repo.upstreams.save(cachedRecord); await repo.upstreams.save(buildCustomUpstreamRecord({ id: 'up_cold', name: 'Cold', sortOrder: 20 })); - await repo.upstreams.saveModelsCache('up_cached', { updatedAt: cachedRecord.updatedAt, config: cachedRecord.config }, { + await seedModelsCache(repo.upstreams, 'up_cached', { updatedAt: cachedRecord.updatedAt, config: cachedRecord.config }, { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index 8a465fbe1..a37f7603c 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -617,14 +617,16 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(); } - saveModelsCache(id: string, generation: ModelsCacheGeneration, cache: Omit): Promise { + saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { + if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); const existing = this.store.get(id); if (!existing || existing.updatedAt !== generation.updatedAt || serializeStoredConfig(existing.config) !== serializeStoredConfig(generation.config)) return Promise.resolve(false); existing.modelsCache = { revision: cache.revision, fetchedAt: cache.fetchedAt, models: [...cache.models], lastError: null }; return Promise.resolve(true); } - saveModelsCacheError(id: string, generation: ModelsCacheGeneration, error: NonNullable): Promise { + saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise { + if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); const existing = this.store.get(id); if (!existing || existing.updatedAt !== generation.updatedAt || serializeStoredConfig(existing.config) !== serializeStoredConfig(generation.config)) return Promise.resolve(false); if (existing.modelsCache) existing.modelsCache.lastError = error; @@ -632,16 +634,6 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(true); } - saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { - if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); - return this.saveModelsCache(id, generation, cache); - } - - saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise { - if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); - return this.saveModelsCacheError(id, generation, error); - } - claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise { const stored = this.store.get(id); if (!stored || stored.updatedAt !== generation.updatedAt || serializeStoredConfig(stored.config) !== serializeStoredConfig(generation.config)) return Promise.resolve({ kind: 'generation-mismatch' }); diff --git a/packages/gateway/__tests__/repo/models-cache-fixture.ts b/packages/gateway/__tests__/repo/models-cache-fixture.ts new file mode 100644 index 000000000..282d5e697 --- /dev/null +++ b/packages/gateway/__tests__/repo/models-cache-fixture.ts @@ -0,0 +1,30 @@ +import type { ModelsCacheGeneration, UpstreamRepo } from '../../src/repo/types.ts'; +import type { UpstreamModelsCache } from '@floway-dev/provider'; + +export const seedModelsCache = async ( + repo: UpstreamRepo, + id: string, + generation: ModelsCacheGeneration, + cache: Omit, +): Promise => { + const token = crypto.randomUUID(); + const claim = await repo.claimModelsRefresh(id, generation, token, Date.now(), Number.MIN_SAFE_INTEGER, true); + if (claim.kind !== 'claimed') return false; + const saved = await repo.saveClaimedModelsCache(id, generation, token, cache); + await repo.completeModelsRefreshSuccess(id, token); + return saved; +}; + +export const seedModelsCacheError = async ( + repo: UpstreamRepo, + id: string, + generation: ModelsCacheGeneration, + error: NonNullable, +): Promise => { + const token = crypto.randomUUID(); + const claim = await repo.claimModelsRefresh(id, generation, token, Date.now(), Number.MIN_SAFE_INTEGER, true); + if (claim.kind !== 'claimed') return false; + const saved = await repo.saveClaimedModelsCacheError(id, generation, token, error); + await repo.completeModelsRefreshSuccess(id, token); + return saved; +}; diff --git a/packages/gateway/__tests__/repo/sql_test.ts b/packages/gateway/__tests__/repo/sql_test.ts index d1f8f5475..796216a03 100644 --- a/packages/gateway/__tests__/repo/sql_test.ts +++ b/packages/gateway/__tests__/repo/sql_test.ts @@ -1,6 +1,7 @@ import { test } from 'vitest'; import { createSqliteTestDb } from './test-sqlite.ts'; +import { seedModelsCache, seedModelsCacheError } from './models-cache-fixture.ts'; import { MODEL_CATALOG_REVISION } from '../../src/data-plane/providers/models-cache.ts'; import { SqlRepo, UPSTREAM_STATE_WRITE_ATTEMPTS } from '../../src/repo/sql.ts'; import type { SqlDatabase, SqlPreparedStatement } from '@floway-dev/platform'; @@ -51,7 +52,7 @@ test('SQL upstream repo preserves nested own __proto__ fields in opaque config a test('SQL upstream repo round-trips the cached catalog and its revision', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await repo.saveModelsCache('up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], @@ -87,7 +88,7 @@ test('SQL upstream repo rejects shape-invalid JSON in a cached catalog with row test('SQL upstream repo preserves opaque provider data while restoring only the model enabledFlags Set', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await repo.saveModelsCache('up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ @@ -116,21 +117,21 @@ test('SQL upstream repo hydrates deeply nested opaque provider data without recu assertEquals(ownValue(model?.providerData, 'next') !== undefined, true); }); -test('SQL upstream repo saveModelsCacheError annotates a cached catalog and saveModelsCache clears it', async () => { +test('SQL upstream repo annotates a cached catalog and successful publication clears the error', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await repo.saveModelsCache('up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], }); - await repo.saveModelsCacheError('up_test', generationFor(baseRecord()), { message: 'boom', at: 1_700_000_500_000 }); + await seedModelsCacheError(repo, 'up_test', generationFor(baseRecord()), { message: 'boom', at: 1_700_000_500_000 }); const annotated = (await repo.getById('up_test'))?.modelsCache; assertEquals(annotated?.lastError, { message: 'boom', at: 1_700_000_500_000 }); assertEquals(annotated?.models.map(model => model.id), ['cached-model']); - await repo.saveModelsCache('up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_001_000_000, models: [stubProviderModel({ id: 'refreshed-model' })], @@ -138,11 +139,11 @@ test('SQL upstream repo saveModelsCacheError annotates a cached catalog and save assertEquals((await repo.getById('up_test'))?.modelsCache?.lastError, null); }); -test('SQL upstream repo saveModelsCacheError persists an immediately-stale empty catalog on first failure', async () => { +test('SQL upstream repo persists an immediately-stale empty catalog on first failure', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await repo.saveModelsCacheError('up_test', generationFor(baseRecord()), { message: 'boom', at: 1_700_000_500_000 }); + await seedModelsCacheError(repo, 'up_test', generationFor(baseRecord()), { message: 'boom', at: 1_700_000_500_000 }); assertEquals((await repo.getById('up_test'))?.modelsCache, { revision: MODEL_CATALOG_REVISION, @@ -155,7 +156,7 @@ test('SQL upstream repo saveModelsCacheError persists an immediately-stale empty test('SQL upstream repo saveClearingModelsCache updates the row and removes the cached catalog atomically', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await repo.saveModelsCache('up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], @@ -171,12 +172,12 @@ test('SQL upstream repo saveClearingModelsCache updates the row and removes the assertEquals(stored?.name, 'New identity'); assertEquals(stored?.modelsCache, null); - const staleCatalogSaved = await repo.saveModelsCache('up_test', generationFor(baseRecord()), { + const staleCatalogSaved = await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { revision: 7, fetchedAt: 1_700_001_000_000, models: [stubProviderModel({ id: 'stale-model' })], }); - const staleErrorSaved = await repo.saveModelsCacheError('up_test', generationFor(baseRecord()), { message: 'stale error', at: 1_700_001_000_000 }); + const staleErrorSaved = await seedModelsCacheError(repo, 'up_test', generationFor(baseRecord()), { message: 'stale error', at: 1_700_001_000_000 }); assertEquals(staleCatalogSaved, false); assertEquals(staleErrorSaved, false); assertEquals((await repo.getById('up_test'))?.modelsCache, null); @@ -192,7 +193,7 @@ test('SQL model-cache generation accepts semantically equal noncanonical config const parsed = await repo.getById(record.id); if (!parsed) throw new Error('upstream row missing'); - const saved = await repo.saveModelsCache(record.id, generationFor(parsed), { + const saved = await seedModelsCache(repo, record.id, generationFor(parsed), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_001_000_000, models: [stubProviderModel({ id: 'cached-model' })], @@ -205,7 +206,7 @@ test('SQL model-cache generation accepts semantically equal noncanonical config test('SQL upstream repo save leaves an existing cached catalog alone', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await repo.saveModelsCache('up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index a21816b43..b09c11bef 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -948,38 +948,6 @@ class SqlUpstreamRepo implements UpstreamRepo { await this.db.prepare('DELETE FROM upstreams').run(); } - // Written only here and never by save(): an operator edit carries whatever - // catalog the request happened to read, and folding that back in would let a - // rename race a refresh. - async saveModelsCache(id: string, generation: ModelsCacheGeneration, cache: Omit): Promise { - const rawConfig = await this.modelsCacheWriteConfig(id, generation); - if (rawConfig === null) return false; - const result = await this.db - .prepare('UPDATE upstreams SET models_cache_json = ? WHERE id = ? AND updated_at = ? AND config_json = ?') - .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, generation.updatedAt, rawConfig) - .run(); - return (result.meta.changes ?? 0) > 0; - } - - // A cold failure persists an empty, immediately-stale catalog so the error - // remains visible without making the failed attempt look soft-fresh. An - // existing last-known-good catalog keeps its models and fetch timestamp. - async saveModelsCacheError(id: string, generation: ModelsCacheGeneration, error: NonNullable): Promise { - const rawConfig = await this.modelsCacheWriteConfig(id, generation); - if (rawConfig === null) return false; - const coldFailure = encodeUpstreamModelsCache({ - revision: MODEL_CATALOG_REVISION, - fetchedAt: 0, - models: [], - lastError: error, - }); - const result = await this.db - .prepare("UPDATE upstreams SET models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END WHERE id = ? AND updated_at = ? AND config_json = ?") - .bind(coldFailure, JSON.stringify(error), id, generation.updatedAt, rawConfig) - .run(); - return (result.meta.changes ?? 0) > 0; - } - async saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { const rawConfig = await this.modelsCacheWriteConfig(id, generation); if (rawConfig === null) return false; @@ -993,6 +961,8 @@ class SqlUpstreamRepo implements UpstreamRepo { async saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise { const rawConfig = await this.modelsCacheWriteConfig(id, generation); if (rawConfig === null) return false; + // A cold failure remains immediately stale while preserving the error for + // the next request and dashboard read. const coldFailure = encodeUpstreamModelsCache({ revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }); const result = await this.db .prepare("UPDATE upstreams SET models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END WHERE id = ? AND updated_at = ? AND config_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index f04663a15..b958c534a 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -265,8 +265,6 @@ export interface UpstreamRepo { // Catalog-cache writes are conditional on the row generation that started // the fetch. A superseded provider can finish serving its own request, but // cannot publish models or errors under newer credentials/configuration. - saveModelsCache(id: string, generation: ModelsCacheGeneration, cache: Omit): Promise; - saveModelsCacheError(id: string, generation: ModelsCacheGeneration, error: NonNullable): Promise; saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise; saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise; claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise; From 37e750953d45c0b049e83611a57391273f7ddda8 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:40:35 +0800 Subject: [PATCH 28/46] docs(gateway): distinguish snapshots from refresh work Reserve fetch and round-trip terminology for background upstream I/O, describe blocking warm coordination accurately, and limit client-visible refresh-error reporting to resolution failures. --- docs/RESOLUTION.md | 5 +++-- .../src/control-plane/shared/warm-models-cache.ts | 8 ++++---- packages/gateway/src/data-plane/providers/resolution.ts | 9 ++++----- .../gateway/src/data-plane/shared/listing/addressable.ts | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/RESOLUTION.md b/docs/RESOLUTION.md index 946b10015..4a5804bda 100644 --- a/docs/RESOLUTION.md +++ b/docs/RESOLUTION.md @@ -101,8 +101,9 @@ catalog feeds: `toPublicModel` projects an `InternalModel` onto the public DTO. Gemini uses its own projection, Codex synthesizes its client-catalog shape, and the control plane adds dashboard-only fields. Listing and request resolution are separate -consumers of the same persisted snapshots and both report recorded refresh -failures from `modelsCache.lastError`. +consumers of the same persisted snapshots. Request-resolution failures report +recorded refresh errors from `modelsCache.lastError`; listing serves the +available snapshot without exposing those names. ## Addressable surfaces diff --git a/packages/gateway/src/control-plane/shared/warm-models-cache.ts b/packages/gateway/src/control-plane/shared/warm-models-cache.ts index c680566eb..aa29774fe 100644 --- a/packages/gateway/src/control-plane/shared/warm-models-cache.ts +++ b/packages/gateway/src/control-plane/shared/warm-models-cache.ts @@ -10,10 +10,10 @@ import { logInfo } from '@floway-dev/provider-claude-code'; const errorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error); -// Populate the SWR model cache synchronously after saving an upstream so the -// next dashboard read sees the new catalog. The cache layer persists upstream -// fetch failures in `lastError`; errors escaping that layer are internal and -// must remain observable without aborting the surrounding control-plane write. +// Wait synchronously for an eligible refresh or an active owner. A persisted +// cooldown suppresses a new attempt and leaves the current snapshot in place. +// Refresh failures persist in `lastError`; errors escaping the cache layer are +// internal and remain observable without aborting the control-plane write. // // Returns what the row holds afterwards so the caller can answer with the // freshness this warm produced rather than the snapshot it read before saving. diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index 1db8f825f..289382a57 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -15,8 +15,8 @@ import { isAbortError, type Fetcher, type ModelCandidate } from '@floway-dev/pro // apply: an `unprefixed`-addressable upstream is probed with the inbound id // verbatim; a `prefixed`-addressable upstream is probed with the inbound id // minus its configured prefix when (and only when) the inbound carries that -// prefix. Both branches are evaluated against the same SWR-cached catalog -// fetch — a single upstream typically contributes at most one candidate, +// prefix. Both branches are evaluated against the same persisted catalog +// snapshot — a single upstream typically contributes at most one candidate, // but a catalog that publishes both the bare and prefixed forms can match // twice and both go through. // @@ -192,9 +192,8 @@ export const enumerateModelCandidates = async ({ upstreamIds: readonly string[] | null; model: string; kind: ModelKind; - // Threaded into `enumerateRealModelCandidates` so the per-upstream - // catalog lookup hits the SWR-cached `fetchUpstreamModelsCached` instead - // of round-tripping to the upstream on every request. + // Threaded into `enumerateRealModelCandidates` so stale snapshot access can + // submit or join a separate background refresh trigger. scheduler: BackgroundScheduler; // Runtime location tag for this request — see GatewayCtx.runtimeLocation. // Threaded into the per-request fetcher so colo-scoped fallback entries diff --git a/packages/gateway/src/data-plane/shared/listing/addressable.ts b/packages/gateway/src/data-plane/shared/listing/addressable.ts index aa1ca2895..7194aa0ad 100644 --- a/packages/gateway/src/data-plane/shared/listing/addressable.ts +++ b/packages/gateway/src/data-plane/shared/listing/addressable.ts @@ -47,9 +47,9 @@ export const listedRealModels = (entries: readonly AddressableIdEntry[]): readon // Enumerate every inbound id the data plane accepts under `upstreamFilter`, // tagged with whether the id participates in the default `/v1/models` -// listing. Fans out per upstream the same way `collectProviderModels` does, -// re-uses the SWR cache so the catalog refresh round-trip is shared with -// `getModelsFromProviders`. +// listing. Fans out persisted snapshot reads the same way +// `collectProviderModels` does; repeated stale access may submit or join the +// same separate background refresh trigger. export const enumerateAddressableModelIds = async ( upstreamFilter: readonly string[] | null, fetcherForUpstream: (upstreamId: string) => Fetcher, From 87f897ab2e3608872e4ec7556f7366b2aa40aa35 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 04:50:26 +0800 Subject: [PATCH 29/46] fix(gateway): close refresh coordination races Track active owner tokens so cross-runtime warms observe durable completion without timestamp heuristics, let explicit force bypass local warm waiters, and avoid false backoff when claim release fails. Preserve cooldown across metadata generations while invalidating active owners, clear stale catalogs when fetch identity changes, expose warm side effects honestly in test calls, and cover the composed route behavior. --- .../__tests__/node-sqlite-repo_test.ts | 2 +- .../control-plane/models/routes_test.ts | 20 ++--- .../control-plane/upstreams/routes_test.ts | 34 +++++++++ .../__tests__/data-plane/audio/http_test.ts | 38 +++++----- .../data-plane/codex/routes_images_test.ts | 8 +- .../data-plane/completions/http_test.ts | 20 ++--- .../data-plane/embeddings/http_test.ts | 18 ++--- .../__tests__/data-plane/images/http_test.ts | 22 +++--- .../data-plane/models/gemini_test.ts | 24 +++--- .../__tests__/data-plane/models/http_test.ts | 38 +++++----- .../data-plane/providers/models-cache_test.ts | 29 ++++++- .../__tests__/data-plane/rerank/serve_test.ts | 36 ++++----- .../shared/passthrough-serve_test.ts | 18 ++--- packages/gateway/__tests__/repo/memory.ts | 9 ++- .../__tests__/repo/models-cache-fixture.ts | 4 +- .../__tests__/repo/models-refresh_test.ts | 31 ++++---- .../src/control-plane/upstreams/routes.ts | 6 +- .../src/data-plane/providers/models-cache.ts | 76 +++++++++++-------- packages/gateway/src/repo/sql.ts | 41 ++++++---- packages/gateway/src/repo/types.ts | 5 +- 20 files changed, 289 insertions(+), 190 deletions(-) diff --git a/apps/platform-node/__tests__/node-sqlite-repo_test.ts b/apps/platform-node/__tests__/node-sqlite-repo_test.ts index 248e3ff95..4eac84374 100644 --- a/apps/platform-node/__tests__/node-sqlite-repo_test.ts +++ b/apps/platform-node/__tests__/node-sqlite-repo_test.ts @@ -99,7 +99,7 @@ test('repository JSON codecs round-trip upstream, alias, and Responses state thr config: { opaque: { value: true } }, }; const cacheToken = 'node-cache-fixture'; - const cacheClaim = await repo.upstreams.claimModelsRefresh('up_node', cacheGeneration, cacheToken, Date.now(), Number.MIN_SAFE_INTEGER, true); + const cacheClaim = await repo.upstreams.claimModelsRefresh('up_node', cacheGeneration, cacheToken, Date.now(), Number.MIN_SAFE_INTEGER, true, null); if (cacheClaim.kind !== 'claimed') throw new Error('expected model-cache fixture claim'); await repo.upstreams.saveClaimedModelsCache('up_node', cacheGeneration, cacheToken, { revision: MODEL_CATALOG_REVISION, diff --git a/packages/gateway/__tests__/control-plane/models/routes_test.ts b/packages/gateway/__tests__/control-plane/models/routes_test.ts index 7fa8423d2..da31c469a 100644 --- a/packages/gateway/__tests__/control-plane/models/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/models/routes_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { buildCustomUpstreamRecord, copilotModels, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, copilotModels, requestAppWithWarmModels, setupAppTest } from '../../test-utils/app.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; import { assert, assertEquals, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -36,7 +36,7 @@ test('/api/models returns an empty catalog when the gateway has no upstreams', a const { adminSession, repo } = await setupAppTest(); await repo.upstreams.deleteAll(); - const response = await requestApp('/api/models?aliases=false&include_unlisted=true', { + const response = await requestAppWithWarmModels('/api/models?aliases=false&include_unlisted=true', { headers: { 'x-floway-session': adminSession }, }); assertEquals(response.status, 200); @@ -70,7 +70,7 @@ test('/api/models exposes each upstream as { kind, id } so multi-provider models throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/api/models', { headers: { 'x-api-key': apiKey.key } }); + const response = await requestAppWithWarmModels('/api/models', { headers: { 'x-api-key': apiKey.key } }); assertEquals(response.status, 200); const body = (await response.json()) as { data: Array> }; @@ -122,7 +122,7 @@ test('/api/models is scoped to the caller\'s effective upstreams — a removed u const session = (await repo.sessions.create(2)).id; await withMockedFetch(modelsFetchHandler, async () => { - const response = await requestApp('/api/models', { headers: { 'x-floway-session': session } }); + const response = await requestAppWithWarmModels('/api/models', { headers: { 'x-floway-session': session } }); assertEquals(response.status, 200); const body = (await response.json()) as { data: Array<{ id: string }> }; const ids = body.data.map(model => model.id).sort(); @@ -137,7 +137,7 @@ test('/api/models appends visible alias entries with aliasedFrom alongside real await repo.upstreams.save(buildCustomUpstreamRecord({ id: 'up_custom_models', sortOrder: 100 })); await withMockedFetch(modelsFetchHandler, async () => { - const response = await requestApp('/api/models', { headers: { 'x-api-key': apiKey.key } }); + const response = await requestAppWithWarmModels('/api/models', { headers: { 'x-api-key': apiKey.key } }); assertEquals(response.status, 200); const body = (await response.json()) as { data: Array<{ id: string; display_name: string; upstreams: Array<{ kind: string; id: string; name: string }> }> }; assertEquals(body.data.some(model => model.id === 'custom-model'), true); @@ -166,7 +166,7 @@ test('/api/models for an admin session returns the gateway-wide catalog, bypassi }); await withMockedFetch(modelsFetchHandler, async () => { - const response = await requestApp('/api/models', { headers: { 'x-floway-session': adminSession } }); + const response = await requestAppWithWarmModels('/api/models', { headers: { 'x-floway-session': adminSession } }); assertEquals(response.status, 200); const ids = ((await response.json()) as { data: Array<{ id: string }> }).data.map(m => m.id).sort(); assertEquals(ids.includes('azure-public'), true); @@ -211,7 +211,7 @@ test('/api/models — admin sees raw alias.targets; non-admin sees the caller-na const nonAdminSession = (await repo.sessions.create(2)).id; await withMockedFetch(modelsFetchHandler, async () => { - const adminResponse = await requestApp('/api/models', { headers: { 'x-floway-session': adminSession } }); + const adminResponse = await requestAppWithWarmModels('/api/models', { headers: { 'x-floway-session': adminSession } }); assertEquals(adminResponse.status, 200); const adminBody = (await adminResponse.json()) as { data: Array<{ id: string; aliasedFrom?: { targets: Array<{ target_model_id: string }> } }> }; const adminMix = adminBody.data.find(m => m.id === 'mix'); @@ -221,7 +221,7 @@ test('/api/models — admin sees raw alias.targets; non-admin sees the caller-na ['custom-model', 'typo-no-such-model'], ); - const nonAdminResponse = await requestApp('/api/models', { headers: { 'x-floway-session': nonAdminSession } }); + const nonAdminResponse = await requestAppWithWarmModels('/api/models', { headers: { 'x-floway-session': nonAdminSession } }); assertEquals(nonAdminResponse.status, 200); const nonAdminBody = (await nonAdminResponse.json()) as { data: Array<{ id: string; aliasedFrom?: { targets: Array<{ target_model_id: string }> } }> }; const nonAdminMix = nonAdminBody.data.find(m => m.id === 'mix'); @@ -294,8 +294,8 @@ test('/api/models — admin self-restriction does NOT leak per-alias metadata va await withMockedFetch(() => { throw new Error('unexpected outbound fetch'); }, async () => { const [adminRes, nonAdminRes] = await Promise.all([ - requestApp('/api/models', { headers: { 'x-floway-session': adminSession } }), - requestApp('/api/models', { headers: { 'x-floway-session': nonAdminSession } }), + requestAppWithWarmModels('/api/models', { headers: { 'x-floway-session': adminSession } }), + requestAppWithWarmModels('/api/models', { headers: { 'x-floway-session': nonAdminSession } }), ]); const adminBody = (await adminRes.json()) as { data: Array<{ id: string; limits?: { max_context_window_tokens?: number } }> }; const nonAdminBody = (await nonAdminRes.json()) as { data: Array<{ id: string; limits?: { max_context_window_tokens?: number } }> }; diff --git a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts index a932b0fb9..5e7eac646 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts @@ -3,6 +3,7 @@ import { test } from 'vitest'; import { blueprintUpstreamRecord, upstreamRecordToFullJson } from '../../../src/control-plane/upstreams/serialize.ts'; import { MODEL_LISTING_FAILURE_CODE } from '../../../src/data-plane/models/shared.ts'; import { MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; +import { modelsRefreshRetryAt } from '../../../src/repo/models-refresh-contract.ts'; import { seedModelsCache, seedModelsCacheError } from '../../repo/models-cache-fixture.ts'; import { MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import type { UpstreamProviderKind, UpstreamRecord } from '@floway-dev/provider'; @@ -779,6 +780,39 @@ test('PATCH /api/upstreams warms the models cache before responding', async () = assertEquals(patched.modelsCache.lastError, null); }); +test('PATCH /api/upstreams metadata warm preserves refresh backoff', async () => { + const { repo, adminSession } = await setupAppTest(); + await repo.upstreams.deleteAll(); + const created = await withMockedFetch( + () => jsonResponse({ object: 'list', data: [{ id: 'cached-model' }] }), + async () => await (await requestApp('/api/upstreams', authed(adminSession, createBody()))).json() as { id: string }, + ); + const generation = await getCacheGeneration(repo, created.id); + const now = Date.now(); + const claim = await repo.upstreams.claimModelsRefresh(created.id, generation, 'failed-refresh', now, now - 900_000, false, null); + if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); + await repo.upstreams.completeModelsRefreshFailure(created.id, 'failed-refresh', 1, modelsRefreshRetryAt(now, 0)); + let modelRequests = 0; + + await withMockedFetch( + () => { + modelRequests++; + return jsonResponse({ object: 'list', data: [{ id: 'unexpected-model' }] }); + }, + async () => { + const response = await requestApp(`/api/upstreams/${created.id}`, { + method: 'PATCH', + headers: { 'content-type': 'application/json', 'x-floway-session': adminSession }, + body: JSON.stringify({ name: 'Metadata only' }), + }); + assertEquals(response.status, 200); + }, + ); + + assertEquals(modelRequests, 0); + assertEquals((await repo.upstreams.getById(created.id))?.modelsCache?.models.map(model => model.id), ['cached-model']); +}); + test('POST /api/upstreams/list-models without an id still serves draft preview', async () => { const { adminSession } = await setupAppTest(); diff --git a/packages/gateway/__tests__/data-plane/audio/http_test.ts b/packages/gateway/__tests__/data-plane/audio/http_test.ts index 3db7f001a..251be70cc 100644 --- a/packages/gateway/__tests__/data-plane/audio/http_test.ts +++ b/packages/gateway/__tests__/data-plane/audio/http_test.ts @@ -1,7 +1,7 @@ import { test, vi } from 'vitest'; import type { InMemoryRepo } from '../../repo/memory.ts'; -import { flushAsyncWork, MOCKED_FETCH_EGRESS, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { flushAsyncWork, MOCKED_FETCH_EGRESS, requestAppWithWarmModels, setupAppTest } from '../../test-utils/app.ts'; import type { ModelPricing } from '@floway-dev/protocols/common'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; @@ -56,7 +56,7 @@ const transcriptionForm = (fields: readonly [string, string][] = []): FormData = test('/v1/audio/transcriptions requires multipart model and file fields', async () => { const { apiKey } = await setupAppTest(); - const json = await requestApp('/v1/audio/transcriptions', { + const json = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: '{}', @@ -65,7 +65,7 @@ test('/v1/audio/transcriptions requires multipart model and file fields', async const missingFile = new FormData(); missingFile.append('model', 'gpt-4o-transcribe'); - const noFile = await requestApp('/v1/audio/transcriptions', { + const noFile = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: missingFile, }); assertEquals(noFile.status, 400); @@ -90,7 +90,7 @@ test('/v1/audio/transcriptions preserves multipart fields, headers, JSON body, a }); }, async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([ @@ -136,7 +136,7 @@ test('/v1/audio/transcriptions forwards VTT verbatim and records request-only us headers: { 'content-type': 'text/vtt', 'x-subtitle-source': 'upstream' }, }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['response_format', 'vtt']]), }); assertEquals(response.headers.get('content-type'), 'text/vtt'); @@ -158,7 +158,7 @@ test('/v1/audio/transcriptions skips JSON parsing for text responses without war await withMockedFetch( () => new Response('plain transcript', { headers: { 'content-type': 'text/plain' } }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['response_format', 'text']]), }); assertEquals(await response.text(), 'plain transcript'); @@ -178,7 +178,7 @@ test('/v1/audio/transcriptions warns on malformed declared JSON while forwarding await withMockedFetch( () => new Response('{not-json', { headers: { 'content-type': 'application/json' } }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm(), }); assertEquals(response.status, 200); @@ -202,7 +202,7 @@ test('/v1/audio/transcriptions preserves unknown future usage metrics as request await withMockedFetch( () => Response.json(upstreamBody), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm(), }); assertEquals(response.status, 200); @@ -227,7 +227,7 @@ test('/v1/audio/transcriptions preserves malformed declared usage and records re { headers: { 'x-provider-trace': 'malformed-usage', 'set-cookie': 'upstream-session=secret' } }, ), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm(), }); assertEquals(response.status, 200); @@ -255,7 +255,7 @@ test('/v1/audio/transcriptions does not invent a content type for an untyped raw await withMockedFetch( () => new Response(new TextEncoder().encode('plain transcript')), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['response_format', 'text']]), }); assertEquals(response.headers.get('content-type'), null); @@ -272,7 +272,7 @@ test('/v1/audio/transcriptions records duration under the per-second metric', as await withMockedFetch( () => Response.json({ text: 'hello', duration: 91.8, usage: { type: 'duration', seconds: 91 } }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['response_format', 'verbose_json']]), }); assertEquals(response.status, 200); @@ -292,7 +292,7 @@ test('/v1/audio/transcriptions preserves duration usage unpriced when the model await withMockedFetch( () => Response.json({ text: 'hello', usage: { type: 'duration', seconds: 75 } }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm(), }); assertEquals(response.status, 200); @@ -313,7 +313,7 @@ test('/v1/audio/transcriptions preserves token usage unpriced when the model is await withMockedFetch( () => Response.json({ text: 'hello', usage: { type: 'tokens', input_tokens: 12, output_tokens: 8, total_tokens: 20 } }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm(), }); assertEquals(response.status, 200); @@ -342,7 +342,7 @@ test('/v1/audio/transcriptions streams through transcript.text.done without addi '', ].join('\n'), { headers: { 'content-type': 'Text/Event-Stream; charset=utf-8', 'x-stream-trace': 'trace-sse' } }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['stream', 'true']]), }); assertEquals(response.status, 200); @@ -375,7 +375,7 @@ test('/v1/audio/transcriptions preserves a terminal stream event with malformed { headers: { 'content-type': 'text/event-stream' } }, ), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['stream', 'true']]), }); assertEquals(response.status, 200); @@ -410,7 +410,7 @@ test('/v1/audio/transcriptions completes and cancels an upstream kept open after }, }), { headers: { 'content-type': 'text/event-stream' } }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['stream', 'true']]), }); const text = await response.text(); @@ -429,7 +429,7 @@ test('/v1/audio/transcriptions treats EOF without transcript.text.done as a fail headers: { 'content-type': 'text/event-stream' }, }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['stream', 'true']]), }); assertEquals(response.status, 200); @@ -449,7 +449,7 @@ test('/v1/audio/transcriptions counts a bodyless SSE response as a failed reques await withMockedFetch( () => new Response(null, { headers: { 'content-type': 'text/event-stream', 'x-empty-trace': 'empty-sse' } }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm([['stream', 'true']]), }); assertEquals(response.status, 502); @@ -472,7 +472,7 @@ test('/v1/audio/transcriptions forwards exhausted upstream errors and records th headers: { 'content-type': 'application/json', 'retry-after': '4', 'x-error-trace': 'trace-error' }, }), async () => { - const response = await requestApp('/v1/audio/transcriptions', { + const response = await requestAppWithWarmModels('/v1/audio/transcriptions', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: transcriptionForm(), }); assertEquals(response.status, 422); diff --git a/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts b/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts index 12bb2ddfc..2795372fe 100644 --- a/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts +++ b/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import type { InMemoryRepo } from '../../repo/memory.ts'; -import { copilotModels, MOCKED_FETCH_EGRESS, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { copilotModels, MOCKED_FETCH_EGRESS, requestAppWithWarmModels, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; const PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/wEAAAAASUVORK5CYII='; @@ -64,7 +64,7 @@ test('Codex provider-relative image generation reuses the public image-generatio throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/azure-api.codex/images/generations', { + const response = await requestAppWithWarmModels('/azure-api.codex/images/generations', { method: 'POST', headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a fox in space', quality: 'high' }), @@ -99,7 +99,7 @@ test('Codex provider-relative image edits reuse the public JSON handler', async throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/azure-api.codex/images/edits', { + const response = await requestAppWithWarmModels('/azure-api.codex/images/edits', { method: 'POST', headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, body: JSON.stringify({ @@ -142,7 +142,7 @@ test('Codex inline data URL edits egress as multipart uploads', async () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/azure-api.codex/images/edits', { + const response = await requestAppWithWarmModels('/azure-api.codex/images/edits', { method: 'POST', headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, body: JSON.stringify({ diff --git a/packages/gateway/__tests__/data-plane/completions/http_test.ts b/packages/gateway/__tests__/data-plane/completions/http_test.ts index 21ff80222..207d9c750 100644 --- a/packages/gateway/__tests__/data-plane/completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/completions/http_test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; import { installDumpStubs } from '../../dump/test-fixtures.ts'; -import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -70,7 +70,7 @@ test('/v1/completions non-streaming forwards body to upstream /v1/completions an throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/completions', { + const response = await requestAppWithWarmModels('/v1/completions', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'davinci-002', prompt: 'hello' }), @@ -108,7 +108,7 @@ test('/v1/completions streaming forces stream_options.include_usage upstream', a throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/completions', { + const response = await requestAppWithWarmModels('/v1/completions', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'davinci-002', prompt: 'hello', stream: true }), @@ -136,7 +136,7 @@ test('/v1/completions streaming strips usage chunk when client did not request i throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/completions', { + const response = await requestAppWithWarmModels('/v1/completions', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'davinci-002', prompt: 'hello', stream: true }), @@ -171,7 +171,7 @@ test('/v1/completions streaming forwards usage chunk when the client opted in', throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/completions', { + const response = await requestAppWithWarmModels('/v1/completions', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ @@ -192,7 +192,7 @@ test('/v1/completions streaming forwards usage chunk when the client opted in', test('/v1/completions rejects malformed body with the standard 400', async () => { const { apiKey } = await setupAppTest(); - const response = await requestApp('/v1/completions', { + const response = await requestAppWithWarmModels('/v1/completions', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: '{not json', @@ -225,7 +225,7 @@ test('/v1/completions rejects a model without the completions endpoint with the })); await warmModelsForTest(); - const response = await requestApp('/v1/completions', { + const response = await requestAppWithWarmModels('/v1/completions', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'davinci-002', prompt: 'hello' }), @@ -255,7 +255,7 @@ test('/v1/completions handler also serves the unversioned /completions path', as throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/completions', { + const response = await requestAppWithWarmModels('/completions', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'davinci-002', prompt: 'x' }), @@ -288,7 +288,7 @@ test('/v1/completions non-streaming records usage row, performance neutral row ( usage: { prompt_tokens: 7, completion_tokens: 2, total_tokens: 9 }, })), async () => { - const response = await requestApp('/v1/completions', { + const response = await requestAppWithWarmModels('/v1/completions', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'davinci-002', prompt: 'hello' }), @@ -332,7 +332,7 @@ test('/v1/completions streaming records usage row, performance neutral row (text await withMockedFetch( () => Promise.resolve(completionStream()), async () => { - const response = await requestApp('/v1/completions', { + const response = await requestAppWithWarmModels('/v1/completions', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'davinci-002', prompt: 'hello', stream: true }), diff --git a/packages/gateway/__tests__/data-plane/embeddings/http_test.ts b/packages/gateway/__tests__/data-plane/embeddings/http_test.ts index cde73f4a1..63b405afa 100644 --- a/packages/gateway/__tests__/data-plane/embeddings/http_test.ts +++ b/packages/gateway/__tests__/data-plane/embeddings/http_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; -import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, requestAppWithWarmModels, setupAppTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; @@ -46,7 +46,7 @@ test('/v1/embeddings wraps scalar string input for Copilot upstream', async () = throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', @@ -101,7 +101,7 @@ test('/v1/embeddings records usage under request model when upstream omits model throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', @@ -165,7 +165,7 @@ test('/v1/embeddings records request and upstream performance', async () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', @@ -242,7 +242,7 @@ test('/v1/embeddings routes to custom upstream when model is only declared there throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', @@ -303,7 +303,7 @@ test('/v1/embeddings rejects model on custom upstream without /embeddings capabi throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', @@ -359,7 +359,7 @@ test('/v1/embeddings reports the failed upstream parenthetically when /v1/models throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', @@ -432,7 +432,7 @@ test('/v1/embeddings reports the failed upstream even when a sibling upstream\'s throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', @@ -475,7 +475,7 @@ test('/v1/embeddings rejects malformed body at the provider-independent boundary throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', diff --git a/packages/gateway/__tests__/data-plane/images/http_test.ts b/packages/gateway/__tests__/data-plane/images/http_test.ts index ad28eb3ca..67f10565d 100644 --- a/packages/gateway/__tests__/data-plane/images/http_test.ts +++ b/packages/gateway/__tests__/data-plane/images/http_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; -import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, MOCKED_FETCH_EGRESS, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, MOCKED_FETCH_EGRESS, requestAppWithWarmModels, setupAppTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; @@ -9,7 +9,7 @@ const PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8 test('/v1/images/generations rejects malformed JSON body with 400', async () => { const { apiKey } = await setupAppTest(); - const response = await requestApp('/v1/images/generations', { + const response = await requestAppWithWarmModels('/v1/images/generations', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: 'not json', @@ -19,7 +19,7 @@ test('/v1/images/generations rejects malformed JSON body with 400', async () => test('/v1/images/generations rejects body without model with 400', async () => { const { apiKey } = await setupAppTest(); - const response = await requestApp('/v1/images/generations', { + const response = await requestAppWithWarmModels('/v1/images/generations', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ prompt: 'hi' }), @@ -43,7 +43,7 @@ test('/v1/images/generations 404s when no upstream provides the model', async () throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/images/generations', { + const response = await requestAppWithWarmModels('/v1/images/generations', { method: 'POST', headers: { 'content-type': 'application/json; charset=utf-8', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'no-such-model', prompt: 'hi' }), @@ -55,7 +55,7 @@ test('/v1/images/generations 404s when no upstream provides the model', async () test('/v1/images/edits rejects malformed JSON with 400', async () => { const { apiKey } = await setupAppTest(); - const response = await requestApp('/v1/images/edits', { + const response = await requestAppWithWarmModels('/v1/images/edits', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: 'not json', @@ -65,7 +65,7 @@ test('/v1/images/edits rejects malformed JSON with 400', async () => { test('/v1/images/edits rejects JSON without a model with 400', async () => { const { apiKey } = await setupAppTest(); - const response = await requestApp('/v1/images/edits', { + const response = await requestAppWithWarmModels('/v1/images/edits', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ prompt: 'hi', images: [{ file_id: 'file-image' }] }), @@ -77,7 +77,7 @@ test('/v1/images/edits rejects multipart body without model field with 400', asy const { apiKey } = await setupAppTest(); const form = new FormData(); form.append('prompt', 'hi'); - const response = await requestApp('/v1/images/edits', { + const response = await requestAppWithWarmModels('/v1/images/edits', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: form, @@ -116,7 +116,7 @@ test('/v1/images/generations rejects model on custom upstream without /images/ge throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/images/generations', { + const response = await requestAppWithWarmModels('/v1/images/generations', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'gpt-4o', prompt: 'hi' }), @@ -165,7 +165,7 @@ test('/v1/images/generations forwards a JSON request through a custom upstream a throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/images/generations', { + const response = await requestAppWithWarmModels('/v1/images/generations', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a shiba in space' }), @@ -235,7 +235,7 @@ test('/v1/images/edits forwards a multipart request through an Azure model and r form.append('model', 'gpt-image-2'); form.append('prompt', 'replace sky with aurora'); form.append('image', new Blob([new Uint8Array([1, 2, 3])], { type: 'image/png' }), 'photo.png'); - const response = await requestApp('/v1/images/edits', { + const response = await requestAppWithWarmModels('/v1/images/edits', { method: 'POST', headers: { 'x-api-key': apiKey.key }, body: form, @@ -289,7 +289,7 @@ test('/v1/images/edits forwards JSON image references through a custom provider' throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/images/edits', { + const response = await requestAppWithWarmModels('/v1/images/edits', { method: 'POST', headers: { 'content-type': 'Application/Vnd.OpenAI+JSON; charset=utf-8', 'x-api-key': apiKey.key }, body: JSON.stringify({ diff --git a/packages/gateway/__tests__/data-plane/models/gemini_test.ts b/packages/gateway/__tests__/data-plane/models/gemini_test.ts index 40389ac4c..289e1c16e 100644 --- a/packages/gateway/__tests__/data-plane/models/gemini_test.ts +++ b/packages/gateway/__tests__/data-plane/models/gemini_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { buildCustomUpstreamRecord, copilotModels, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, copilotModels, requestAppWithWarmModels, setupAppTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals } from '@floway-dev/test-utils'; @@ -62,7 +62,7 @@ test('/v1beta/models lists Copilot LLM models in Gemini model shape', async () = throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1beta/models', { + const response = await requestAppWithWarmModels('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); @@ -110,7 +110,7 @@ test('/v1beta/models/:modelId returns one Gemini model or Google RPC 404', async throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const found = await requestApp('/v1beta/models/gpt-gemini-get', { + const found = await requestAppWithWarmModels('/v1beta/models/gpt-gemini-get', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(found.status, 200); @@ -118,7 +118,7 @@ test('/v1beta/models/:modelId returns one Gemini model or Google RPC 404', async assertEquals(model.name, 'models/gpt-gemini-get'); assertEquals(model.supportedGenerationMethods, ['generateContent', 'streamGenerateContent', 'countTokens']); - const missing = await requestApp('/v1beta/models/missing-model', { + const missing = await requestAppWithWarmModels('/v1beta/models/missing-model', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(missing.status, 404); @@ -166,7 +166,7 @@ test('/v1beta/models includes custom upstream LLM models', async () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const listResp = await requestApp('/v1beta/models', { + const listResp = await requestAppWithWarmModels('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(listResp.status, 200); @@ -176,7 +176,7 @@ test('/v1beta/models includes custom upstream LLM models', async () => { assertEquals(list.models[0].displayName, 'Custom LLM Model'); assertEquals(list.models[0].supportedGenerationMethods, ['generateContent', 'streamGenerateContent', 'countTokens']); - const getResp = await requestApp('/v1beta/models/custom-llm-model', { + const getResp = await requestAppWithWarmModels('/v1beta/models/custom-llm-model', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(getResp.status, 200); @@ -219,7 +219,7 @@ test('/v1beta/models excludes custom upstream embedding-only models', async () = throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const listResp = await requestApp('/v1beta/models', { + const listResp = await requestAppWithWarmModels('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(listResp.status, 200); @@ -259,7 +259,7 @@ test('/v1beta/models hides upstream identity when a provider returns an invalid throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1beta/models', { + const response = await requestAppWithWarmModels('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(response.status, 200); @@ -301,7 +301,7 @@ test('/v1beta/models hides upstream HTTP error bodies', async () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1beta/models', { + const response = await requestAppWithWarmModels('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(response.status, 200); @@ -341,7 +341,7 @@ test('/v1beta/models hides thrown upstream request errors', async () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1beta/models', { + const response = await requestAppWithWarmModels('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(response.status, 200); @@ -384,7 +384,7 @@ test('/v1beta/models hides malformed upstream response bodies', async () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1beta/models', { + const response = await requestAppWithWarmModels('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(response.status, 200); @@ -440,7 +440,7 @@ test('/v1beta/models emits visible aliases as models/ entries with d throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1beta/models', { + const response = await requestAppWithWarmModels('/v1beta/models', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(response.status, 200); diff --git a/packages/gateway/__tests__/data-plane/models/http_test.ts b/packages/gateway/__tests__/data-plane/models/http_test.ts index 940e9c207..16d31b3d5 100644 --- a/packages/gateway/__tests__/data-plane/models/http_test.ts +++ b/packages/gateway/__tests__/data-plane/models/http_test.ts @@ -1,6 +1,6 @@ import { expect, test, vi } from 'vitest'; -import { buildCopilotUpstreamRecord, buildCustomUpstreamRecord, copilotModels, flushAsyncWork, requestApp as requestAppCold, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCopilotUpstreamRecord, buildCustomUpstreamRecord, copilotModels, flushAsyncWork, requestApp as requestAppCold, requestAppWithWarmModels, setupAppTest } from '../../test-utils/app.ts'; import type { ModelKind } from '@floway-dev/protocols/common'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals } from '@floway-dev/test-utils'; @@ -93,7 +93,7 @@ test('/v1/models returns merged model list from Copilot and custom upstreams', a throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/models', { + const response = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key }, }); @@ -156,14 +156,14 @@ test('/v1/models returns merged model list from Copilot and custom upstreams', a assertEquals(model.description, undefined); } - const anthropicResponse = await requestApp('/models', { + const anthropicResponse = await requestAppWithWarmModels('/models', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(anthropicResponse.status, 200); assertEquals(await anthropicResponse.json(), body); // Dashboard adds two UI-only fields on top of the public DTO. - const controlResponse = await requestApp('/api/models', { + const controlResponse = await requestAppWithWarmModels('/api/models', { headers: { 'x-api-key': apiKey.key }, }); assertEquals(controlResponse.status, 200); @@ -234,7 +234,7 @@ test('Codex User-Agents receive the Codex catalog from root model-list paths', a ['/models', 'codex-tui/0.0.1-unified.catalog'], ['/v1/models', 'codex_cli_rs/0.0.1-unified.catalog'], ] as const) { - const codexResponse = await requestApp(path, { + const codexResponse = await requestAppWithWarmModels(path, { headers: { 'x-api-key': apiKey.key, 'user-agent': userAgent }, }); assertEquals(codexResponse.status, 200); @@ -306,7 +306,7 @@ test('/models returns the same superset payload as /v1/models', async () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/models', { + const response = await requestAppWithWarmModels('/models', { headers: { 'x-api-key': apiKey.key }, }); @@ -382,7 +382,7 @@ test('/v1/models hides upstream identity when a provider returns an invalid mode throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/models', { + const response = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key }, }); @@ -439,7 +439,7 @@ test('/v1/models surfaces healthy upstream models when another upstream catalog throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/models', { + const response = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key }, }); @@ -484,7 +484,7 @@ test('public model list endpoints hide upstream HTTP error bodies and headers', }, async () => { for (const path of ['/v1/models', '/models', '/api/models']) { - const response = await requestApp(path, { + const response = await requestAppWithWarmModels(path, { headers: { 'x-api-key': apiKey.key }, }); assertEquals(response.status, 200); @@ -524,7 +524,7 @@ test('public model list endpoints hide thrown upstream request errors', async () }, async () => { for (const path of ['/v1/models', '/models', '/api/models']) { - const response = await requestApp(path, { + const response = await requestAppWithWarmModels(path, { headers: { 'x-api-key': apiKey.key }, }); assertEquals(response.status, 200); @@ -565,7 +565,7 @@ test('public model list endpoints hide malformed upstream response bodies', asyn }, async () => { for (const path of ['/v1/models', '/models', '/api/models']) { - const response = await requestApp(path, { + const response = await requestAppWithWarmModels(path, { headers: { 'x-api-key': apiKey.key }, }); assertEquals(response.status, 200); @@ -582,7 +582,7 @@ test('/v1/models surfaces the actionable "no upstream configured" hint when no p await repo.upstreams.deleteAll(); clearInProcessCopilotTokenCache(); - const response = await requestApp('/v1/models', { + const response = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key }, }); @@ -646,7 +646,7 @@ test('/v1/models returns the id-sorted union of every connected GitHub account', throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/models', { + const response = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key }, }); @@ -695,7 +695,7 @@ test('/v1/models returns the last real error when every account model load fails throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/models', { + const response = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key }, }); @@ -782,7 +782,7 @@ test('/v1/models appends visible aliases with their aliasedFrom block and folds throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/models', { headers: { 'x-api-key': apiKey.key } }); + const response = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key } }); assertEquals(response.status, 200); const body = (await response.json()) as { data: Array<{ id: string; display_name: string; aliasedFrom?: { selection: string } }> }; const ids = body.data.map(model => model.id); @@ -859,7 +859,7 @@ test('/v1/models folds a real-id collision onto the alias even when the alias po throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/models', { headers: { 'x-api-key': apiKey.key } }); + const response = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key } }); assertEquals(response.status, 200); const body = (await response.json()) as { data: Array<{ id: string; display_name: string; aliasedFrom?: { selection: string } }> }; const shadowRows = body.data.filter(model => model.id === 'orphan-shadow'); @@ -917,7 +917,7 @@ test('/v1/models serves Anthropic-shape rows with a [1m] suffix on 1M-capable id throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const claudeCodeResp = await requestApp('/v1/models', { + const claudeCodeResp = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key, 'user-agent': 'claude-code/2.1.206' }, }); assertEquals(claudeCodeResp.status, 200); @@ -980,7 +980,7 @@ test('/v1/models serves Anthropic-shape rows with a [1m] suffix on 1M-capable id assertEquals(haiku.max_tokens, 64_000); // Non-Claude-Code caller: Floway's PublicModel superset is unchanged. - const openAiResp = await requestApp('/v1/models', { + const openAiResp = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key, 'user-agent': 'openai-python/1.42.0' }, }); assertEquals(openAiResp.status, 200); @@ -1035,7 +1035,7 @@ test('/v1/models serves Anthropic-shape rows without a [1m] suffix when no model throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/models', { + const response = await requestAppWithWarmModels('/v1/models', { headers: { 'x-api-key': apiKey.key, 'user-agent': 'claude-code/2.1.206' }, }); assertEquals(response.status, 200); diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 44bb5b94a..deb9d8d62 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -248,7 +248,7 @@ describe('fetchUpstreamModelsCached', () => { test('synchronous warm waits for a refresh owned by another runtime', async () => { const repo = await setupRepo(); const now = Date.now(); - await expect(repo.upstreams.claimModelsRefresh(UPSTREAM_ID, CACHE_GENERATION, 'remote-owner', now, now - 900_000, false)) + await expect(repo.upstreams.claimModelsRefresh(UPSTREAM_ID, CACHE_GENERATION, 'remote-owner', now, now - 900_000, false, null)) .resolves.toEqual({ kind: 'claimed', failureCount: 0 }); const localFetch = vi.fn(async () => [aModel('duplicate-local-model')]); const warming = warmUpstreamModels(stubInstance(localFetch), directFetcher); @@ -269,6 +269,33 @@ describe('fetchUpstreamModelsCached', () => { expect(localFetch).not.toHaveBeenCalled(); }); + test('explicit force bypasses a local warm waiting on another runtime', async () => { + const repo = await setupRepo(); + const now = Date.now(); + await repo.upstreams.claimModelsRefresh(UPSTREAM_ID, CACHE_GENERATION, 'remote-owner', now, now - 900_000, false, null); + const fetchFn = vi.fn(async () => [aModel('forced-model')]); + const instance = stubInstance(fetchFn, null, CACHE_GENERATION, 'shared-warm-force-key'); + const warming = warmUpstreamModels(instance, directFetcher); + await new Promise(resolve => setTimeout(resolve, 20)); + + const forced = fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher, force: true }); + await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); + expect((await forced).map(model => model.id)).toEqual(['forced-model']); + expect((await warming).map(model => model.id)).toEqual(['forced-model']); + }); + + test('a success-claim release failure does not install upstream failure backoff', async () => { + const repo = await setupRepo(); + const completeFailure = vi.spyOn(repo.upstreams, 'completeModelsRefreshFailure'); + vi.spyOn(repo.upstreams, 'completeModelsRefreshSuccess').mockRejectedValueOnce(new Error('release failed')); + const instance = stubInstance(async () => [aModel('published-model')]); + + await expect(fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher, force: true })) + .rejects.toThrow('release failed'); + expect(completeFailure).not.toHaveBeenCalled(); + expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['published-model']); + }); + test('a superseded generation neither joins nor overwrites the current catalog', async () => { const repo = await setupRepo(); let resolveOld: ((models: ProviderModel[]) => void) | null = null; diff --git a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts index 88e62dca8..0fb4bdc0a 100644 --- a/packages/gateway/__tests__/data-plane/rerank/serve_test.ts +++ b/packages/gateway/__tests__/data-plane/rerank/serve_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import type { Repo } from '../../../src/repo/types.ts'; -import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels, setupAppTest } from '../../test-utils/app.ts'; import type { ModelPricing, RerankTarget } from '@floway-dev/protocols/common'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -59,7 +59,7 @@ test('/v1/rerank translates Cohere v1 to v2 and records Cohere search units', as }), { status: 200, headers: { 'content-type': 'application/json', 'x-api-warning': 'trial quota', 'x-request-id': 'upstream-request' } }); }, async () => { - const response = await requestApp('/v1/rerank', { + const response = await requestAppWithWarmModels('/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ @@ -111,7 +111,7 @@ test('/v2/rerank accepts null Cohere meta and records request-only usage', async await withMockedFetch( () => jsonResponse({ id: 'request-no-usage', results: [], meta: null }), async () => { - const response = await requestApp('/v2/rerank', { + const response = await requestAppWithWarmModels('/v2/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }), @@ -134,7 +134,7 @@ test('/v2/rerank preserves same-protocol successes with malformed usage as reque await withMockedFetch( () => jsonResponse({ id: 'request-bad-usage', results: [], meta: { tokens: 3 } }), async () => { - const response = await requestApp('/v2/rerank', { + const response = await requestAppWithWarmModels('/v2/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }), @@ -174,7 +174,7 @@ test('/jina/v1/rerank preserves same-dialect extensions and records token usage' return jsonResponse(upstreamResponse); }, async () => { - const response = await requestApp('/jina/v1/rerank', { + const response = await requestAppWithWarmModels('/jina/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ @@ -214,7 +214,7 @@ test('/jina/v1/rerank accepts a same-dialect success without usage', async () => await withMockedFetch( () => jsonResponse(upstreamResponse), async () => { - const response = await requestApp('/jina/v1/rerank', { + const response = await requestAppWithWarmModels('/jina/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }), @@ -246,7 +246,7 @@ test('/jina/v1/rerank sends image inputs to DashScope native and accepts cross-p }); }, async () => { - const response = await requestApp('/jina/v1/rerank', { + const response = await requestAppWithWarmModels('/jina/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query, documents: [document] }), @@ -287,7 +287,7 @@ test('/voyage/v1/rerank translates a DashScope native response', async () => { }); }, async () => { - const response = await requestApp('/voyage/v1/rerank', { + const response = await requestAppWithWarmModels('/voyage/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ @@ -325,7 +325,7 @@ test('/v2/rerank rejects Cohere v1-only fields before dispatch', async () => { return jsonResponse({ results: [] }); }, async () => { - const response = await requestApp('/v2/rerank', { + const response = await requestAppWithWarmModels('/v2/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ @@ -352,7 +352,7 @@ test('upstream rerank errors are forwarded and still record a request-only usage headers: { 'content-type': 'application/json', 'retry-after': '7', 'x-request-id': 'request-error' }, }), async () => { - const response = await requestApp('/v1/rerank', { + const response = await requestAppWithWarmModels('/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }), @@ -386,7 +386,7 @@ test('a concrete token metric remains unpriced when only rerank searches have a results: [{ index: 0, relevance_score: 0.8 }], }), async () => { - const response = await requestApp('/jina/v1/rerank', { + const response = await requestAppWithWarmModels('/jina/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }), @@ -413,7 +413,7 @@ test('target-incompatible source controls return 400 without dispatch', async () return jsonResponse({ results: [] }); }, async () => { - const response = await requestApp('/jina/v1/rerank', { + const response = await requestAppWithWarmModels('/jina/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ @@ -441,7 +441,7 @@ test('Jina image inputs reject pure-text targets before dispatch', async () => { return jsonResponse({ results: [] }); }, async () => { - const response = await requestApp('/jina/v1/rerank', { + const response = await requestAppWithWarmModels('/jina/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ @@ -470,7 +470,7 @@ test('same-protocol success forwards opaque result items while still recording u await withMockedFetch( () => jsonResponse(upstreamBody), async () => { - const response = await requestApp('/jina/v1/rerank', { + const response = await requestAppWithWarmModels('/jina/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }), @@ -498,7 +498,7 @@ test('cross-protocol success still validates result items before rendering', asy results: [{ relevance_score: 0.8 }], }), async () => { - const response = await requestApp('/voyage/v1/rerank', { + const response = await requestAppWithWarmModels('/voyage/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }), @@ -523,7 +523,7 @@ test('same-protocol malformed JSON is forwarded as request-only usage', async () await withMockedFetch( () => new Response('{not-json', { status: 200, headers: { 'content-type': 'application/json' } }), async () => { - const response = await requestApp('/jina/v1/rerank', { + const response = await requestAppWithWarmModels('/jina/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }), @@ -549,7 +549,7 @@ test('usage parsed before a cross-protocol render failure is still recorded', as meta: { billed_units: { search_units: 2 } }, }), async () => { - const response = await requestApp('/v1/rerank', { + const response = await requestAppWithWarmModels('/v1/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ @@ -572,7 +572,7 @@ test('usage parsed before a cross-protocol render failure is still recorded', as test('there is no unversioned /rerank route', async () => { const { apiKey } = await setupAppTest(); - const response = await requestApp('/rerank', { + const response = await requestAppWithWarmModels('/rerank', { method: 'POST', headers: requestHeaders(apiKey.key), body: JSON.stringify({ model: 'public-reranker', query: 'query', documents: ['one'] }), diff --git a/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts b/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts index 36731db65..da31220a4 100644 --- a/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/passthrough-serve_test.ts @@ -17,7 +17,7 @@ import { test, vi } from 'vitest'; -import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels as requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels, setupAppTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; @@ -69,7 +69,7 @@ test('passthrough-serve: usage-record failure does not turn upstream 2xx into 50 throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', @@ -119,7 +119,7 @@ test('passthrough-serve: Custom resolves configured ingress header rules before throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', @@ -159,7 +159,7 @@ test('passthrough-serve: non-JSON 2xx upstream body is forwarded verbatim with a throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), @@ -218,7 +218,7 @@ test('passthrough-serve: response header blocklist preserves vendor metadata and throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), @@ -273,7 +273,7 @@ test('passthrough-serve: alias whose targets have no kind-matching binding surfa throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'embed-fast', input: 'hi' }), @@ -334,7 +334,7 @@ test('passthrough-serve: 5xx from the first candidate falls through to the next throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), @@ -372,7 +372,7 @@ test('passthrough-serve: when every candidate returns non-2xx the most recent up throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), @@ -414,7 +414,7 @@ test('passthrough-serve: throw during rollover attributes the error perf row to throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const response = await requestApp('/v1/embeddings', { + const response = await requestAppWithWarmModels('/v1/embeddings', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'custom-embed-model', input: 'hi' }), diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index a37f7603c..f077c2c5b 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -579,7 +579,8 @@ class MemoryUpstreamRepo implements UpstreamRepo { ? { ...upstream, createdAt: existing.createdAt, modelsCache: existing.modelsCache } : { ...upstream, modelsCache: null }; this.store.set(preserved.id, cloneUpstreamRecord(preserved)); - this.modelsRefreshes.delete(preserved.id); + const refresh = this.modelsRefreshes.get(preserved.id); + if (refresh) this.modelsRefreshes.set(preserved.id, { ...refresh, claimToken: null, claimedAt: null }); return Promise.resolve(); } @@ -634,13 +635,15 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(true); } - claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise { + claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean, observedActiveToken: string | null): Promise { const stored = this.store.get(id); if (!stored || stored.updatedAt !== generation.updatedAt || serializeStoredConfig(stored.config) !== serializeStoredConfig(generation.config)) return Promise.resolve({ kind: 'generation-mismatch' }); const existing = this.modelsRefreshes.get(id); + if (!force && observedActiveToken !== null && existing === undefined) return Promise.resolve({ kind: 'completed' }); if (!force && existing !== undefined) { - if (existing.claimToken !== null && existing.claimedAt! > staleClaimedBefore) return Promise.resolve({ kind: 'active' }); + if (existing.claimToken !== null && existing.claimedAt! > staleClaimedBefore) return Promise.resolve({ kind: 'active', token: existing.claimToken }); if (existing.retryAt > now) return Promise.resolve({ kind: 'backoff' }); + if (observedActiveToken !== null && existing.claimToken === null) return Promise.resolve({ kind: 'completed' }); } this.modelsRefreshes.set(id, { failCount: existing?.failCount ?? 0, diff --git a/packages/gateway/__tests__/repo/models-cache-fixture.ts b/packages/gateway/__tests__/repo/models-cache-fixture.ts index 282d5e697..707feed72 100644 --- a/packages/gateway/__tests__/repo/models-cache-fixture.ts +++ b/packages/gateway/__tests__/repo/models-cache-fixture.ts @@ -8,7 +8,7 @@ export const seedModelsCache = async ( cache: Omit, ): Promise => { const token = crypto.randomUUID(); - const claim = await repo.claimModelsRefresh(id, generation, token, Date.now(), Number.MIN_SAFE_INTEGER, true); + const claim = await repo.claimModelsRefresh(id, generation, token, Date.now(), Number.MIN_SAFE_INTEGER, true, null); if (claim.kind !== 'claimed') return false; const saved = await repo.saveClaimedModelsCache(id, generation, token, cache); await repo.completeModelsRefreshSuccess(id, token); @@ -22,7 +22,7 @@ export const seedModelsCacheError = async ( error: NonNullable, ): Promise => { const token = crypto.randomUUID(); - const claim = await repo.claimModelsRefresh(id, generation, token, Date.now(), Number.MIN_SAFE_INTEGER, true); + const claim = await repo.claimModelsRefresh(id, generation, token, Date.now(), Number.MIN_SAFE_INTEGER, true, null); if (claim.kind !== 'claimed') return false; const saved = await repo.saveClaimedModelsCacheError(id, generation, token, error); await repo.completeModelsRefreshSuccess(id, token); diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index 92f51055d..0ed60481e 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -38,9 +38,9 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.save(record); let now = 1_800_000_000_000; - const first = await repo.upstreams.claimModelsRefresh(record.id, generation, 'claim-0', now, now - 900_000, false); + const first = await repo.upstreams.claimModelsRefresh(record.id, generation, 'claim-0', now, now - 900_000, false, null); expect(first).toEqual({ kind: 'claimed', failureCount: 0 }); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now, now - 900_000, false)).resolves.toEqual({ kind: 'active' }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now, now - 900_000, false, null)).resolves.toEqual({ kind: 'active', token: 'claim-0' }); const delays = [1, 2, 4, 8, 16, 32, 60, 60].map(minutes => minutes * 60_000); if (first.kind !== 'claimed') throw new Error('expected refresh claim'); @@ -49,9 +49,9 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = const retryAt = modelsRefreshRetryAt(now, claim.failureCount); expect(retryAt - now).toBe(delay); await repo.upstreams.completeModelsRefreshFailure(record.id, `claim-${index}`, claim.failureCount + 1, retryAt); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, `early-${index}`, retryAt - 1, retryAt - 900_001, false)).resolves.toEqual({ kind: 'backoff' }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, `early-${index}`, retryAt - 1, retryAt - 900_001, false, null)).resolves.toEqual({ kind: 'backoff' }); now = retryAt; - const nextClaim = await repo.upstreams.claimModelsRefresh(record.id, generation, `claim-${index + 1}`, now, now - 900_000, false); + const nextClaim = await repo.upstreams.claimModelsRefresh(record.id, generation, `claim-${index + 1}`, now, now - 900_000, false, null); if (nextClaim.kind !== 'claimed') throw new Error('expected refresh claim'); claim = nextClaim; expect(claim.failureCount).toBe(index + 1); @@ -59,9 +59,9 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = const blockedUntil = modelsRefreshRetryAt(now, claim.failureCount); await repo.upstreams.completeModelsRefreshFailure(record.id, `claim-${delays.length}`, claim.failureCount + 1, blockedUntil); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'forced', now + 1, now - 899_999, true)).resolves.toEqual({ kind: 'claimed', failureCount: delays.length + 1 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'forced', now + 1, now - 899_999, true, null)).resolves.toEqual({ kind: 'claimed', failureCount: delays.length + 1 }); await repo.upstreams.completeModelsRefreshSuccess(record.id, 'forced'); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'after-success', now + 2, now - 899_998, false)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'after-success', now + 2, now - 899_998, false, null)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); }); test('recovers abandoned claims and fences tokens, timestamps, and config', async () => { @@ -69,26 +69,26 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.save(record); const now = 1_800_000_000_000; - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'abandoned', now, now - 900_000, false)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'replacement', now + 900_001, now + 1, false)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'abandoned', now, now - 900_000, false, null)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'replacement', now + 900_001, now + 1, false, null)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); await repo.upstreams.completeModelsRefreshSuccess(record.id, 'abandoned'); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now + 900_002, now + 2, false)).resolves.toEqual({ kind: 'active' }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now + 900_002, now + 2, false, null)).resolves.toEqual({ kind: 'active', token: 'replacement' }); const next = { ...record, config: { tenant: 'next' } }; await repo.upstreams.saveClearingModelsCache(next); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'old-config', now + 900_003, now + 3, false)).resolves.toEqual({ kind: 'generation-mismatch' }); - await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'current', now + 900_003, now + 3, false)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'old-config', now + 900_003, now + 3, false, null)).resolves.toEqual({ kind: 'generation-mismatch' }); + await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'current', now + 900_003, now + 3, false, null)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); const newer = { ...next, updatedAt: '2026-08-01T00:01:00.000Z' }; await repo.upstreams.saveClearingModelsCache(newer); - await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'old-time', now + 900_004, now + 4, false)).resolves.toEqual({ kind: 'generation-mismatch' }); + await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'old-time', now + 900_004, now + 4, false, null)).resolves.toEqual({ kind: 'generation-mismatch' }); }); - test('saving a new upstream generation clears its predecessor refresh state', async () => { + test('metadata saves preserve backoff while invalidating an active owner', async () => { const repo = await createRepo(); await repo.upstreams.save(record); const now = 1_800_000_000_000; - const claim = await repo.upstreams.claimModelsRefresh(record.id, generation, 'failed', now, now - 900_000, false); + const claim = await repo.upstreams.claimModelsRefresh(record.id, generation, 'failed', now, now - 900_000, false, null); if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); await repo.upstreams.completeModelsRefreshFailure(record.id, 'failed', 1, modelsRefreshRetryAt(now, 0)); @@ -101,6 +101,7 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = now + 1, now - 899_999, false, - )).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + null, + )).resolves.toEqual({ kind: 'backoff' }); }); }); diff --git a/packages/gateway/src/control-plane/upstreams/routes.ts b/packages/gateway/src/control-plane/upstreams/routes.ts index a9564e437..50b10dbe8 100644 --- a/packages/gateway/src/control-plane/upstreams/routes.ts +++ b/packages/gateway/src/control-plane/upstreams/routes.ts @@ -8,6 +8,7 @@ import { type AuthedContext } from '../../middleware/auth.ts'; import { type CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { isDirectFallbackId, normalizeProxyFallbackList } from '../../repo/proxy-fallback-list.ts'; +import { serializeStoredConfig } from '../../repo/upstream-json.ts'; import { shortId } from '../../shared/short-id.ts'; import type { createUpstreamBody, updateUpstreamBody } from '../schemas.ts'; import { isRecord } from '../shared/field-validators.ts'; @@ -335,7 +336,10 @@ export const updateUpstream = async (c: CtxWithJson>(); +type RefreshMode = 'fetch' | 'warm' | 'trigger'; + +interface InFlightRefresh { + kind: 'fetch' | 'wait'; + promise: Promise; +} + +const inFlight = new Map(); + +const startInFlight = ( + key: string, + kind: InFlightRefresh['kind'], + fn: () => Promise, +): Promise => { + const entry: InFlightRefresh = { kind, promise: fn() }; + inFlight.set(key, entry); + entry.promise.finally(() => { + if (inFlight.get(key) === entry) inFlight.delete(key); + }).catch(() => {}); + return entry.promise; +}; const memoInFlight = ( key: string, + kind: InFlightRefresh['kind'], fn: () => Promise, ): Promise => { const existing = inFlight.get(key); - if (existing) return existing; - const promise = fn(); - inFlight.set(key, promise); - promise.finally(() => { - if (inFlight.get(key) === promise) inFlight.delete(key); - }).catch(() => {}); - return promise; + return existing?.promise ?? startInFlight(key, kind, fn); }; const errorMessage = (err: unknown): string => err instanceof Error ? err.message : String(err); @@ -83,14 +98,12 @@ const runFetch = async ( const runClaimedFetch = async ( instance: GatewayProvider, fetcher: Fetcher, - force: boolean, - waitForActive: boolean, + mode: RefreshMode, loadProvidedModels?: () => Promise, ): Promise => { const repo = getRepo(); const token = crypto.randomUUID(); - const initialFetchedAt = instance.modelsCache?.fetchedAt ?? null; - const initialErrorAt = instance.modelsCache?.lastError?.at ?? null; + let observedActiveToken: string | null = null; let claimed: Extract>, { kind: 'claimed' }>; while (true) { const now = Date.now(); @@ -100,27 +113,28 @@ const runClaimedFetch = async ( token, now, now - MODELS_REFRESH_CLAIM_LEASE_MS, - force, + mode === 'fetch', + observedActiveToken, ); if (outcome.kind === 'claimed') { claimed = outcome; break; } - if (outcome.kind !== 'active' || !waitForActive) return null; + if (mode !== 'warm' || outcome.kind === 'backoff' || outcome.kind === 'generation-mismatch') return null; + if (outcome.kind === 'completed') { + const current = await repo.upstreams.getById(instance.upstreamId); + if (current !== null + && current.updatedAt === instance.modelsCacheGeneration.updatedAt + && serializeStoredConfig(current.config) === serializeStoredConfig(instance.modelsCacheGeneration.config)) instance.modelsCache = current.modelsCache; + return null; + } + observedActiveToken = outcome.token; await new Promise(resolve => setTimeout(resolve, ACTIVE_REFRESH_POLL_MS)); - const current = await repo.upstreams.getById(instance.upstreamId); - if (current === null - || current.updatedAt !== instance.modelsCacheGeneration.updatedAt - || serializeStoredConfig(current.config) !== serializeStoredConfig(instance.modelsCacheGeneration.config)) return null; - instance.modelsCache = current.modelsCache; - if ((current.modelsCache?.fetchedAt ?? null) !== initialFetchedAt - || (current.modelsCache?.lastError?.at ?? null) !== initialErrorAt) return null; } + let models: ProviderModel[]; try { - const models = await runFetch(instance, fetcher, instance.upstreamId, token, loadProvidedModels); - await repo.upstreams.completeModelsRefreshSuccess(instance.upstreamId, token); - return models; + models = await runFetch(instance, fetcher, instance.upstreamId, token, loadProvidedModels); } catch (error) { try { const failureCount = claimed.failureCount + 1; @@ -131,6 +145,8 @@ const runClaimedFetch = async ( } throw error; } + await repo.upstreams.completeModelsRefreshSuccess(instance.upstreamId, token); + return models; }; const inFlightKey = (instance: GatewayProvider): string => { @@ -145,13 +161,13 @@ export const fetchUpstreamModels = async ( ): Promise => { const key = inFlightKey(instance); const existing = inFlight.get(key); - if (existing) { - const joined = await existing; + if (existing?.kind === 'fetch') { + const joined = await existing.promise; if (joined !== null) return joined; if (inFlight.get(key) === existing) inFlight.delete(key); } - const models = await memoInFlight(key, () => runClaimedFetch(instance, fetcher, true, false, loadProvidedModels)); + const models = await startInFlight(key, 'fetch', () => runClaimedFetch(instance, fetcher, 'fetch', loadProvidedModels)); if (models === null) throw new Error(`Failed to force-claim models refresh for ${instance.upstreamId}`); return models; }; @@ -164,12 +180,12 @@ export const warmUpstreamModels = async ( const key = inFlightKey(instance); const existing = inFlight.get(key); if (existing) { - const joined = await existing; + const joined = await existing.promise; if (joined !== null) return joined; if (inFlight.get(key) === existing) inFlight.delete(key); } - const models = await memoInFlight(key, () => runClaimedFetch(instance, fetcher, false, true, loadProvidedModels)); + const models = await memoInFlight(key, 'wait', () => runClaimedFetch(instance, fetcher, 'warm', loadProvidedModels)); return models ?? instance.modelsCache?.models ?? []; }; @@ -180,7 +196,7 @@ export const triggerUpstreamModelsFetch = ( loadProvidedModels?: () => Promise, ): void => { const key = inFlightKey(instance); - scheduler(memoInFlight(key, () => runClaimedFetch(instance, fetcher, false, false, loadProvidedModels)).then(() => {})); + scheduler(memoInFlight(key, 'fetch', () => runClaimedFetch(instance, fetcher, 'trigger', loadProvidedModels)).then(() => {})); }; export const fetchUpstreamModelsCached = async ( diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index b09c11bef..f2a7f2386 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -900,6 +900,9 @@ class SqlUpstreamRepo implements UpstreamRepo { } private async saveRecord(upstream: UpstreamRecord, clearModelsCache: boolean): Promise { + const modelsRefreshUpdate = clearModelsCache + ? 'NULL' + : "CASE WHEN models_refresh_json IS NULL THEN NULL ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) END"; // created_at is deliberately not in the ON CONFLICT update list: the row's first INSERT // wins, and re-saves preserve that timestamp regardless of what the caller passes. await this.db @@ -918,7 +921,7 @@ class SqlUpstreamRepo implements UpstreamRepo { proxy_fallback_list_json = excluded.proxy_fallback_list_json, model_prefix_json = excluded.model_prefix_json, hue = excluded.hue, - models_refresh_json = NULL${clearModelsCache ? ', models_cache_json = NULL' : ''}`, + models_refresh_json = ${modelsRefreshUpdate}${clearModelsCache ? ', models_cache_json = NULL' : ''}`, ) .bind( upstream.id, @@ -971,10 +974,10 @@ class SqlUpstreamRepo implements UpstreamRepo { return (result.meta.changes ?? 0) > 0; } - async claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise { + async claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean, observedActiveToken: string | null): Promise { const rawConfig = await this.modelsCacheWriteConfig(id, generation); if (rawConfig === null) return { kind: 'generation-mismatch' }; - for (let attempt = 0; attempt < 3; attempt += 1) { + while (true) { const row = await this.db .prepare( `UPDATE upstreams @@ -985,19 +988,26 @@ class SqlUpstreamRepo implements UpstreamRepo { 'claimedAt', ? ) WHERE id = ? AND updated_at = ? AND config_json = ? AND ( - ? = 1 - OR models_refresh_json IS NULL - OR ( - coalesce(json_extract(models_refresh_json, '$.retryAt'), 0) <= ? - AND ( - json_extract(models_refresh_json, '$.claimToken') IS NULL - OR json_extract(models_refresh_json, '$.claimedAt') <= ? + ? = 1 OR ( + ? IS NULL AND ( + models_refresh_json IS NULL + OR ( + coalesce(json_extract(models_refresh_json, '$.retryAt'), 0) <= ? + AND ( + json_extract(models_refresh_json, '$.claimToken') IS NULL + OR json_extract(models_refresh_json, '$.claimedAt') <= ? + ) + ) ) + ) OR ( + ? IS NOT NULL + AND json_extract(models_refresh_json, '$.claimToken') = ? + AND json_extract(models_refresh_json, '$.claimedAt') <= ? ) ) RETURNING json_extract(models_refresh_json, '$.failCount') AS fail_count`, ) - .bind(token, now, id, generation.updatedAt, rawConfig, sqliteBoolean(force), now, staleClaimedBefore) + .bind(token, now, id, generation.updatedAt, rawConfig, sqliteBoolean(force), observedActiveToken, now, staleClaimedBefore, observedActiveToken, observedActiveToken, staleClaimedBefore) .first<{ fail_count: number }>(); if (row !== null) return { kind: 'claimed', failureCount: row.fail_count }; @@ -1012,11 +1022,14 @@ class SqlUpstreamRepo implements UpstreamRepo { .bind(id, generation.updatedAt, rawConfig) .first<{ models_refresh_json: string | null; retry_at: number | null; claim_token: string | null; claimed_at: number | null }>(); if (state === null) return { kind: 'generation-mismatch' }; - if (state.models_refresh_json === null) continue; - if (state.claim_token !== null && state.claimed_at !== null && state.claimed_at > staleClaimedBefore) return { kind: 'active' }; + if (state.models_refresh_json === null) { + if (observedActiveToken !== null) return { kind: 'completed' }; + continue; + } + if (state.claim_token !== null && state.claimed_at !== null && state.claimed_at > staleClaimedBefore) return { kind: 'active', token: state.claim_token }; if (state.retry_at !== null && state.retry_at > now) return { kind: 'backoff' }; + if (observedActiveToken !== null) return { kind: 'completed' }; } - throw new Error(`Failed to classify models refresh claim contention for ${id}`); } async completeModelsRefreshSuccess(id: string, token: string): Promise { diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index b958c534a..d1f477272 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -267,7 +267,7 @@ export interface UpstreamRepo { // cannot publish models or errors under newer credentials/configuration. saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise; saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise; - claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean): Promise; + claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean, observedActiveToken: string | null): Promise; completeModelsRefreshSuccess(id: string, token: string): Promise; completeModelsRefreshFailure(id: string, token: string, failureCount: number, retryAt: number): Promise; } @@ -278,8 +278,9 @@ export interface ModelsRefreshClaim { } export type ModelsRefreshClaimResult = ModelsRefreshClaim - | { kind: 'active' } + | { kind: 'active'; token: string } | { kind: 'backoff' } + | { kind: 'completed' } | { kind: 'generation-mismatch' }; export interface ModelsCacheGeneration { From 86c8bb8b6e63bf086bcc8fa8e306940071082a26 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 05:06:15 +0800 Subject: [PATCH 30/46] fix(gateway): finalize model refreshes atomically Publish a successful catalog and release its claim in one owner-fenced write; persist a failed error snapshot and backoff in one corresponding write. Use named claim inputs, durable active tokens, and unbounded forward-progress classification so runtime loss cannot split refresh outcome from coordination state. --- .../__tests__/node-sqlite-repo_test.ts | 5 +- .../data-plane/providers/models-cache_test.ts | 19 +++-- packages/gateway/__tests__/repo/memory.ts | 26 ++----- .../__tests__/repo/models-cache-fixture.ts | 12 ++-- .../__tests__/repo/models-refresh_test.ts | 55 +++++++-------- .../src/data-plane/providers/models-cache.ts | 69 +++++++++---------- packages/gateway/src/repo/sql.ts | 42 ++++------- packages/gateway/src/repo/types.ts | 18 +++-- 8 files changed, 109 insertions(+), 137 deletions(-) diff --git a/apps/platform-node/__tests__/node-sqlite-repo_test.ts b/apps/platform-node/__tests__/node-sqlite-repo_test.ts index 4eac84374..cf92c2802 100644 --- a/apps/platform-node/__tests__/node-sqlite-repo_test.ts +++ b/apps/platform-node/__tests__/node-sqlite-repo_test.ts @@ -99,14 +99,13 @@ test('repository JSON codecs round-trip upstream, alias, and Responses state thr config: { opaque: { value: true } }, }; const cacheToken = 'node-cache-fixture'; - const cacheClaim = await repo.upstreams.claimModelsRefresh('up_node', cacheGeneration, cacheToken, Date.now(), Number.MIN_SAFE_INTEGER, true, null); + const cacheClaim = await repo.upstreams.claimModelsRefresh({ id: 'up_node', generation: cacheGeneration, token: cacheToken, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, force: true, observedActiveToken: null }); if (cacheClaim.kind !== 'claimed') throw new Error('expected model-cache fixture claim'); - await repo.upstreams.saveClaimedModelsCache('up_node', cacheGeneration, cacheToken, { + await repo.upstreams.finalizeModelsRefreshSuccess('up_node', cacheGeneration, cacheToken, { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_786_000_000_000, models: [stubProviderModel({ id: 'node-model', enabledFlags: new Set(['vendor-kimi'] as const) })], }); - await repo.upstreams.completeModelsRefreshSuccess('up_node', cacheToken); await repo.modelAliases.insert({ id: 'alias_node', name: 'node-alias', diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index deb9d8d62..661b02ea0 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -248,7 +248,7 @@ describe('fetchUpstreamModelsCached', () => { test('synchronous warm waits for a refresh owned by another runtime', async () => { const repo = await setupRepo(); const now = Date.now(); - await expect(repo.upstreams.claimModelsRefresh(UPSTREAM_ID, CACHE_GENERATION, 'remote-owner', now, now - 900_000, false, null)) + await expect(repo.upstreams.claimModelsRefresh({ id: UPSTREAM_ID, generation: CACHE_GENERATION, token: 'remote-owner', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null })) .resolves.toEqual({ kind: 'claimed', failureCount: 0 }); const localFetch = vi.fn(async () => [aModel('duplicate-local-model')]); const warming = warmUpstreamModels(stubInstance(localFetch), directFetcher); @@ -258,12 +258,11 @@ describe('fetchUpstreamModelsCached', () => { await new Promise(resolve => setTimeout(resolve, 20)); expect(settled).toBe(false); - await repo.upstreams.saveClaimedModelsCache(UPSTREAM_ID, CACHE_GENERATION, 'remote-owner', { + await repo.upstreams.finalizeModelsRefreshSuccess(UPSTREAM_ID, CACHE_GENERATION, 'remote-owner', { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [aModel('remote-model')], }); - await repo.upstreams.completeModelsRefreshSuccess(UPSTREAM_ID, 'remote-owner'); expect((await warming).map(model => model.id)).toEqual(['remote-model']); expect(localFetch).not.toHaveBeenCalled(); @@ -272,7 +271,7 @@ describe('fetchUpstreamModelsCached', () => { test('explicit force bypasses a local warm waiting on another runtime', async () => { const repo = await setupRepo(); const now = Date.now(); - await repo.upstreams.claimModelsRefresh(UPSTREAM_ID, CACHE_GENERATION, 'remote-owner', now, now - 900_000, false, null); + await repo.upstreams.claimModelsRefresh({ id: UPSTREAM_ID, generation: CACHE_GENERATION, token: 'remote-owner', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null }); const fetchFn = vi.fn(async () => [aModel('forced-model')]); const instance = stubInstance(fetchFn, null, CACHE_GENERATION, 'shared-warm-force-key'); const warming = warmUpstreamModels(instance, directFetcher); @@ -284,16 +283,16 @@ describe('fetchUpstreamModelsCached', () => { expect((await warming).map(model => model.id)).toEqual(['forced-model']); }); - test('a success-claim release failure does not install upstream failure backoff', async () => { + test('an atomic success-finalize failure does not install upstream failure backoff', async () => { const repo = await setupRepo(); - const completeFailure = vi.spyOn(repo.upstreams, 'completeModelsRefreshFailure'); - vi.spyOn(repo.upstreams, 'completeModelsRefreshSuccess').mockRejectedValueOnce(new Error('release failed')); + const finalizeFailure = vi.spyOn(repo.upstreams, 'finalizeModelsRefreshFailure'); + vi.spyOn(repo.upstreams, 'finalizeModelsRefreshSuccess').mockRejectedValueOnce(new Error('finalize failed')); const instance = stubInstance(async () => [aModel('published-model')]); await expect(fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher, force: true })) - .rejects.toThrow('release failed'); - expect(completeFailure).not.toHaveBeenCalled(); - expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['published-model']); + .rejects.toThrow('finalize failed'); + expect(finalizeFailure).not.toHaveBeenCalled(); + expect(await storedCache(repo)).toBeNull(); }); test('a superseded generation neither joins nor overwrites the current catalog', async () => { diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index f077c2c5b..2ae974e5e 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -26,6 +26,7 @@ import type { AgentSetupRepository, BackoffRow, ModelsCacheGeneration, + ModelsRefreshClaimInput, ModelsRefreshClaimResult, ModelAliasesRepo, ModelAliasRecord, @@ -618,24 +619,27 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(); } - saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { + finalizeModelsRefreshSuccess(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); const existing = this.store.get(id); if (!existing || existing.updatedAt !== generation.updatedAt || serializeStoredConfig(existing.config) !== serializeStoredConfig(generation.config)) return Promise.resolve(false); existing.modelsCache = { revision: cache.revision, fetchedAt: cache.fetchedAt, models: [...cache.models], lastError: null }; + this.modelsRefreshes.delete(id); return Promise.resolve(true); } - saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise { + finalizeModelsRefreshFailure(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable, failureCount: number, retryAt: number): Promise { if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); const existing = this.store.get(id); if (!existing || existing.updatedAt !== generation.updatedAt || serializeStoredConfig(existing.config) !== serializeStoredConfig(generation.config)) return Promise.resolve(false); if (existing.modelsCache) existing.modelsCache.lastError = error; else existing.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }; + this.modelsRefreshes.set(id, { failCount: failureCount, retryAt, claimToken: null, claimedAt: null }); return Promise.resolve(true); } - claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean, observedActiveToken: string | null): Promise { + claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { + const { id, generation, token, now, staleClaimedBefore, force, observedActiveToken } = input; const stored = this.store.get(id); if (!stored || stored.updatedAt !== generation.updatedAt || serializeStoredConfig(stored.config) !== serializeStoredConfig(generation.config)) return Promise.resolve({ kind: 'generation-mismatch' }); const existing = this.modelsRefreshes.get(id); @@ -654,22 +658,6 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve({ kind: 'claimed', failureCount: existing?.failCount ?? 0 }); } - completeModelsRefreshSuccess(id: string, token: string): Promise { - if (this.modelsRefreshes.get(id)?.claimToken === token) this.modelsRefreshes.delete(id); - return Promise.resolve(); - } - - completeModelsRefreshFailure(id: string, token: string, failureCount: number, retryAt: number): Promise { - const existing = this.modelsRefreshes.get(id); - if (existing?.claimToken !== token) return Promise.resolve(); - this.modelsRefreshes.set(id, { - failCount: failureCount, - retryAt, - claimToken: null, - claimedAt: null, - }); - return Promise.resolve(); - } } const cloneUpstreamRecord = (upstream: UpstreamRecord): UpstreamRecord => ({ diff --git a/packages/gateway/__tests__/repo/models-cache-fixture.ts b/packages/gateway/__tests__/repo/models-cache-fixture.ts index 707feed72..45506b708 100644 --- a/packages/gateway/__tests__/repo/models-cache-fixture.ts +++ b/packages/gateway/__tests__/repo/models-cache-fixture.ts @@ -8,11 +8,9 @@ export const seedModelsCache = async ( cache: Omit, ): Promise => { const token = crypto.randomUUID(); - const claim = await repo.claimModelsRefresh(id, generation, token, Date.now(), Number.MIN_SAFE_INTEGER, true, null); + const claim = await repo.claimModelsRefresh({ id, generation, token, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, force: true, observedActiveToken: null }); if (claim.kind !== 'claimed') return false; - const saved = await repo.saveClaimedModelsCache(id, generation, token, cache); - await repo.completeModelsRefreshSuccess(id, token); - return saved; + return await repo.finalizeModelsRefreshSuccess(id, generation, token, cache); }; export const seedModelsCacheError = async ( @@ -22,9 +20,7 @@ export const seedModelsCacheError = async ( error: NonNullable, ): Promise => { const token = crypto.randomUUID(); - const claim = await repo.claimModelsRefresh(id, generation, token, Date.now(), Number.MIN_SAFE_INTEGER, true, null); + const claim = await repo.claimModelsRefresh({ id, generation, token, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, force: true, observedActiveToken: null }); if (claim.kind !== 'claimed') return false; - const saved = await repo.saveClaimedModelsCacheError(id, generation, token, error); - await repo.completeModelsRefreshSuccess(id, token); - return saved; + return await repo.finalizeModelsRefreshFailure(id, generation, token, error, 0, 0); }; diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index 0ed60481e..456b6c59d 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -3,6 +3,7 @@ import { describe, expect, test } from 'vitest'; import { InMemoryRepo } from './memory.ts'; import { createSqliteTestDb } from './test-sqlite.ts'; import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; +import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import type { ModelsCacheGeneration, Repo } from '../../src/repo/types.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; @@ -38,9 +39,9 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.save(record); let now = 1_800_000_000_000; - const first = await repo.upstreams.claimModelsRefresh(record.id, generation, 'claim-0', now, now - 900_000, false, null); + const first = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'claim-0', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null }); expect(first).toEqual({ kind: 'claimed', failureCount: 0 }); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now, now - 900_000, false, null)).resolves.toEqual({ kind: 'active', token: 'claim-0' }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'racer', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'claim-0' }); const delays = [1, 2, 4, 8, 16, 32, 60, 60].map(minutes => minutes * 60_000); if (first.kind !== 'claimed') throw new Error('expected refresh claim'); @@ -48,20 +49,20 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = for (const [index, delay] of delays.entries()) { const retryAt = modelsRefreshRetryAt(now, claim.failureCount); expect(retryAt - now).toBe(delay); - await repo.upstreams.completeModelsRefreshFailure(record.id, `claim-${index}`, claim.failureCount + 1, retryAt); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, `early-${index}`, retryAt - 1, retryAt - 900_001, false, null)).resolves.toEqual({ kind: 'backoff' }); + await repo.upstreams.finalizeModelsRefreshFailure(record.id, generation, `claim-${index}`, { message: 'failure', at: now }, claim.failureCount + 1, retryAt); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: `early-${index}`, now: retryAt - 1, staleClaimedBefore: retryAt - 900_001, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'backoff' }); now = retryAt; - const nextClaim = await repo.upstreams.claimModelsRefresh(record.id, generation, `claim-${index + 1}`, now, now - 900_000, false, null); + const nextClaim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: `claim-${index + 1}`, now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null }); if (nextClaim.kind !== 'claimed') throw new Error('expected refresh claim'); claim = nextClaim; expect(claim.failureCount).toBe(index + 1); } const blockedUntil = modelsRefreshRetryAt(now, claim.failureCount); - await repo.upstreams.completeModelsRefreshFailure(record.id, `claim-${delays.length}`, claim.failureCount + 1, blockedUntil); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'forced', now + 1, now - 899_999, true, null)).resolves.toEqual({ kind: 'claimed', failureCount: delays.length + 1 }); - await repo.upstreams.completeModelsRefreshSuccess(record.id, 'forced'); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'after-success', now + 2, now - 899_998, false, null)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await repo.upstreams.finalizeModelsRefreshFailure(record.id, generation, `claim-${delays.length}`, { message: 'failure', at: now }, claim.failureCount + 1, blockedUntil); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'forced', now: now + 1, staleClaimedBefore: now - 899_999, force: true, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: delays.length + 1 }); + await repo.upstreams.finalizeModelsRefreshSuccess(record.id, generation, 'forced', { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [] }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'after-success', now: now + 2, staleClaimedBefore: now - 899_998, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); }); test('recovers abandoned claims and fences tokens, timestamps, and config', async () => { @@ -69,39 +70,39 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.save(record); const now = 1_800_000_000_000; - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'abandoned', now, now - 900_000, false, null)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'replacement', now + 900_001, now + 1, false, null)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - await repo.upstreams.completeModelsRefreshSuccess(record.id, 'abandoned'); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'racer', now + 900_002, now + 2, false, null)).resolves.toEqual({ kind: 'active', token: 'replacement' }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'abandoned', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'replacement', now: now + 900_001, staleClaimedBefore: now + 1, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await repo.upstreams.finalizeModelsRefreshSuccess(record.id, generation, 'abandoned', { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 900_001, models: [] }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'racer', now: now + 900_002, staleClaimedBefore: now + 2, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'replacement' }); const next = { ...record, config: { tenant: 'next' } }; await repo.upstreams.saveClearingModelsCache(next); - await expect(repo.upstreams.claimModelsRefresh(record.id, generation, 'old-config', now + 900_003, now + 3, false, null)).resolves.toEqual({ kind: 'generation-mismatch' }); - await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'current', now + 900_003, now + 3, false, null)).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'old-config', now: now + 900_003, staleClaimedBefore: now + 3, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'generation-mismatch' }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: { updatedAt: next.updatedAt, config: next.config }, token: 'current', now: now + 900_003, staleClaimedBefore: now + 3, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); const newer = { ...next, updatedAt: '2026-08-01T00:01:00.000Z' }; await repo.upstreams.saveClearingModelsCache(newer); - await expect(repo.upstreams.claimModelsRefresh(record.id, { updatedAt: next.updatedAt, config: next.config }, 'old-time', now + 900_004, now + 4, false, null)).resolves.toEqual({ kind: 'generation-mismatch' }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: { updatedAt: next.updatedAt, config: next.config }, token: 'old-time', now: now + 900_004, staleClaimedBefore: now + 4, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'generation-mismatch' }); }); test('metadata saves preserve backoff while invalidating an active owner', async () => { const repo = await createRepo(); await repo.upstreams.save(record); const now = 1_800_000_000_000; - const claim = await repo.upstreams.claimModelsRefresh(record.id, generation, 'failed', now, now - 900_000, false, null); + const claim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'failed', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null }); if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); - await repo.upstreams.completeModelsRefreshFailure(record.id, 'failed', 1, modelsRefreshRetryAt(now, 0)); + await repo.upstreams.finalizeModelsRefreshFailure(record.id, generation, 'failed', { message: 'failure', at: now }, 1, modelsRefreshRetryAt(now, 0)); const next = { ...record, name: 'Renamed', updatedAt: '2026-08-01T00:01:00.000Z' }; await repo.upstreams.save(next); - await expect(repo.upstreams.claimModelsRefresh( - record.id, - { updatedAt: next.updatedAt, config: next.config }, - 'next-generation', - now + 1, - now - 899_999, - false, - null, - )).resolves.toEqual({ kind: 'backoff' }); + await expect(repo.upstreams.claimModelsRefresh({ + id: record.id, + generation: { updatedAt: next.updatedAt, config: next.config }, + token: 'next-generation', + now: now + 1, + staleClaimedBefore: now - 899_999, + force: false, + observedActiveToken: null, + })).resolves.toEqual({ kind: 'backoff' }); }); }); diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index 1caefe8ff..f8ba40b85 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -68,32 +68,8 @@ const errorMessage = (err: unknown): string => err instanceof Error ? err.messag const runFetch = async ( instance: GatewayProvider, fetcher: Fetcher, - key: string, - token: string, loadProvidedModels?: () => Promise, -): Promise => { - const generation = instance.modelsCacheGeneration; - try { - const models = [...await (loadProvidedModels?.() ?? instance.instance.getProvidedModels(fetcher))]; - const entry = { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models, lastError: null }; - const persisted = await getRepo().upstreams.saveClaimedModelsCache(key, generation, token, entry); - // The instance carries the row as it was read at request start, and a - // request reaches this function more than once -- once per alias target - // resolved. Writing the entry back keeps every later read in the request - // seeing what was just persisted, which is what re-querying the row used - // to give us. - if (persisted) instance.modelsCache = entry; - return models; - } catch (err) { - const lastError = { message: errorMessage(err), at: Date.now() }; - const persisted = await getRepo().upstreams.saveClaimedModelsCacheError(key, generation, token, lastError); - if (persisted) { - if (instance.modelsCache) instance.modelsCache.lastError = lastError; - else instance.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError }; - } - throw err; - } -}; +): Promise => [...await (loadProvidedModels?.() ?? instance.instance.getProvidedModels(fetcher))]; const runClaimedFetch = async ( instance: GatewayProvider, @@ -107,15 +83,15 @@ const runClaimedFetch = async ( let claimed: Extract>, { kind: 'claimed' }>; while (true) { const now = Date.now(); - const outcome = await repo.upstreams.claimModelsRefresh( - instance.upstreamId, - instance.modelsCacheGeneration, + const outcome = await repo.upstreams.claimModelsRefresh({ + id: instance.upstreamId, + generation: instance.modelsCacheGeneration, token, now, - now - MODELS_REFRESH_CLAIM_LEASE_MS, - mode === 'fetch', + staleClaimedBefore: now - MODELS_REFRESH_CLAIM_LEASE_MS, + force: mode === 'fetch', observedActiveToken, - ); + }); if (outcome.kind === 'claimed') { claimed = outcome; break; @@ -134,18 +110,39 @@ const runClaimedFetch = async ( let models: ProviderModel[]; try { - models = await runFetch(instance, fetcher, instance.upstreamId, token, loadProvidedModels); + models = await runFetch(instance, fetcher, loadProvidedModels); } catch (error) { + const failureCount = claimed.failureCount + 1; + const now = Date.now(); + const lastError = { message: errorMessage(error), at: now }; try { - const failureCount = claimed.failureCount + 1; - const now = Date.now(); - await repo.upstreams.completeModelsRefreshFailure(instance.upstreamId, token, failureCount, modelsRefreshRetryAt(now, claimed.failureCount)); + const finalized = await repo.upstreams.finalizeModelsRefreshFailure( + instance.upstreamId, + instance.modelsCacheGeneration, + token, + lastError, + failureCount, + modelsRefreshRetryAt(now, claimed.failureCount), + ); + if (finalized) { + if (instance.modelsCache) instance.modelsCache.lastError = lastError; + else instance.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError }; + } } catch (backoffError) { throw new AggregateError([error, backoffError], errorMessage(error)); } throw error; } - await repo.upstreams.completeModelsRefreshSuccess(instance.upstreamId, token); + const entry = { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models, lastError: null }; + const finalized = await repo.upstreams.finalizeModelsRefreshSuccess( + instance.upstreamId, + instance.modelsCacheGeneration, + token, + entry, + ); + // The instance is reused across alias targets in one request, so publish the + // finalized snapshot locally as well as durably. + if (finalized) instance.modelsCache = entry; return models; }; diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index f2a7f2386..87ed6d66a 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -19,6 +19,7 @@ import type { AgentSetupRepository, BackoffRow, ModelsCacheGeneration, + ModelsRefreshClaimInput, ModelsRefreshClaimResult, ModelAliasesRepo, ModelAliasRecord, @@ -951,30 +952,36 @@ class SqlUpstreamRepo implements UpstreamRepo { await this.db.prepare('DELETE FROM upstreams').run(); } - async saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { + async finalizeModelsRefreshSuccess(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { const rawConfig = await this.modelsCacheWriteConfig(id, generation); if (rawConfig === null) return false; const result = await this.db - .prepare("UPDATE upstreams SET models_cache_json = ? WHERE id = ? AND updated_at = ? AND config_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") + .prepare("UPDATE upstreams SET models_cache_json = ?, models_refresh_json = NULL WHERE id = ? AND updated_at = ? AND config_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, generation.updatedAt, rawConfig, token) .run(); return (result.meta.changes ?? 0) > 0; } - async saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise { + async finalizeModelsRefreshFailure(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable, failureCount: number, retryAt: number): Promise { const rawConfig = await this.modelsCacheWriteConfig(id, generation); if (rawConfig === null) return false; // A cold failure remains immediately stale while preserving the error for // the next request and dashboard read. const coldFailure = encodeUpstreamModelsCache({ revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }); const result = await this.db - .prepare("UPDATE upstreams SET models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END WHERE id = ? AND updated_at = ? AND config_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") - .bind(coldFailure, JSON.stringify(error), id, generation.updatedAt, rawConfig, token) + .prepare( + `UPDATE upstreams SET + models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END, + models_refresh_json = json_object('failCount', ?, 'retryAt', ?, 'claimToken', NULL, 'claimedAt', NULL) + WHERE id = ? AND updated_at = ? AND config_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?`, + ) + .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, generation.updatedAt, rawConfig, token) .run(); return (result.meta.changes ?? 0) > 0; } - async claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean, observedActiveToken: string | null): Promise { + async claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { + const { id, generation, token, now, staleClaimedBefore, force, observedActiveToken } = input; const rawConfig = await this.modelsCacheWriteConfig(id, generation); if (rawConfig === null) return { kind: 'generation-mismatch' }; while (true) { @@ -1032,29 +1039,6 @@ class SqlUpstreamRepo implements UpstreamRepo { } } - async completeModelsRefreshSuccess(id: string, token: string): Promise { - await this.db - .prepare("UPDATE upstreams SET models_refresh_json = NULL WHERE id = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") - .bind(id, token) - .run(); - } - - async completeModelsRefreshFailure(id: string, token: string, failureCount: number, retryAt: number): Promise { - await this.db - .prepare( - `UPDATE upstreams - SET models_refresh_json = json_object( - 'failCount', ?, - 'retryAt', ?, - 'claimToken', NULL, - 'claimedAt', NULL - ) - WHERE id = ? AND json_extract(models_refresh_json, '$.claimToken') = ?`, - ) - .bind(failureCount, retryAt, id, token) - .run(); - } - private async modelsCacheWriteConfig(id: string, generation: ModelsCacheGeneration): Promise { const row = await this.db .prepare('SELECT updated_at, config_json FROM upstreams WHERE id = ?') diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index d1f477272..68c3204c0 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -265,11 +265,19 @@ export interface UpstreamRepo { // Catalog-cache writes are conditional on the row generation that started // the fetch. A superseded provider can finish serving its own request, but // cannot publish models or errors under newer credentials/configuration. - saveClaimedModelsCache(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise; - saveClaimedModelsCacheError(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable): Promise; - claimModelsRefresh(id: string, generation: ModelsCacheGeneration, token: string, now: number, staleClaimedBefore: number, force: boolean, observedActiveToken: string | null): Promise; - completeModelsRefreshSuccess(id: string, token: string): Promise; - completeModelsRefreshFailure(id: string, token: string, failureCount: number, retryAt: number): Promise; + finalizeModelsRefreshSuccess(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise; + finalizeModelsRefreshFailure(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable, failureCount: number, retryAt: number): Promise; + claimModelsRefresh(input: ModelsRefreshClaimInput): Promise; +} + +export interface ModelsRefreshClaimInput { + id: string; + generation: ModelsCacheGeneration; + token: string; + now: number; + staleClaimedBefore: number; + force: boolean; + observedActiveToken: string | null; } export interface ModelsRefreshClaim { From d311a126fdc72dbad4d98034f5bbc91bd00b1bf7 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 05:06:31 +0800 Subject: [PATCH 31/46] fix(gateway): reset catalogs on fetch identity changes Centralize upstream persistence around the provider model-fetch identity. Metadata-only saves retain cooldown and stale catalogs, while config, state, or proxy changes clear catalog and backoff before synchronous warm across PATCH, OAuth, and import flows. --- .../control-plane/upstreams/routes_test.ts | 4 ++-- .../src/control-plane/data-transfer/routes.ts | 3 ++- .../shared/save-upstream-for-models.ts | 15 +++++++++++++++ .../src/control-plane/upstreams/claude-code.ts | 5 +++-- .../gateway/src/control-plane/upstreams/codex.ts | 3 ++- .../src/control-plane/upstreams/copilot.ts | 4 ++-- .../gateway/src/control-plane/upstreams/routes.ts | 7 ++----- .../gateway/src/data-plane/providers/registry.ts | 10 ++++------ 8 files changed, 32 insertions(+), 19 deletions(-) create mode 100644 packages/gateway/src/control-plane/shared/save-upstream-for-models.ts diff --git a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts index 5e7eac646..9561d1b63 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts @@ -789,9 +789,9 @@ test('PATCH /api/upstreams metadata warm preserves refresh backoff', async () => ); const generation = await getCacheGeneration(repo, created.id); const now = Date.now(); - const claim = await repo.upstreams.claimModelsRefresh(created.id, generation, 'failed-refresh', now, now - 900_000, false, null); + const claim = await repo.upstreams.claimModelsRefresh({ id: created.id, generation, token: 'failed-refresh', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null }); if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); - await repo.upstreams.completeModelsRefreshFailure(created.id, 'failed-refresh', 1, modelsRefreshRetryAt(now, 0)); + await repo.upstreams.finalizeModelsRefreshFailure(created.id, generation, 'failed-refresh', { message: 'failed refresh', at: now }, 1, modelsRefreshRetryAt(now, 0)); let modelRequests = 0; await withMockedFetch( diff --git a/packages/gateway/src/control-plane/data-transfer/routes.ts b/packages/gateway/src/control-plane/data-transfer/routes.ts index b0033de2e..a81e6d8ea 100644 --- a/packages/gateway/src/control-plane/data-transfer/routes.ts +++ b/packages/gateway/src/control-plane/data-transfer/routes.ts @@ -18,6 +18,7 @@ import { DIRECT_FALLBACK_IDS } from '../../repo/proxy-fallback-list.ts'; import type { ApiKey, PerformanceTelemetryRecord, UsageRecord, User, WebSearchUsageRecord } from '../../repo/types.ts'; import { type exportQuery, type importBody } from '../schemas.ts'; import { warmModelsCache } from '../shared/warm-models-cache.ts'; +import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; import { type FullSerializedUpstreamRecord, upstreamRecordToFullJson } from '../upstreams/serialize.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; @@ -189,7 +190,7 @@ export const importData = async (c: CtxWithJson) => { } for (const record of usage) await repo.usage.set(record); for (const record of searchUsage) await repo.webSearchUsage.set(record); - for (const upstream of upstreams) await repo.upstreams.save(upstream); + for (const upstream of upstreams) await saveUpstreamForModels(await repo.upstreams.getById(upstream.id), upstream); await Promise.all(upstreams.map(upstream => warmModelsCache(upstream, c))); for (const record of performance) await repo.performance.set(record); await repo.webSearchConfig.save(searchConfig); diff --git a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts new file mode 100644 index 000000000..1ac5dcc5d --- /dev/null +++ b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts @@ -0,0 +1,15 @@ +import { modelsFetchIdentity } from '../../data-plane/providers/registry.ts'; +import { getRepo } from '../../repo/index.ts'; +import type { UpstreamRecord } from '@floway-dev/provider'; + +export const saveUpstreamForModels = async ( + previous: UpstreamRecord | null, + next: UpstreamRecord, +): Promise => { + const upstreams = getRepo().upstreams; + if (previous !== null && modelsFetchIdentity(previous) === modelsFetchIdentity(next)) { + await upstreams.save(next); + } else { + await upstreams.saveClearingModelsCache(next); + } +}; diff --git a/packages/gateway/src/control-plane/upstreams/claude-code.ts b/packages/gateway/src/control-plane/upstreams/claude-code.ts index 6516e5608..72c6916ae 100644 --- a/packages/gateway/src/control-plane/upstreams/claude-code.ts +++ b/packages/gateway/src/control-plane/upstreams/claude-code.ts @@ -6,6 +6,7 @@ import { getRepo } from '../../repo/index.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { claudeCodeOAuthAuthorizeUrlBody, claudeCodeOAuthExchangeBody, claudeCodeOAuthRefreshBody, claudeCodeProbeBody, claudeCodeSetupTokenAuthorizeUrlBody, claudeCodeSetupTokenExchangeBody } from '../schemas.ts'; import { warmModelsCache } from '../shared/warm-models-cache.ts'; +import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; import type { Fetcher, UpstreamRecord } from '@floway-dev/provider'; import { type ClaudeCodeAccountCredential, @@ -78,7 +79,7 @@ export const claudeCodeOAuthExchange = async (c: CtxWithJson): string => + serializeStoredConfig({ kind: record.kind, config: record.config, state: record.state, proxyFallbackList: record.proxyFallbackList }); + export const createProvider = ( record: UpstreamRecord, cacheGeneration: ModelsCacheGeneration = { updatedAt: record.updatedAt, config: record.config }, @@ -31,12 +34,7 @@ export const createProvider = ( return { ...provider, modelsCacheGeneration: cacheGeneration, - modelsFetchIdentity: serializeStoredConfig({ - kind: record.kind, - config: record.config, - state: record.state, - proxyFallbackList: record.proxyFallbackList, - }), + modelsFetchIdentity: modelsFetchIdentity(record), }; }; From decf03be60a2ca4ea04f5cf0f121b90b7f52d9d1 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 05:09:25 +0800 Subject: [PATCH 32/46] fix(gateway): keep superseded warms blocking When a forced owner steals a warm claim during upstream I/O, follow that durable owner through atomic finalization before returning the synchronous warm response. --- .../data-plane/providers/models-cache_test.ts | 24 +++++++++++++++++++ .../src/data-plane/providers/models-cache.ts | 11 ++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 661b02ea0..7a8a9edb4 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -283,6 +283,30 @@ describe('fetchUpstreamModelsCached', () => { expect((await warming).map(model => model.id)).toEqual(['forced-model']); }); + test('a warm superseded during I/O waits for the forced owner to finalize', async () => { + await setupRepo(); + let resolveWarm: ((models: ProviderModel[]) => void) | null = null; + let resolveForce: ((models: ProviderModel[]) => void) | null = null; + const fetchFn = vi.fn() + .mockImplementationOnce(() => new Promise(resolve => { resolveWarm = resolve; })) + .mockImplementationOnce(() => new Promise(resolve => { resolveForce = resolve; })); + const instance = stubInstance(fetchFn, null, CACHE_GENERATION, 'warm-force-race'); + const warming = warmUpstreamModels(instance, directFetcher); + await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); + const forced = fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher, force: true }); + await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(2)); + + let warmSettled = false; + void warming.finally(() => { warmSettled = true; }); + resolveWarm!([aModel('superseded-warm-model')]); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(warmSettled).toBe(false); + + resolveForce!([aModel('forced-winner-model')]); + expect((await forced).map(model => model.id)).toEqual(['forced-winner-model']); + expect((await warming).map(model => model.id)).toEqual(['forced-winner-model']); + }); + test('an atomic success-finalize failure does not install upstream failure backoff', async () => { const repo = await setupRepo(); const finalizeFailure = vi.spyOn(repo.upstreams, 'finalizeModelsRefreshFailure'); diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index f8ba40b85..a6a7d0539 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -76,10 +76,11 @@ const runClaimedFetch = async ( fetcher: Fetcher, mode: RefreshMode, loadProvidedModels?: () => Promise, + initialObservedActiveToken: string | null = null, ): Promise => { const repo = getRepo(); const token = crypto.randomUUID(); - let observedActiveToken: string | null = null; + let observedActiveToken = initialObservedActiveToken; let claimed: Extract>, { kind: 'claimed' }>; while (true) { const now = Date.now(); @@ -128,6 +129,10 @@ const runClaimedFetch = async ( if (instance.modelsCache) instance.modelsCache.lastError = lastError; else instance.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError }; } + if (!finalized && mode === 'warm') { + const winner = await runClaimedFetch(instance, fetcher, 'warm', loadProvidedModels, token); + return winner ?? instance.modelsCache?.models ?? []; + } } catch (backoffError) { throw new AggregateError([error, backoffError], errorMessage(error)); } @@ -143,6 +148,10 @@ const runClaimedFetch = async ( // The instance is reused across alias targets in one request, so publish the // finalized snapshot locally as well as durably. if (finalized) instance.modelsCache = entry; + else if (mode === 'warm') { + const winner = await runClaimedFetch(instance, fetcher, 'warm', loadProvidedModels, token); + return winner ?? instance.modelsCache?.models ?? []; + } return models; }; From 4f97a8b46d92583759d9b4e4acef05c58cf2824e Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 05:21:40 +0800 Subject: [PATCH 33/46] style(gateway): order refresh coordination imports --- .../control-plane/upstreams/copilot-device-login_test.ts | 2 +- packages/gateway/__tests__/repo/models-refresh_test.ts | 2 +- packages/gateway/__tests__/repo/sql_test.ts | 2 +- packages/gateway/src/control-plane/data-transfer/routes.ts | 2 +- packages/gateway/src/control-plane/upstreams/claude-code.ts | 2 +- packages/gateway/src/control-plane/upstreams/codex.ts | 2 +- packages/gateway/src/control-plane/upstreams/copilot.ts | 2 +- packages/gateway/src/control-plane/upstreams/routes.ts | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts index 89bcd563d..656341185 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts @@ -15,8 +15,8 @@ vi.mock('../../../src/data-plane/providers/models-cache.ts', () => ({ clearInFlightForTesting: () => {}, })); -import { buildCopilotUpstreamRecord, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; +import { buildCopilotUpstreamRecord, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, assertStringIncludes, jsonResponse, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; const githubUser = { diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index 456b6c59d..f04e7fb23 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -2,8 +2,8 @@ import { describe, expect, test } from 'vitest'; import { InMemoryRepo } from './memory.ts'; import { createSqliteTestDb } from './test-sqlite.ts'; -import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; +import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import type { ModelsCacheGeneration, Repo } from '../../src/repo/types.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; diff --git a/packages/gateway/__tests__/repo/sql_test.ts b/packages/gateway/__tests__/repo/sql_test.ts index 796216a03..81cba9397 100644 --- a/packages/gateway/__tests__/repo/sql_test.ts +++ b/packages/gateway/__tests__/repo/sql_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; -import { createSqliteTestDb } from './test-sqlite.ts'; import { seedModelsCache, seedModelsCacheError } from './models-cache-fixture.ts'; +import { createSqliteTestDb } from './test-sqlite.ts'; import { MODEL_CATALOG_REVISION } from '../../src/data-plane/providers/models-cache.ts'; import { SqlRepo, UPSTREAM_STATE_WRITE_ATTEMPTS } from '../../src/repo/sql.ts'; import type { SqlDatabase, SqlPreparedStatement } from '@floway-dev/platform'; diff --git a/packages/gateway/src/control-plane/data-transfer/routes.ts b/packages/gateway/src/control-plane/data-transfer/routes.ts index a81e6d8ea..dbe717b83 100644 --- a/packages/gateway/src/control-plane/data-transfer/routes.ts +++ b/packages/gateway/src/control-plane/data-transfer/routes.ts @@ -17,8 +17,8 @@ import { getRepo } from '../../repo/index.ts'; import { DIRECT_FALLBACK_IDS } from '../../repo/proxy-fallback-list.ts'; import type { ApiKey, PerformanceTelemetryRecord, UsageRecord, User, WebSearchUsageRecord } from '../../repo/types.ts'; import { type exportQuery, type importBody } from '../schemas.ts'; -import { warmModelsCache } from '../shared/warm-models-cache.ts'; import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; +import { warmModelsCache } from '../shared/warm-models-cache.ts'; import { type FullSerializedUpstreamRecord, upstreamRecordToFullJson } from '../upstreams/serialize.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; diff --git a/packages/gateway/src/control-plane/upstreams/claude-code.ts b/packages/gateway/src/control-plane/upstreams/claude-code.ts index 72c6916ae..c62cb0cb7 100644 --- a/packages/gateway/src/control-plane/upstreams/claude-code.ts +++ b/packages/gateway/src/control-plane/upstreams/claude-code.ts @@ -5,8 +5,8 @@ import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { claudeCodeOAuthAuthorizeUrlBody, claudeCodeOAuthExchangeBody, claudeCodeOAuthRefreshBody, claudeCodeProbeBody, claudeCodeSetupTokenAuthorizeUrlBody, claudeCodeSetupTokenExchangeBody } from '../schemas.ts'; -import { warmModelsCache } from '../shared/warm-models-cache.ts'; import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; +import { warmModelsCache } from '../shared/warm-models-cache.ts'; import type { Fetcher, UpstreamRecord } from '@floway-dev/provider'; import { type ClaudeCodeAccountCredential, diff --git a/packages/gateway/src/control-plane/upstreams/codex.ts b/packages/gateway/src/control-plane/upstreams/codex.ts index 23b277662..732444aab 100644 --- a/packages/gateway/src/control-plane/upstreams/codex.ts +++ b/packages/gateway/src/control-plane/upstreams/codex.ts @@ -4,8 +4,8 @@ import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { codexOAuthAuthorizeUrlBody, codexOAuthExchangeBody, codexOAuthRefreshBody } from '../schemas.ts'; -import { warmModelsCache } from '../shared/warm-models-cache.ts'; import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; +import { warmModelsCache } from '../shared/warm-models-cache.ts'; import type { Fetcher, UpstreamRecord } from '@floway-dev/provider'; import { buildCodexAuthorizeUrl, diff --git a/packages/gateway/src/control-plane/upstreams/copilot.ts b/packages/gateway/src/control-plane/upstreams/copilot.ts index 7f39b602c..12917b364 100644 --- a/packages/gateway/src/control-plane/upstreams/copilot.ts +++ b/packages/gateway/src/control-plane/upstreams/copilot.ts @@ -5,8 +5,8 @@ import { getRepo } from '../../repo/index.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { copilotOAuthDeviceLoginPollBody, copilotOAuthDeviceLoginStartBody, copilotQuotaBody } from '../schemas.ts'; import { isRecord } from '../shared/field-validators.ts'; -import { warmModelsCache } from '../shared/warm-models-cache.ts'; import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; +import { warmModelsCache } from '../shared/warm-models-cache.ts'; import type { Fetcher, UpstreamRecord } from '@floway-dev/provider'; import { assertCopilotUpstreamRecord, diff --git a/packages/gateway/src/control-plane/upstreams/routes.ts b/packages/gateway/src/control-plane/upstreams/routes.ts index e5d3c2125..ea7f5aaaf 100644 --- a/packages/gateway/src/control-plane/upstreams/routes.ts +++ b/packages/gateway/src/control-plane/upstreams/routes.ts @@ -11,9 +11,9 @@ import { isDirectFallbackId, normalizeProxyFallbackList } from '../../repo/proxy import { shortId } from '../../shared/short-id.ts'; import type { createUpstreamBody, updateUpstreamBody } from '../schemas.ts'; import { isRecord } from '../shared/field-validators.ts'; +import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; import { nextSortOrder } from '../shared/sort-order.ts'; import { warmModelsCache } from '../shared/warm-models-cache.ts'; -import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; import { normalizeModelPrefix, ALL_PROVIDER_KINDS, From 81ed5be3f9aa17af61dd3741954d8c6542073398 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 05:33:14 +0800 Subject: [PATCH 34/46] fix(gateway): advance model refresh migration Resolve the new main-branch telemetry migration at 0077 by assigning model refresh coordination the next migration number. --- ...stream_models_refresh.sql => 0078_upstream_models_refresh.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/gateway/migrations/{0077_upstream_models_refresh.sql => 0078_upstream_models_refresh.sql} (100%) diff --git a/packages/gateway/migrations/0077_upstream_models_refresh.sql b/packages/gateway/migrations/0078_upstream_models_refresh.sql similarity index 100% rename from packages/gateway/migrations/0077_upstream_models_refresh.sql rename to packages/gateway/migrations/0078_upstream_models_refresh.sql From d6deaa87134eb865fb7eb97bcde7702887c798e5 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 05:37:32 +0800 Subject: [PATCH 35/46] test(gateway): keep disconnect fixture cold Bypass the warm test wrapper so the merged client-disconnect regression continues to prove no cold catalog trigger dispatches after cancellation. --- .../gateway/__tests__/data-plane/providers/resolution_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index 76fa5b234..72c624cfd 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -41,7 +41,7 @@ test('enumerateModelCandidates blocks a cold catalog fetch after client disconne controller.abort(reason); let fetches = 0; - await withMockedFetch( + await withMockedFetchRaw( () => { fetches += 1; return jsonResponse({ object: 'list', data: [] }); From 1d8195e5e0fa2714f2dbc162fcf67bc6d971fda8 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 05:43:08 +0800 Subject: [PATCH 36/46] fix(gateway): detach catalog refresh from client disconnect Keep scheduler-owned model refreshes on the raw upstream fetcher while retaining client-aware response lifetimes only for inference candidates. Cover disconnect after snapshot scheduling but before persisted claim dispatch. --- .../data-plane/providers/resolution_test.ts | 44 ++++++++++++++++++- .../src/data-plane/providers/resolution.ts | 33 +++++++++----- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index 72c624cfd..1cb38b664 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from 'vitest'; +import { describe, expect, test, vi } from 'vitest'; import { clearInFlightForTesting, fetchUpstreamModels } from '../../../src/data-plane/providers/models-cache.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; @@ -66,6 +66,48 @@ test('enumerateModelCandidates blocks a cold catalog fetch after client disconne ); }); +test('a scheduled cold refresh survives disconnect before its claim dispatches', async () => { + const { repo } = await setupAppTest(); + await repo.upstreams.deleteAll(); + await repo.upstreams.save(buildCustomUpstreamRecord()); + const originalClaim = repo.upstreams.claimModelsRefresh.bind(repo.upstreams); + let releaseClaim: (() => void) | null = null; + vi.spyOn(repo.upstreams, 'claimModelsRefresh').mockImplementation(async input => { + await new Promise(resolve => { releaseClaim = resolve; }); + return await originalClaim(input); + }); + const controller = new AbortController(); + const background: Promise[] = []; + let fetches = 0; + + await withMockedFetchRaw( + () => { + fetches++; + return jsonResponse({ object: 'list', data: [{ id: 'eventual-model' }] }); + }, + async () => { + await enumerateModelCandidates({ + upstreamIds: null, + model: 'eventual-model', + kind: 'chat', + scheduler: promise => { background.push(promise); }, + runtimeLocation: 'TEST', + clientDisconnectSignal: controller.signal, + }); + await vi.waitFor(() => expect(releaseClaim).not.toBeNull()); + controller.abort(new Error('client disconnected')); + releaseClaim!(); + await Promise.all(background); + }, + ); + + expect(fetches).toBe(1); + expect((await repo.upstreams.getById('up_custom'))?.modelsCache).toMatchObject({ + lastError: null, + models: [{ id: 'eventual-model' }], + }); +}); + test('enumerateModelCandidates strips an -YYYYMMDD suffix when nothing matched and retries across every visible upstream', async () => { const { repo } = await setupAppTest(); diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index 5019ef1d3..ef1240992 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -29,7 +29,8 @@ const enumerateOneUpstreamCandidates = async ( provider: GatewayProvider, modelId: string, kind: ModelKind, - fetcher: Fetcher, + catalogFetcher: Fetcher, + candidateFetcher: Fetcher, scheduler: BackgroundScheduler, ): Promise<{ candidates: ModelCandidate[]; sawAnyId: boolean; modelsError: boolean }> => { const cfg = provider.modelPrefix; @@ -44,7 +45,7 @@ const enumerateOneUpstreamCandidates = async ( } if (lookupIds.length === 0) return { candidates: [], sawAnyId: false, modelsError: false }; - const providedModels = await fetchUpstreamModelsCached(provider, { scheduler, fetcher }); + const providedModels = await fetchUpstreamModelsCached(provider, { scheduler, fetcher: catalogFetcher }); const disabled = new Set(provider.disabledPublicModelIds); const candidates: ModelCandidate[] = []; let sawAnyId = false; @@ -53,7 +54,7 @@ const enumerateOneUpstreamCandidates = async ( if (!match) continue; sawAnyId = true; if (match.kind === kind) { - candidates.push({ provider, model: internalModelFromProviderModel(match, provider.upstreamId), fetcher }); + candidates.push({ provider, model: internalModelFromProviderModel(match, provider.upstreamId), fetcher: candidateFetcher }); } } return { candidates, sawAnyId, modelsError: provider.modelsCache?.lastError != null }; @@ -61,8 +62,9 @@ const enumerateOneUpstreamCandidates = async ( // Walk every visible upstream in configured order. Snapshot reads never wait // for upstream model-list I/O; cold and stale rows submit background refresh. -// Client disconnect prevents snapshot work that has not dispatched, while a -// retained operation already in flight runs to completion. +// Client disconnect prevents snapshot work that has not dispatched. Once a +// refresh is scheduled, its raw fetcher belongs to the background lifecycle; +// only inference candidates receive the retained client-aware wrapper. // // `sawAnyId` aggregates the per-upstream signal: true when at least one // upstream's catalog carried the inbound id under any kind. The caller @@ -76,6 +78,7 @@ export const enumerateRealModelCandidates = async ( fetcherForUpstream: (upstreamId: string) => Fetcher, scheduler: BackgroundScheduler, clientDisconnectSignal?: AbortSignal, + candidateFetcherForUpstream: (upstreamId: string) => Fetcher = fetcherForUpstream, ): Promise<{ readonly candidates: readonly ModelCandidate[]; readonly sawAnyId: boolean; @@ -83,7 +86,14 @@ export const enumerateRealModelCandidates = async ( }> => { const settled = await Promise.allSettled(providers.map(provider => { clientDisconnectSignal?.throwIfAborted(); - return enumerateOneUpstreamCandidates(provider, modelId, kind, fetcherForUpstream(provider.upstreamId), scheduler); + return enumerateOneUpstreamCandidates( + provider, + modelId, + kind, + fetcherForUpstream(provider.upstreamId), + candidateFetcherForUpstream(provider.upstreamId), + scheduler, + ); })); const failedUpstreams: string[] = []; @@ -123,17 +133,18 @@ const resolveRealCandidates = async ( fetcherForUpstream: (upstreamId: string) => Fetcher, scheduler: BackgroundScheduler, clientDisconnectSignal?: AbortSignal, + candidateFetcherForUpstream: (upstreamId: string) => Fetcher = fetcherForUpstream, ): Promise<{ readonly candidates: readonly ModelCandidate[]; readonly sawModel: boolean; readonly failedUpstreams: readonly string[]; }> => { - const first = await enumerateRealModelCandidates(modelId, kind, providers, fetcherForUpstream, scheduler, clientDisconnectSignal); + const first = await enumerateRealModelCandidates(modelId, kind, providers, fetcherForUpstream, scheduler, clientDisconnectSignal, candidateFetcherForUpstream); if (first.candidates.length > 0 || first.sawAnyId || !DATED_SUFFIX.test(modelId)) { return { candidates: first.candidates, sawModel: first.sawAnyId, failedUpstreams: first.failedUpstreams }; } const stripped = modelId.replace(DATED_SUFFIX, ''); - const second = await enumerateRealModelCandidates(stripped, kind, providers, fetcherForUpstream, scheduler, clientDisconnectSignal); + const second = await enumerateRealModelCandidates(stripped, kind, providers, fetcherForUpstream, scheduler, clientDisconnectSignal, candidateFetcherForUpstream); return { candidates: second.candidates, sawModel: second.sawAnyId, @@ -211,7 +222,7 @@ export const enumerateModelCandidates = async ({ readonly failedUpstreams: readonly string[]; }> => { const createFetcherForUpstream = await createPerRequestFetcher(runtimeLocation); - const fetcherForUpstream = (upstreamId: string): Fetcher => { + const candidateFetcherForUpstream = (upstreamId: string): Fetcher => { const fetcher = createFetcherForUpstream(upstreamId); return clientDisconnectSignal === undefined ? fetcher @@ -221,7 +232,7 @@ export const enumerateModelCandidates = async ({ const alias = await getRepo().modelAliases.getByName(model); if (alias === null) { - return await resolveRealCandidates(model, kind, providers, fetcherForUpstream, scheduler, clientDisconnectSignal); + return await resolveRealCandidates(model, kind, providers, createFetcherForUpstream, scheduler, clientDisconnectSignal, candidateFetcherForUpstream); } // Walk every target, tag each returned candidate with the target's rule @@ -233,7 +244,7 @@ export const enumerateModelCandidates = async ({ let sawAny = false; const flat: ModelCandidate[] = []; for (const target of orderAliasTargets(alias)) { - const result = await resolveRealCandidates(target.target_model_id, kind, providers, fetcherForUpstream, scheduler, clientDisconnectSignal); + const result = await resolveRealCandidates(target.target_model_id, kind, providers, createFetcherForUpstream, scheduler, clientDisconnectSignal, candidateFetcherForUpstream); for (const name of result.failedUpstreams) aggregatedFailed.add(name); if (result.sawModel) sawAny = true; for (const candidate of result.candidates) { From 835606c46bb3154e4120df62cfa2594960af72e9 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 05:48:01 +0800 Subject: [PATCH 37/46] refactor(gateway): name resolution fetch lifecycles Thread catalog and inference fetchers through named resolution context objects so their distinct background and client-disconnect lifecycles cannot be swapped positionally. --- .../data-plane/providers/resolution_test.ts | 6 +-- .../src/data-plane/providers/resolution.ts | 48 ++++++++++++------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index 1cb38b664..5a4af160d 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -270,7 +270,7 @@ test('enumerateRealModelCandidates only loads the selected providers\' catalogs' await fetchUpstreamModels(providers[0], directFetcher); const warmed = (await listModelProviders(null)).find(provider => provider.upstreamId === 'up_first'); if (!warmed) throw new Error('warmed provider missing'); - const { candidates } = await enumerateRealModelCandidates('target-model', 'chat', [warmed], () => directFetcher, testScheduler); + const { candidates } = await enumerateRealModelCandidates('target-model', 'chat', [warmed], { catalogFetcherForUpstream: () => directFetcher, scheduler: testScheduler }); assertEquals(candidates[0]?.model.id, 'target-model'); assertEquals(candidates[0]?.provider.upstreamId, 'up_first'); @@ -313,8 +313,8 @@ test('enumerateRealModelCandidates rejects a model id disabled on that upstream await warmModelsForTest(); const providers = await listModelProviders(null); - const enabled = await enumerateRealModelCandidates('enabled-model', 'chat', providers, () => directFetcher, testScheduler); - const disabled = await enumerateRealModelCandidates('disabled-model', 'chat', providers, () => directFetcher, testScheduler); + const enabled = await enumerateRealModelCandidates('enabled-model', 'chat', providers, { catalogFetcherForUpstream: () => directFetcher, scheduler: testScheduler }); + const disabled = await enumerateRealModelCandidates('disabled-model', 'chat', providers, { catalogFetcherForUpstream: () => directFetcher, scheduler: testScheduler }); assertEquals(enabled.candidates[0]?.model.id, 'enabled-model'); assertEquals(disabled.candidates.length, 0); }); diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index ef1240992..bc0b31e1f 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -29,10 +29,13 @@ const enumerateOneUpstreamCandidates = async ( provider: GatewayProvider, modelId: string, kind: ModelKind, - catalogFetcher: Fetcher, - candidateFetcher: Fetcher, - scheduler: BackgroundScheduler, + context: { + catalogFetcher: Fetcher; + candidateFetcher: Fetcher; + scheduler: BackgroundScheduler; + }, ): Promise<{ candidates: ModelCandidate[]; sawAnyId: boolean; modelsError: boolean }> => { + const { catalogFetcher, candidateFetcher, scheduler } = context; const cfg = provider.modelPrefix; const lookupIds: string[] = []; if (cfg === null) { @@ -75,24 +78,30 @@ export const enumerateRealModelCandidates = async ( modelId: string, kind: ModelKind, providers: readonly GatewayProvider[], - fetcherForUpstream: (upstreamId: string) => Fetcher, - scheduler: BackgroundScheduler, - clientDisconnectSignal?: AbortSignal, - candidateFetcherForUpstream: (upstreamId: string) => Fetcher = fetcherForUpstream, + context: { + catalogFetcherForUpstream: (upstreamId: string) => Fetcher; + candidateFetcherForUpstream?: (upstreamId: string) => Fetcher; + scheduler: BackgroundScheduler; + clientDisconnectSignal?: AbortSignal; + }, ): Promise<{ readonly candidates: readonly ModelCandidate[]; readonly sawAnyId: boolean; readonly failedUpstreams: readonly string[]; }> => { + const { catalogFetcherForUpstream, scheduler, clientDisconnectSignal } = context; + const candidateFetcherForUpstream = context.candidateFetcherForUpstream ?? catalogFetcherForUpstream; const settled = await Promise.allSettled(providers.map(provider => { clientDisconnectSignal?.throwIfAborted(); return enumerateOneUpstreamCandidates( provider, modelId, kind, - fetcherForUpstream(provider.upstreamId), - candidateFetcherForUpstream(provider.upstreamId), - scheduler, + { + catalogFetcher: catalogFetcherForUpstream(provider.upstreamId), + candidateFetcher: candidateFetcherForUpstream(provider.upstreamId), + scheduler, + }, ); })); @@ -130,21 +139,18 @@ const resolveRealCandidates = async ( modelId: string, kind: ModelKind, providers: readonly GatewayProvider[], - fetcherForUpstream: (upstreamId: string) => Fetcher, - scheduler: BackgroundScheduler, - clientDisconnectSignal?: AbortSignal, - candidateFetcherForUpstream: (upstreamId: string) => Fetcher = fetcherForUpstream, + context: Parameters[3], ): Promise<{ readonly candidates: readonly ModelCandidate[]; readonly sawModel: boolean; readonly failedUpstreams: readonly string[]; }> => { - const first = await enumerateRealModelCandidates(modelId, kind, providers, fetcherForUpstream, scheduler, clientDisconnectSignal, candidateFetcherForUpstream); + const first = await enumerateRealModelCandidates(modelId, kind, providers, context); if (first.candidates.length > 0 || first.sawAnyId || !DATED_SUFFIX.test(modelId)) { return { candidates: first.candidates, sawModel: first.sawAnyId, failedUpstreams: first.failedUpstreams }; } const stripped = modelId.replace(DATED_SUFFIX, ''); - const second = await enumerateRealModelCandidates(stripped, kind, providers, fetcherForUpstream, scheduler, clientDisconnectSignal, candidateFetcherForUpstream); + const second = await enumerateRealModelCandidates(stripped, kind, providers, context); return { candidates: second.candidates, sawModel: second.sawAnyId, @@ -229,10 +235,16 @@ export const enumerateModelCandidates = async ({ : retainUpstreamFetcher(fetcher, clientDisconnectSignal, scheduler); }; const providers = await listModelProviders(upstreamIds); + const resolutionContext = { + catalogFetcherForUpstream: createFetcherForUpstream, + candidateFetcherForUpstream, + scheduler, + clientDisconnectSignal, + }; const alias = await getRepo().modelAliases.getByName(model); if (alias === null) { - return await resolveRealCandidates(model, kind, providers, createFetcherForUpstream, scheduler, clientDisconnectSignal, candidateFetcherForUpstream); + return await resolveRealCandidates(model, kind, providers, resolutionContext); } // Walk every target, tag each returned candidate with the target's rule @@ -244,7 +256,7 @@ export const enumerateModelCandidates = async ({ let sawAny = false; const flat: ModelCandidate[] = []; for (const target of orderAliasTargets(alias)) { - const result = await resolveRealCandidates(target.target_model_id, kind, providers, createFetcherForUpstream, scheduler, clientDisconnectSignal, candidateFetcherForUpstream); + const result = await resolveRealCandidates(target.target_model_id, kind, providers, resolutionContext); for (const name of result.failedUpstreams) aggregatedFailed.add(name); if (result.sawModel) sawAny = true; for (const candidate of result.candidates) { From 42517756a4ee8c3e3d4ed7142ed6879c009cd036 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 16:22:15 +0800 Subject: [PATCH 38/46] refactor(gateway): unify resolution fetcher ownership Carry one raw per-upstream fetcher through catalog resolution and model candidates. Apply client-disconnect retention only at inference dispatch boundaries, while scheduled catalog refreshes retain their independent background lifecycle.\n\nLock the contract by asserting that candidate enumeration preserves the supplied fetcher identity. --- .../data-plane/providers/resolution_test.ts | 7 ++-- .../src/data-plane/providers/resolution.ts | 32 ++++++------------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index 5a4af160d..877e3781f 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -270,10 +270,11 @@ test('enumerateRealModelCandidates only loads the selected providers\' catalogs' await fetchUpstreamModels(providers[0], directFetcher); const warmed = (await listModelProviders(null)).find(provider => provider.upstreamId === 'up_first'); if (!warmed) throw new Error('warmed provider missing'); - const { candidates } = await enumerateRealModelCandidates('target-model', 'chat', [warmed], { catalogFetcherForUpstream: () => directFetcher, scheduler: testScheduler }); + const { candidates } = await enumerateRealModelCandidates('target-model', 'chat', [warmed], { fetcherForUpstream: () => directFetcher, scheduler: testScheduler }); assertEquals(candidates[0]?.model.id, 'target-model'); assertEquals(candidates[0]?.provider.upstreamId, 'up_first'); + expect(candidates[0]?.fetcher).toBe(directFetcher); // Every enumerated candidate seeds `providerModels[provider.upstreamId]` // so `providerModelOf(candidate)` resolves at dispatch time. assertEquals(Object.keys(realProviderModels(candidates[0]?.model)), ['up_first']); @@ -313,8 +314,8 @@ test('enumerateRealModelCandidates rejects a model id disabled on that upstream await warmModelsForTest(); const providers = await listModelProviders(null); - const enabled = await enumerateRealModelCandidates('enabled-model', 'chat', providers, { catalogFetcherForUpstream: () => directFetcher, scheduler: testScheduler }); - const disabled = await enumerateRealModelCandidates('disabled-model', 'chat', providers, { catalogFetcherForUpstream: () => directFetcher, scheduler: testScheduler }); + const enabled = await enumerateRealModelCandidates('enabled-model', 'chat', providers, { fetcherForUpstream: () => directFetcher, scheduler: testScheduler }); + const disabled = await enumerateRealModelCandidates('disabled-model', 'chat', providers, { fetcherForUpstream: () => directFetcher, scheduler: testScheduler }); assertEquals(enabled.candidates[0]?.model.id, 'enabled-model'); assertEquals(disabled.candidates.length, 0); }); diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index bc0b31e1f..1990c2280 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -6,7 +6,6 @@ import { listModelProviders, type GatewayProvider } from './registry.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; import { getRepo } from '../../repo/index.ts'; import type { ModelAliasRecord } from '../../repo/types.ts'; -import { retainUpstreamFetcher } from '../shared/retained-response.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ModelKind } from '@floway-dev/protocols/common'; import { isAbortError, type Fetcher, type ModelCandidate } from '@floway-dev/provider'; @@ -30,12 +29,11 @@ const enumerateOneUpstreamCandidates = async ( modelId: string, kind: ModelKind, context: { - catalogFetcher: Fetcher; - candidateFetcher: Fetcher; + fetcher: Fetcher; scheduler: BackgroundScheduler; }, ): Promise<{ candidates: ModelCandidate[]; sawAnyId: boolean; modelsError: boolean }> => { - const { catalogFetcher, candidateFetcher, scheduler } = context; + const { fetcher, scheduler } = context; const cfg = provider.modelPrefix; const lookupIds: string[] = []; if (cfg === null) { @@ -48,7 +46,7 @@ const enumerateOneUpstreamCandidates = async ( } if (lookupIds.length === 0) return { candidates: [], sawAnyId: false, modelsError: false }; - const providedModels = await fetchUpstreamModelsCached(provider, { scheduler, fetcher: catalogFetcher }); + const providedModels = await fetchUpstreamModelsCached(provider, { scheduler, fetcher }); const disabled = new Set(provider.disabledPublicModelIds); const candidates: ModelCandidate[] = []; let sawAnyId = false; @@ -57,7 +55,7 @@ const enumerateOneUpstreamCandidates = async ( if (!match) continue; sawAnyId = true; if (match.kind === kind) { - candidates.push({ provider, model: internalModelFromProviderModel(match, provider.upstreamId), fetcher: candidateFetcher }); + candidates.push({ provider, model: internalModelFromProviderModel(match, provider.upstreamId), fetcher }); } } return { candidates, sawAnyId, modelsError: provider.modelsCache?.lastError != null }; @@ -66,8 +64,8 @@ const enumerateOneUpstreamCandidates = async ( // Walk every visible upstream in configured order. Snapshot reads never wait // for upstream model-list I/O; cold and stale rows submit background refresh. // Client disconnect prevents snapshot work that has not dispatched. Once a -// refresh is scheduled, its raw fetcher belongs to the background lifecycle; -// only inference candidates receive the retained client-aware wrapper. +// refresh is scheduled, the scheduler owns its lifetime. Inference lifecycle +// policy is applied later, where a selected candidate is actually dispatched. // // `sawAnyId` aggregates the per-upstream signal: true when at least one // upstream's catalog carried the inbound id under any kind. The caller @@ -79,8 +77,7 @@ export const enumerateRealModelCandidates = async ( kind: ModelKind, providers: readonly GatewayProvider[], context: { - catalogFetcherForUpstream: (upstreamId: string) => Fetcher; - candidateFetcherForUpstream?: (upstreamId: string) => Fetcher; + fetcherForUpstream: (upstreamId: string) => Fetcher; scheduler: BackgroundScheduler; clientDisconnectSignal?: AbortSignal; }, @@ -89,8 +86,7 @@ export const enumerateRealModelCandidates = async ( readonly sawAnyId: boolean; readonly failedUpstreams: readonly string[]; }> => { - const { catalogFetcherForUpstream, scheduler, clientDisconnectSignal } = context; - const candidateFetcherForUpstream = context.candidateFetcherForUpstream ?? catalogFetcherForUpstream; + const { fetcherForUpstream, scheduler, clientDisconnectSignal } = context; const settled = await Promise.allSettled(providers.map(provider => { clientDisconnectSignal?.throwIfAborted(); return enumerateOneUpstreamCandidates( @@ -98,8 +94,7 @@ export const enumerateRealModelCandidates = async ( modelId, kind, { - catalogFetcher: catalogFetcherForUpstream(provider.upstreamId), - candidateFetcher: candidateFetcherForUpstream(provider.upstreamId), + fetcher: fetcherForUpstream(provider.upstreamId), scheduler, }, ); @@ -228,16 +223,9 @@ export const enumerateModelCandidates = async ({ readonly failedUpstreams: readonly string[]; }> => { const createFetcherForUpstream = await createPerRequestFetcher(runtimeLocation); - const candidateFetcherForUpstream = (upstreamId: string): Fetcher => { - const fetcher = createFetcherForUpstream(upstreamId); - return clientDisconnectSignal === undefined - ? fetcher - : retainUpstreamFetcher(fetcher, clientDisconnectSignal, scheduler); - }; const providers = await listModelProviders(upstreamIds); const resolutionContext = { - catalogFetcherForUpstream: createFetcherForUpstream, - candidateFetcherForUpstream, + fetcherForUpstream: createFetcherForUpstream, scheduler, clientDisconnectSignal, }; From 4b323d23e0abe0e33014b2d05c33ec60862661ce Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 17:29:30 +0800 Subject: [PATCH 39/46] refactor(gateway): make catalog refresh ownership explicit Replace overlapping cached/force/fetch modes with operation-shaped snapshot, explicit fetch, warm, and background scheduling boundaries. Explicit requests now join durable owners and bypass only cooldown, eliminating claim preemption and in-flight replacement races.\n\nSeparate provider-owned catalog identity from refresh-generation and operator-input identities. Fence durable writes against the full static fetch generation, preserve snapshots across safe credential rotation, reset cooldown for changed operator inputs, and apply control-plane replacements through an atomic compare-and-save operation.\n\nConsolidate save, warm, and readback across create, update, OAuth, and imports; reuse one transport catalog for batch imports. Make scheduled ownership and locationless egress explicit, isolate malformed upstreams, and constrain persisted refresh state. --- apps/platform-cloudflare/entry.ts | 2 +- .../data-transfer/routes_test.ts | 2 +- .../upstreams/copilot-device-login_test.ts | 3 +- .../control-plane/upstreams/routes_test.ts | 31 +-- .../chat/shared/target-picker_test.ts | 2 +- .../data-plane/providers/catalog_test.ts | 2 +- .../data-plane/providers/models-cache_test.ts | 210 ++++++++---------- .../data-plane/providers/registry_test.ts | 18 +- packages/gateway/__tests__/repo/memory.ts | 54 +++-- .../__tests__/repo/models-cache-fixture.ts | 8 +- .../__tests__/repo/models-refresh_test.ts | 116 +++++++--- .../repo/proxy-fallback-list_test.ts | 2 + packages/gateway/__tests__/repo/sql_test.ts | 19 +- .../scheduled/models-refresh_test.ts | 51 ++++- packages/gateway/__tests__/scheduled_test.ts | 6 +- .../0078_upstream_models_refresh.sql | 30 ++- .../src/control-plane/data-transfer/routes.ts | 9 +- .../shared/save-upstream-for-models.ts | 64 +++++- .../control-plane/shared/warm-models-cache.ts | 33 --- .../control-plane/upstreams/claude-code.ts | 9 +- .../src/control-plane/upstreams/codex.ts | 6 +- .../src/control-plane/upstreams/copilot.ts | 6 +- .../src/control-plane/upstreams/models.ts | 52 ++--- .../src/control-plane/upstreams/routes.ts | 9 +- .../src/data-plane/providers/catalog.ts | 12 +- .../src/data-plane/providers/models-cache.ts | 194 ++++++++-------- .../src/data-plane/providers/registry.ts | 9 +- .../src/data-plane/providers/resolution.ts | 10 +- .../data-plane/shared/listing/addressable.ts | 4 +- packages/gateway/src/dial/fetcher.ts | 5 +- packages/gateway/src/dial/per-request.ts | 2 +- .../gateway/src/repo/models-cache-contract.ts | 32 +++ .../gateway/src/repo/proxy-fallback-list.ts | 10 +- packages/gateway/src/repo/sql.ts | 169 ++++++++++---- packages/gateway/src/repo/types.ts | 30 ++- packages/gateway/src/scheduled.ts | 12 +- .../gateway/src/scheduled/models-refresh.ts | 17 +- packages/provider-azure/src/index.ts | 2 + packages/provider-claude-code/src/index.ts | 5 + packages/provider-codex/src/index.ts | 5 + .../__tests__/provider_test.ts | 2 +- packages/provider-copilot/src/index.ts | 5 + packages/provider-custom/src/index.ts | 2 + packages/provider-ollama/src/fetch-models.ts | 2 +- packages/provider-ollama/src/index.ts | 2 + packages/provider/src/provider.ts | 8 +- 46 files changed, 797 insertions(+), 486 deletions(-) delete mode 100644 packages/gateway/src/control-plane/shared/warm-models-cache.ts diff --git a/apps/platform-cloudflare/entry.ts b/apps/platform-cloudflare/entry.ts index 00da2c4b6..7ff6960d8 100644 --- a/apps/platform-cloudflare/entry.ts +++ b/apps/platform-cloudflare/entry.ts @@ -25,6 +25,6 @@ export default { scheduled(_controller: unknown, env: CloudflareEnv, ctx: ExecutionContext) { const { db } = bootstrapCloudflarePlatform(env); initRepo(new SqlRepo(db)); - ctx.waitUntil(runScheduledMaintenance('SCHEDULED', promise => ctx.waitUntil(promise))); + ctx.waitUntil(runScheduledMaintenance(null, promise => ctx.waitUntil(promise))); }, }; diff --git a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts index 9b8e36cf7..a36e11fe5 100644 --- a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts @@ -1,7 +1,7 @@ import { Hono } from 'hono'; import { expect, test, vi } from 'vitest'; -// The import handler warms the SWR models cache for every saved upstream by +// The import handler warms the persisted models snapshot for every saved upstream by // calling each provider's getProvidedModels, which for Copilot / Custom would // make real upstream HTTP requests the test sandbox cannot serve and hang // until the vitest timeout. Stub the cache layer to a no-op so the import diff --git a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts index 656341185..0604f55a4 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts @@ -16,6 +16,7 @@ vi.mock('../../../src/data-plane/providers/models-cache.ts', () => ({ })); import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; +import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; import { buildCopilotUpstreamRecord, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, assertStringIncludes, jsonResponse, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; @@ -370,7 +371,7 @@ test('/api/upstreams/copilot/oauth/device-login/poll clears the previous identit const existing = buildCopilotUpstreamRecord(githubAccount, { id: 'up_switch_identity' }); await repo.upstreams.deleteAll(); await repo.upstreams.save(existing); - await seedModelsCache(repo.upstreams, existing.id, { updatedAt: existing.updatedAt, config: existing.config }, { + await seedModelsCache(repo.upstreams, existing.id, modelsCacheGeneration(existing), { revision: 1, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'old-tenant-model' })], diff --git a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts index 9561d1b63..8127ff783 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import { blueprintUpstreamRecord, upstreamRecordToFullJson } from '../../../src/control-plane/upstreams/serialize.ts'; import { MODEL_LISTING_FAILURE_CODE } from '../../../src/data-plane/models/shared.ts'; import { MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; -import { modelsRefreshRetryAt } from '../../../src/repo/models-refresh-contract.ts'; +import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; import { seedModelsCache, seedModelsCacheError } from '../../repo/models-cache-fixture.ts'; import { MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import type { UpstreamProviderKind, UpstreamRecord } from '@floway-dev/provider'; @@ -438,21 +438,24 @@ test('GET /api/upstreams attaches models-cache freshness to every row', async () config: { baseUrl: 'https://a.example.com', authStyle: 'bearer', apiKey: 'x', endpoints: { chatCompletions: {} }, ingressHeadersRules: [] }, state: null, }; - await repo.upstreams.save({ ...baseRow, id: 'up_fresh', name: 'Fresh', sortOrder: 0 }); - await repo.upstreams.save({ ...baseRow, id: 'up_warm', name: 'Warm', sortOrder: 1 }); - await repo.upstreams.save({ ...baseRow, id: 'up_failed', name: 'Failed', sortOrder: 2 }); - - await seedModelsCache(repo.upstreams, 'up_warm', { updatedAt: baseRow.updatedAt, config: baseRow.config }, { + const freshRecord = { ...baseRow, id: 'up_fresh', name: 'Fresh', sortOrder: 0 }; + const warmRecord = { ...baseRow, id: 'up_warm', name: 'Warm', sortOrder: 1 }; + const failedRecord = { ...baseRow, id: 'up_failed', name: 'Failed', sortOrder: 2 }; + await repo.upstreams.save(freshRecord); + await repo.upstreams.save(warmRecord); + await repo.upstreams.save(failedRecord); + + await seedModelsCache(repo.upstreams, 'up_warm', modelsCacheGeneration(warmRecord), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [{ id: 'm1', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], }); - await seedModelsCache(repo.upstreams, 'up_failed', { updatedAt: baseRow.updatedAt, config: baseRow.config }, { + await seedModelsCache(repo.upstreams, 'up_failed', modelsCacheGeneration(failedRecord), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [{ id: 'm1', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], }); - await seedModelsCacheError(repo.upstreams, 'up_failed', { updatedAt: baseRow.updatedAt, config: baseRow.config }, { message: 'boom', at: 1_700_000_500_000 }); + await seedModelsCacheError(repo.upstreams, 'up_failed', modelsCacheGeneration(failedRecord), { message: 'boom', at: 1_700_000_500_000 }); const list = await requestApp('/api/upstreams', { headers: { 'x-floway-session': adminSession } }); assertEquals(list.status, 200); @@ -650,7 +653,7 @@ test('POST /api/upstreams/list-models rejects a malformed draft config with 400' assertEquals(body.error.includes('apiKey'), true); }); -test('POST /api/upstreams/list-models with a persisted id forces a fresh upstream fetch and updates the SWR cache', async () => { +test('POST /api/upstreams/list-models with matching saved inputs fetches and publishes a fresh snapshot', async () => { const { repo, adminSession } = await setupAppTest(); await repo.upstreams.deleteAll(); const savedRecord: UpstreamRecord = { @@ -789,9 +792,9 @@ test('PATCH /api/upstreams metadata warm preserves refresh backoff', async () => ); const generation = await getCacheGeneration(repo, created.id); const now = Date.now(); - const claim = await repo.upstreams.claimModelsRefresh({ id: created.id, generation, token: 'failed-refresh', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null }); + const claim = await repo.upstreams.claimModelsRefresh({ id: created.id, generation, token: 'failed-refresh', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); - await repo.upstreams.finalizeModelsRefreshFailure(created.id, generation, 'failed-refresh', { message: 'failed refresh', at: now }, 1, modelsRefreshRetryAt(now, 0)); + await repo.upstreams.finalizeModelsRefreshFailure({ id: created.id, generation, token: 'failed-refresh', error: { message: 'failed refresh', at: now }, previousFailureCount: 0, failedAt: now }); let modelRequests = 0; await withMockedFetch( @@ -901,7 +904,7 @@ const getRecord = async (repo: { upstreams: { getById: (id: string) => Promise Promise } }, id: string) => { const record = await getRecord(repo, id); - return { updatedAt: record.updatedAt, config: record.config }; + return modelsCacheGeneration(record); }; test('POST /api/upstreams/codex/oauth/authorize-url stamps SPA-provided challenge + state into the auth.openai.com URL', async () => { @@ -2173,8 +2176,8 @@ test('spec invariant (3): POST /api/upstreams/claude-code/probe does not persist test('spec invariant (3): POST /api/upstreams/list-models ignores record.name mutation on a saved row', async () => { const { repo, adminSession } = await setupAppTest(); await repo.upstreams.deleteAll(); - // Azure sits in the SWR-cached branch alongside copilot / codex / - // claude-code, so this exercises the `fetchUpstreamModelsCached` path a + // Azure publishes through the persisted-snapshot branch alongside Copilot / Codex / + // claude-code, so this exercises the `readUpstreamModelsSnapshotAndScheduleRefresh` path a // future "refresh row metadata" regression would land in. Azure's // getProvidedModels reads directly from config.models — no upstream mock // needed, no credential mint. diff --git a/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts index 4283690a7..82e6e466b 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts @@ -7,7 +7,7 @@ import type { ModelEndpoints } from '@floway-dev/protocols/common'; import type { UpstreamRecord } from '@floway-dev/provider'; import { assertEquals } from '@floway-dev/test-utils'; -// Drains SWR background revalidate so a rejection surfaces in the runner +// Drains the separately scheduled snapshot refresh so a rejection surfaces in the runner // instead of being swallowed. const testScheduler = (promise: Promise): void => { promise.catch(err => console.error('[background]', err)); diff --git a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts index dbf00111e..49396d899 100644 --- a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts @@ -348,7 +348,7 @@ test('catalog assembly: a rejected provider does not block other providers', asy // End-to-end listing checks for the prefix policy. The catalog walk goes // through getModelsFromProviders, which threads custom upstreams' /v1/models -// responses through fetchUpstreamModelsCached just like production does. +// responses through readUpstreamModelsSnapshotAndScheduleRefresh just like production does. describe('catalog listing under modelPrefix', () => { test('null prefix lists bare ids only (today\'s behavior)', async () => { const { repo } = await setupAppTest(); diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 7a8a9edb4..0621b3b53 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -1,11 +1,11 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { clearInFlightForTesting, fetchUpstreamModelsCached, MODEL_CATALOG_REVISION, warmUpstreamModels } from '../../../src/data-plane/providers/models-cache.ts'; +import { clearInFlightForTesting, fetchUpstreamModels, readUpstreamModelsSnapshotAndScheduleRefresh, MODEL_CATALOG_REVISION, warmUpstreamModels } from '../../../src/data-plane/providers/models-cache.ts'; import type { GatewayProvider } from '../../../src/data-plane/providers/registry.ts'; import { initRepo } from '../../../src/repo/index.ts'; +import { modelsFetchIdentity } from '../../../src/repo/models-cache-contract.ts'; import { SqlRepo } from '../../../src/repo/sql.ts'; import type { ModelsCacheGeneration } from '../../../src/repo/types.ts'; -import { serializeStoredConfig } from '../../../src/repo/upstream-json.ts'; import { InMemoryRepo } from '../../repo/memory.ts'; import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; import { createSqliteTestDb } from '../../repo/test-sqlite.ts'; @@ -13,10 +13,16 @@ import { directFetcher, type ProviderModel, type UpstreamModelsCache } from '@fl import { stubProvider, stubProviderModel } from '@floway-dev/test-utils'; const UPSTREAM_ID = 'up_a'; -const CACHE_GENERATION = vi.hoisted(() => ({ +const CACHE_CONFIG = { identity: 'old' }; +const fetchIdentityForConfig = (config: unknown): string => modelsFetchIdentity({ + kind: 'custom', + config, + proxyFallbackList: [], +}); +const CACHE_GENERATION: ModelsCacheGeneration = { updatedAt: '2026-08-01T00:00:00.000Z', - config: { identity: 'old' }, -})); + fetchIdentity: fetchIdentityForConfig(CACHE_CONFIG), +}; const aModel = (id: string): ProviderModel => stubProviderModel({ id }); @@ -24,7 +30,7 @@ const stubInstance = ( fetchFn: () => Promise, modelsCache: UpstreamModelsCache | null = null, generation: ModelsCacheGeneration = CACHE_GENERATION, - fetchIdentity = serializeStoredConfig(generation.config), + fetchIdentity = generation.fetchIdentity, ): GatewayProvider => ({ upstreamId: UPSTREAM_ID, kind: 'custom', @@ -34,8 +40,7 @@ const stubInstance = ( modelPrefix: null, modelsCache, instance: stubProvider({ getProvidedModels: fetchFn }), - modelsCacheGeneration: generation, - modelsFetchIdentity: fetchIdentity, + modelsCacheGeneration: { ...generation, fetchIdentity }, }); const setupRepo = async (): Promise => { @@ -49,7 +54,7 @@ const setupRepo = async (): Promise => { sortOrder: 0, createdAt: '2026-08-01T00:00:00.000Z', updatedAt: CACHE_GENERATION.updatedAt, - config: CACHE_GENERATION.config, + config: CACHE_CONFIG, state: null, modelsCache: null, flagOverrides: {}, @@ -87,19 +92,19 @@ beforeEach(() => { clearInFlightForTesting(); }); -describe('fetchUpstreamModelsCached', () => { +describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { test('cold cache returns immediately and refreshes in the background', async () => { const repo = await setupRepo(); let resolveFetch: ((models: ProviderModel[]) => void) | null = null; const fetchFn = vi.fn(() => new Promise(resolve => { resolveFetch = resolve; })); const scheduled = captureScheduled(); - const result = await fetchUpstreamModelsCached( + const result = readUpstreamModelsSnapshotAndScheduleRefresh( stubInstance(fetchFn), { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); - expect(result).toEqual([]); + expect(result.models).toEqual([]); expect(scheduled.promises).toHaveLength(1); await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); resolveFetch!([aModel('m1')]); @@ -113,48 +118,42 @@ describe('fetchUpstreamModelsCached', () => { const fetchFn = vi.fn(async () => [aModel('fresh')]); const scheduled = captureScheduled(); - const result = await fetchUpstreamModelsCached( + const result = readUpstreamModelsSnapshotAndScheduleRefresh( stubInstance(fetchFn, cache), { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); - expect(result.map(model => model.id)).toEqual(['cached']); + expect(result.models.map(model => model.id)).toEqual(['cached']); expect(scheduled.promises).toEqual([]); expect(fetchFn).not.toHaveBeenCalled(); }); - test('every stale age remains SWR forever', async () => { + test('snapshots remain usable regardless of age while refresh runs separately', async () => { const repo = await setupRepo(); const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 365 * 24 * 60 * 60_000, models: [aModel('stale')] }); let resolveFetch: ((models: ProviderModel[]) => void) | null = null; const fetchFn = vi.fn(() => new Promise(resolve => { resolveFetch = resolve; })); const scheduled = captureScheduled(); - const result = await fetchUpstreamModelsCached( + const result = readUpstreamModelsSnapshotAndScheduleRefresh( stubInstance(fetchFn, cache), { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); - expect(result.map(model => model.id)).toEqual(['stale']); + expect(result.models.map(model => model.id)).toEqual(['stale']); await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); resolveFetch!([aModel('fresh')]); await scheduled.promises[0]; expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['fresh']); }); - test('force is the explicit fetch operation and blocks for a fresh result', async () => { + test('explicit fetch blocks for a fresh result', async () => { const repo = await setupRepo(); const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 1000, models: [aModel('stored')] }); const fetchFn = vi.fn(async () => [aModel('fresh')]); - const scheduled = captureScheduled(); - - const result = await fetchUpstreamModelsCached( - stubInstance(fetchFn, cache), - { scheduler: scheduled.scheduler, fetcher: directFetcher, force: true }, - ); + const result = await fetchUpstreamModels(stubInstance(fetchFn, cache), directFetcher); expect(result.map(model => model.id)).toEqual(['fresh']); - expect(scheduled.promises).toEqual([]); expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['fresh']); }); @@ -165,13 +164,13 @@ describe('fetchUpstreamModelsCached', () => { const instance = stubInstance(fetchFn); const scheduled = captureScheduled(); - const [first, second] = await Promise.all([ - fetchUpstreamModelsCached(instance, { scheduler: scheduled.scheduler, fetcher: directFetcher }), - fetchUpstreamModelsCached(instance, { scheduler: scheduled.scheduler, fetcher: directFetcher }), - ]); + const [first, second] = [ + readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: scheduled.scheduler, fetcher: directFetcher }), + readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: scheduled.scheduler, fetcher: directFetcher }), + ]; - expect(first).toEqual([]); - expect(second).toEqual([]); + expect(first.models).toEqual([]); + expect(second.models).toEqual([]); await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); resolveFetch!([aModel('m1')]); await Promise.all(scheduled.promises); @@ -186,12 +185,12 @@ describe('fetchUpstreamModelsCached', () => { const instance = stubInstance(fetchFn, cache); const firstScheduled = captureScheduled(); - expect((await fetchUpstreamModelsCached(instance, { scheduler: firstScheduled.scheduler, fetcher: directFetcher })).map(model => model.id)).toEqual(['stale']); + expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: firstScheduled.scheduler, fetcher: directFetcher }).models.map(model => model.id)).toEqual(['stale']); await expect(firstScheduled.promises[0]).rejects.toThrow('boom'); clearInFlightForTesting(); const secondScheduled = captureScheduled(); - expect((await fetchUpstreamModelsCached(instance, { scheduler: secondScheduled.scheduler, fetcher: directFetcher })).map(model => model.id)).toEqual(['stale']); + expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: secondScheduled.scheduler, fetcher: directFetcher }).models.map(model => model.id)).toEqual(['stale']); await expect(secondScheduled.promises[0]).resolves.toBeUndefined(); expect(fetchFn).toHaveBeenCalledTimes(1); expect((await storedCache(repo))?.lastError?.message).toContain('boom'); @@ -205,42 +204,42 @@ describe('fetchUpstreamModelsCached', () => { const instance = stubInstance(fetchFn); const firstScheduled = captureScheduled(); - await expect(fetchUpstreamModelsCached(instance, { scheduler: firstScheduled.scheduler, fetcher: directFetcher })).resolves.toEqual([]); + expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: firstScheduled.scheduler, fetcher: directFetcher }).models).toEqual([]); await expect(firstScheduled.promises[0]).rejects.toThrow('boom'); expect(await storedCache(repo)).toMatchObject({ fetchedAt: 0, models: [], lastError: { message: 'boom' } }); clearInFlightForTesting(); now += 59_999; const backedOff = captureScheduled(); - await expect(fetchUpstreamModelsCached(instance, { scheduler: backedOff.scheduler, fetcher: directFetcher })).resolves.toEqual([]); + expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: backedOff.scheduler, fetcher: directFetcher }).models).toEqual([]); await expect(backedOff.promises[0]).resolves.toBeUndefined(); expect(fetchFn).toHaveBeenCalledTimes(1); clearInFlightForTesting(); now += 1; const retry = captureScheduled(); - await expect(fetchUpstreamModelsCached(instance, { scheduler: retry.scheduler, fetcher: directFetcher })).resolves.toEqual([]); + expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: retry.scheduler, fetcher: directFetcher }).models).toEqual([]); await expect(retry.promises[0]).rejects.toThrow('boom'); expect(fetchFn).toHaveBeenCalledTimes(2); }); - test('synchronous warm respects backoff while explicit force bypasses it', async () => { + test('synchronous warm respects backoff while explicit fetch bypasses it', async () => { const repo = await setupRepo(); const now = 1_800_000_000_000; vi.spyOn(Date, 'now').mockReturnValue(now); - const failing = stubInstance(async () => { throw new Error('boom'); }, null, CACHE_GENERATION, 'backoff-source'); + const failing = stubInstance(async () => { throw new Error('boom'); }); const scheduled = captureScheduled(); - await fetchUpstreamModelsCached(failing, { scheduler: scheduled.scheduler, fetcher: directFetcher }); + readUpstreamModelsSnapshotAndScheduleRefresh(failing, { scheduler: scheduled.scheduler, fetcher: directFetcher }); await expect(scheduled.promises[0]).rejects.toThrow('boom'); clearInFlightForTesting(); const fetchFn = vi.fn(async () => [aModel('recovered')]); const cache = await storedCache(repo); - const warming = stubInstance(fetchFn, cache, CACHE_GENERATION, 'warm-during-backoff'); + const warming = stubInstance(fetchFn, cache); await expect(warmUpstreamModels(warming, directFetcher)).resolves.toEqual([]); expect(fetchFn).not.toHaveBeenCalled(); - await expect(fetchUpstreamModelsCached(warming, { scheduler: () => {}, fetcher: directFetcher, force: true })) + await expect(fetchUpstreamModels(warming, directFetcher)) .resolves.toEqual([aModel('recovered')]); expect(fetchFn).toHaveBeenCalledTimes(1); }); @@ -248,7 +247,7 @@ describe('fetchUpstreamModelsCached', () => { test('synchronous warm waits for a refresh owned by another runtime', async () => { const repo = await setupRepo(); const now = Date.now(); - await expect(repo.upstreams.claimModelsRefresh({ id: UPSTREAM_ID, generation: CACHE_GENERATION, token: 'remote-owner', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null })) + await expect(repo.upstreams.claimModelsRefresh({ id: UPSTREAM_ID, generation: CACHE_GENERATION, token: 'remote-owner', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null })) .resolves.toEqual({ kind: 'claimed', failureCount: 0 }); const localFetch = vi.fn(async () => [aModel('duplicate-local-model')]); const warming = warmUpstreamModels(stubInstance(localFetch), directFetcher); @@ -258,53 +257,51 @@ describe('fetchUpstreamModelsCached', () => { await new Promise(resolve => setTimeout(resolve, 20)); expect(settled).toBe(false); - await repo.upstreams.finalizeModelsRefreshSuccess(UPSTREAM_ID, CACHE_GENERATION, 'remote-owner', { - revision: MODEL_CATALOG_REVISION, - fetchedAt: now + 1, - models: [aModel('remote-model')], + await repo.upstreams.finalizeModelsRefreshSuccess({ + id: UPSTREAM_ID, + generation: CACHE_GENERATION, + token: 'remote-owner', + cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [aModel('remote-model')] }, }); expect((await warming).map(model => model.id)).toEqual(['remote-model']); expect(localFetch).not.toHaveBeenCalled(); }); - test('explicit force bypasses a local warm waiting on another runtime', async () => { + test('explicit fetch follows the durable owner already awaited by a local warm', async () => { const repo = await setupRepo(); const now = Date.now(); - await repo.upstreams.claimModelsRefresh({ id: UPSTREAM_ID, generation: CACHE_GENERATION, token: 'remote-owner', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null }); - const fetchFn = vi.fn(async () => [aModel('forced-model')]); - const instance = stubInstance(fetchFn, null, CACHE_GENERATION, 'shared-warm-force-key'); + await repo.upstreams.claimModelsRefresh({ id: UPSTREAM_ID, generation: CACHE_GENERATION, token: 'remote-owner', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); + const fetchFn = vi.fn(async () => [aModel('duplicate-local-model')]); + const instance = stubInstance(fetchFn); const warming = warmUpstreamModels(instance, directFetcher); await new Promise(resolve => setTimeout(resolve, 20)); - const forced = fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher, force: true }); - await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); - expect((await forced).map(model => model.id)).toEqual(['forced-model']); - expect((await warming).map(model => model.id)).toEqual(['forced-model']); + const explicit = fetchUpstreamModels(instance, directFetcher); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(fetchFn).not.toHaveBeenCalled(); + await repo.upstreams.finalizeModelsRefreshSuccess({ + id: UPSTREAM_ID, + generation: CACHE_GENERATION, + token: 'remote-owner', + cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [aModel('remote-model')] }, + }); + expect((await explicit).map(model => model.id)).toEqual(['remote-model']); + expect((await warming).map(model => model.id)).toEqual(['remote-model']); }); - test('a warm superseded during I/O waits for the forced owner to finalize', async () => { + test('explicit fetch joins a warm that already owns the durable refresh', async () => { await setupRepo(); let resolveWarm: ((models: ProviderModel[]) => void) | null = null; - let resolveForce: ((models: ProviderModel[]) => void) | null = null; - const fetchFn = vi.fn() - .mockImplementationOnce(() => new Promise(resolve => { resolveWarm = resolve; })) - .mockImplementationOnce(() => new Promise(resolve => { resolveForce = resolve; })); - const instance = stubInstance(fetchFn, null, CACHE_GENERATION, 'warm-force-race'); + const fetchFn = vi.fn(() => new Promise(resolve => { resolveWarm = resolve; })); + const instance = stubInstance(fetchFn); const warming = warmUpstreamModels(instance, directFetcher); await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); - const forced = fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher, force: true }); - await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(2)); - - let warmSettled = false; - void warming.finally(() => { warmSettled = true; }); - resolveWarm!([aModel('superseded-warm-model')]); - await new Promise(resolve => setTimeout(resolve, 20)); - expect(warmSettled).toBe(false); - - resolveForce!([aModel('forced-winner-model')]); - expect((await forced).map(model => model.id)).toEqual(['forced-winner-model']); - expect((await warming).map(model => model.id)).toEqual(['forced-winner-model']); + const explicit = fetchUpstreamModels(instance, directFetcher); + resolveWarm!([aModel('warm-owner-model')]); + expect((await explicit).map(model => model.id)).toEqual(['warm-owner-model']); + expect((await warming).map(model => model.id)).toEqual(['warm-owner-model']); + expect(fetchFn).toHaveBeenCalledTimes(1); }); test('an atomic success-finalize failure does not install upstream failure backoff', async () => { @@ -313,7 +310,7 @@ describe('fetchUpstreamModelsCached', () => { vi.spyOn(repo.upstreams, 'finalizeModelsRefreshSuccess').mockRejectedValueOnce(new Error('finalize failed')); const instance = stubInstance(async () => [aModel('published-model')]); - await expect(fetchUpstreamModelsCached(instance, { scheduler: () => {}, fetcher: directFetcher, force: true })) + await expect(fetchUpstreamModels(instance, directFetcher)) .rejects.toThrow('finalize failed'); expect(finalizeFailure).not.toHaveBeenCalled(); expect(await storedCache(repo)).toBeNull(); @@ -324,21 +321,19 @@ describe('fetchUpstreamModelsCached', () => { let resolveOld: ((models: ProviderModel[]) => void) | null = null; const oldFetch = vi.fn(() => new Promise(resolve => { resolveOld = resolve; })); const oldScheduled = captureScheduled(); - await fetchUpstreamModelsCached( - stubInstance(oldFetch, null, CACHE_GENERATION, 'same-fetch'), + readUpstreamModelsSnapshotAndScheduleRefresh( + stubInstance(oldFetch), { scheduler: oldScheduled.scheduler, fetcher: directFetcher }, ); await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); - const nextGeneration = { updatedAt: CACHE_GENERATION.updatedAt, config: { identity: 'new' } }; + const nextConfig = { identity: 'new' }; + const nextGeneration = { updatedAt: CACHE_GENERATION.updatedAt, fetchIdentity: fetchIdentityForConfig(nextConfig) }; const current = await repo.upstreams.getById(UPSTREAM_ID); if (!current) throw new Error('upstream row missing'); - await repo.upstreams.saveClearingModelsCache({ ...current, updatedAt: nextGeneration.updatedAt, config: nextGeneration.config }); + await repo.upstreams.replaceForModels({ previous: current, upstream: { ...current, updatedAt: nextGeneration.updatedAt, config: nextConfig }, cachePolicy: 'clear' }); const newFetch = vi.fn(async () => [aModel('new-tenant-model')]); - const newResult = await fetchUpstreamModelsCached( - stubInstance(newFetch, null, nextGeneration, 'same-fetch'), - { scheduler: () => {}, fetcher: directFetcher, force: true }, - ); + const newResult = await fetchUpstreamModels(stubInstance(newFetch, null, nextGeneration), directFetcher); expect(newResult.map(model => model.id)).toEqual(['new-tenant-model']); resolveOld!([aModel('old-tenant-model')]); @@ -348,55 +343,44 @@ describe('fetchUpstreamModelsCached', () => { expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['new-tenant-model']); }); - test('a forced fetch prevents an older claim from publishing a late success', async () => { + test('explicit fetch joins an older background refresh instead of preempting it', async () => { const repo = await setupRepo(); let resolveOld: ((models: ProviderModel[]) => void) | null = null; const oldFetch = vi.fn(() => new Promise(resolve => { resolveOld = resolve; })); const oldScheduled = captureScheduled(); - await fetchUpstreamModelsCached( - stubInstance(oldFetch, null, CACHE_GENERATION, 'old-claim'), + readUpstreamModelsSnapshotAndScheduleRefresh( + stubInstance(oldFetch), { scheduler: oldScheduled.scheduler, fetcher: directFetcher }, ); await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); - const forced = await fetchUpstreamModelsCached( - stubInstance(async () => [aModel('forced-model')], null, CACHE_GENERATION, 'forced-claim'), - { scheduler: () => {}, fetcher: directFetcher, force: true }, - ); - expect(forced.map(model => model.id)).toEqual(['forced-model']); - + const explicitFetch = vi.fn(async () => [aModel('duplicate-explicit-model')]); + const explicit = fetchUpstreamModels(stubInstance(explicitFetch), directFetcher); resolveOld!([aModel('late-old-model')]); + expect((await explicit).map(model => model.id)).toEqual(['late-old-model']); await oldScheduled.promises[0]; - expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['forced-model']); + expect(explicitFetch).not.toHaveBeenCalled(); + expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['late-old-model']); }); - test('a forced fetch prevents an older claim from publishing a late error', async () => { + test('explicit fetch surfaces failure from an older background owner', async () => { const repo = await setupRepo(); let rejectOld: ((error: Error) => void) | null = null; const oldFetch = vi.fn(() => new Promise((_resolve, reject) => { rejectOld = reject; })); const oldScheduled = captureScheduled(); - await fetchUpstreamModelsCached( - stubInstance(oldFetch, null, CACHE_GENERATION, 'old-error-claim'), + readUpstreamModelsSnapshotAndScheduleRefresh( + stubInstance(oldFetch), { scheduler: oldScheduled.scheduler, fetcher: directFetcher }, ); await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); - await fetchUpstreamModelsCached( - stubInstance(async () => [aModel('forced-model')], null, CACHE_GENERATION, 'forced-error-claim'), - { scheduler: () => {}, fetcher: directFetcher, force: true }, - ); + const explicitFetch = vi.fn(async () => [aModel('duplicate-explicit-model')]); + const explicit = fetchUpstreamModels(stubInstance(explicitFetch), directFetcher); rejectOld!(new Error('late old failure')); await expect(oldScheduled.promises[0]).rejects.toThrow('late old failure'); - - expect(await storedCache(repo)).toMatchObject({ - models: [{ id: 'forced-model' }], - lastError: null, - }); - - clearInFlightForTesting(); - const recovery = vi.fn(async () => [aModel('post-race-model')]); - await warmUpstreamModels(stubInstance(recovery, await storedCache(repo), CACHE_GENERATION, 'post-race'), directFetcher); - expect(recovery).toHaveBeenCalledTimes(1); + await expect(explicit).rejects.toThrow('late old failure'); + expect(explicitFetch).not.toHaveBeenCalled(); + expect(await storedCache(repo)).toMatchObject({ models: [], lastError: { message: 'late old failure' } }); }); test('catalog revision mismatch is cold and refreshes without blocking', async () => { @@ -409,12 +393,12 @@ describe('fetchUpstreamModelsCached', () => { const fetchFn = vi.fn(async () => [aModel('current-catalog')]); const scheduled = captureScheduled(); - const result = await fetchUpstreamModelsCached( + const result = readUpstreamModelsSnapshotAndScheduleRefresh( stubInstance(fetchFn, cache), { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); - expect(result).toEqual([]); + expect(result.models).toEqual([]); await scheduled.promises[0]; expect((await storedCache(repo))?.revision).toBe(MODEL_CATALOG_REVISION); }); @@ -452,12 +436,12 @@ describe('fetchUpstreamModelsCached', () => { expect(hydrated.modelsCache).toBeNull(); const fetchFn = vi.fn(async () => [aModel('current-catalog')]); const scheduled = captureScheduled(); - const result = await fetchUpstreamModelsCached( - stubInstance(fetchFn, hydrated.modelsCache, { updatedAt: hydrated.updatedAt, config: hydrated.config }), + const result = readUpstreamModelsSnapshotAndScheduleRefresh( + stubInstance(fetchFn, hydrated.modelsCache, { updatedAt: hydrated.updatedAt, fetchIdentity: modelsFetchIdentity(hydrated) }), { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); - expect(result).toEqual([]); + expect(result.models).toEqual([]); await scheduled.promises[0]; expect((await repo.upstreams.getById(UPSTREAM_ID))?.modelsCache?.revision).toBe(MODEL_CATALOG_REVISION); }); diff --git a/packages/gateway/__tests__/data-plane/providers/registry_test.ts b/packages/gateway/__tests__/data-plane/providers/registry_test.ts index 6dd1be7cf..a368294c1 100644 --- a/packages/gateway/__tests__/data-plane/providers/registry_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/registry_test.ts @@ -1,11 +1,21 @@ -import { test } from 'vitest'; +import { expect, test } from 'vitest'; import { MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; -import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; +import { listModelProviders, modelsCatalogIdentity } from '../../../src/data-plane/providers/registry.ts'; +import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; import { buildCopilotUpstreamRecord, buildCustomUpstreamRecord, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; +test('Copilot catalog identity follows the account rather than rotated credentials', () => { + const first = buildCopilotUpstreamRecord({ token: 'ghu_first', user: { id: 1, login: 'one', avatar_url: '', name: null } }); + const rotated = buildCopilotUpstreamRecord({ token: 'ghu_rotated', user: { id: 1, login: 'one-renamed', avatar_url: '', name: null } }); + const otherAccount = buildCopilotUpstreamRecord({ token: 'ghu_other', user: { id: 2, login: 'two', avatar_url: '', name: null } }); + + assertEquals(modelsCatalogIdentity(first), modelsCatalogIdentity(rotated)); + expect(modelsCatalogIdentity(first)).not.toBe(modelsCatalogIdentity(otherAccount)); +}); + test('listModelProviders creates enabled provider instances with upstream row ids', async () => { const { githubAccount, repo } = await setupAppTest(); await repo.upstreams.deleteAll(); @@ -89,14 +99,14 @@ test('listModelProviders silently drops deleted upstreams from a whitelist', asy }); test('listModelProviders carries each row cached catalog onto its instance', async () => { - // The SWR layer reads the catalog off the instance instead of paying a + // Resolution reads the persisted snapshot off the instance instead of paying a // second round trip, so the row read has to bring it along. const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); const cachedRecord = buildCustomUpstreamRecord({ id: 'up_cached', name: 'Cached', sortOrder: 10 }); await repo.upstreams.save(cachedRecord); await repo.upstreams.save(buildCustomUpstreamRecord({ id: 'up_cold', name: 'Cold', sortOrder: 20 })); - await seedModelsCache(repo.upstreams, 'up_cached', { updatedAt: cachedRecord.updatedAt, config: cachedRecord.config }, { + await seedModelsCache(repo.upstreams, 'up_cached', modelsCacheGeneration(cachedRecord), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index 2c38e3ada..e787030e1 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -3,7 +3,8 @@ import { partitionTelemetryOverviewRecords } from './telemetry-overview-oracle.t import { buildKeyToUserMap } from '../../src/control-plane/shared/key-to-user.ts'; import { normalizeDisabledPublicModelIds } from '../../src/repo/disabled-public-models.ts'; import { normalizeFlagOverrides } from '../../src/repo/flag-overrides.ts'; -import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; +import { MODEL_CATALOG_REVISION, modelsFetchIdentity } from '../../src/repo/models-cache-contract.ts'; +import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; import { normalizeProxyFallbackList } from '../../src/repo/proxy-fallback-list.ts'; import { assertSameStoredResponsesItem, @@ -31,6 +32,8 @@ import type { ModelsCacheGeneration, ModelsRefreshClaimInput, ModelsRefreshClaimResult, + ModelsRefreshFailureInput, + ModelsRefreshSuccessInput, ModelAliasesRepo, ModelAliasRecord, PerformanceDimensions, @@ -759,14 +762,27 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(); } - saveClearingModelsCache(upstream: UpstreamRecord): Promise { + replaceForModels(input: { + previous: UpstreamRecord; + upstream: UpstreamRecord; + cachePolicy: 'preserve' | 'reset-refresh' | 'clear'; + }): Promise { + const { previous, upstream, cachePolicy } = input; const existing = this.store.get(upstream.id); - const next = existing - ? { ...upstream, createdAt: existing.createdAt, modelsCache: null } - : { ...upstream, modelsCache: null }; - this.store.set(next.id, cloneUpstreamRecord(next)); - this.modelsRefreshes.delete(next.id); - return Promise.resolve(); + if (existing === undefined || serializeStoredConfig({ ...existing, modelsCache: null }) !== serializeStoredConfig({ ...previous, modelsCache: null })) return Promise.resolve(false); + const next = cloneUpstreamRecord({ + ...upstream, + createdAt: existing.createdAt, + modelsCache: cachePolicy === 'clear' ? null : existing.modelsCache, + }); + this.store.set(upstream.id, next); + if (cachePolicy === 'preserve') { + const refresh = this.modelsRefreshes.get(upstream.id); + if (refresh !== undefined) this.modelsRefreshes.set(upstream.id, { ...refresh, claimToken: null, claimedAt: null }); + } else { + this.modelsRefreshes.delete(upstream.id); + } + return Promise.resolve(true); } delete(id: string): Promise { @@ -793,19 +809,23 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(); } - finalizeModelsRefreshSuccess(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { + finalizeModelsRefreshSuccess(input: ModelsRefreshSuccessInput): Promise { + const { id, generation, token, cache } = input; if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); const existing = this.store.get(id); - if (!existing || existing.updatedAt !== generation.updatedAt || serializeStoredConfig(existing.config) !== serializeStoredConfig(generation.config)) return Promise.resolve(false); + if (!existing || existing.updatedAt !== generation.updatedAt || modelsFetchIdentity(existing) !== generation.fetchIdentity) return Promise.resolve(false); existing.modelsCache = { revision: cache.revision, fetchedAt: cache.fetchedAt, models: [...cache.models], lastError: null }; this.modelsRefreshes.delete(id); return Promise.resolve(true); } - finalizeModelsRefreshFailure(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable, failureCount: number, retryAt: number): Promise { + finalizeModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise { + const { id, generation, token, error, previousFailureCount, failedAt } = input; + const failureCount = previousFailureCount + 1; + const retryAt = modelsRefreshRetryAt(failedAt, previousFailureCount); if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); const existing = this.store.get(id); - if (!existing || existing.updatedAt !== generation.updatedAt || serializeStoredConfig(existing.config) !== serializeStoredConfig(generation.config)) return Promise.resolve(false); + if (!existing || existing.updatedAt !== generation.updatedAt || modelsFetchIdentity(existing) !== generation.fetchIdentity) return Promise.resolve(false); if (existing.modelsCache) existing.modelsCache.lastError = error; else existing.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }; this.modelsRefreshes.set(id, { failCount: failureCount, retryAt, claimToken: null, claimedAt: null }); @@ -813,15 +833,15 @@ class MemoryUpstreamRepo implements UpstreamRepo { } claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { - const { id, generation, token, now, staleClaimedBefore, force, observedActiveToken } = input; + const { id, generation, token, now, staleClaimedBefore, bypassBackoff, observedActiveToken } = input; const stored = this.store.get(id); - if (!stored || stored.updatedAt !== generation.updatedAt || serializeStoredConfig(stored.config) !== serializeStoredConfig(generation.config)) return Promise.resolve({ kind: 'generation-mismatch' }); + if (!stored || stored.updatedAt !== generation.updatedAt || modelsFetchIdentity(stored) !== generation.fetchIdentity) return Promise.resolve({ kind: 'generation-mismatch' }); const existing = this.modelsRefreshes.get(id); - if (!force && observedActiveToken !== null && existing === undefined) return Promise.resolve({ kind: 'completed' }); - if (!force && existing !== undefined) { + if (observedActiveToken !== null && existing === undefined) return Promise.resolve({ kind: 'completed' }); + if (existing !== undefined) { if (existing.claimToken !== null && existing.claimedAt! > staleClaimedBefore) return Promise.resolve({ kind: 'active', token: existing.claimToken }); - if (existing.retryAt > now) return Promise.resolve({ kind: 'backoff' }); if (observedActiveToken !== null && existing.claimToken === null) return Promise.resolve({ kind: 'completed' }); + if (!bypassBackoff && existing.retryAt > now) return Promise.resolve({ kind: 'backoff' }); } this.modelsRefreshes.set(id, { failCount: existing?.failCount ?? 0, diff --git a/packages/gateway/__tests__/repo/models-cache-fixture.ts b/packages/gateway/__tests__/repo/models-cache-fixture.ts index 45506b708..771bd865b 100644 --- a/packages/gateway/__tests__/repo/models-cache-fixture.ts +++ b/packages/gateway/__tests__/repo/models-cache-fixture.ts @@ -8,9 +8,9 @@ export const seedModelsCache = async ( cache: Omit, ): Promise => { const token = crypto.randomUUID(); - const claim = await repo.claimModelsRefresh({ id, generation, token, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, force: true, observedActiveToken: null }); + const claim = await repo.claimModelsRefresh({ id, generation, token, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, bypassBackoff: true, observedActiveToken: null }); if (claim.kind !== 'claimed') return false; - return await repo.finalizeModelsRefreshSuccess(id, generation, token, cache); + return await repo.finalizeModelsRefreshSuccess({ id, generation, token, cache }); }; export const seedModelsCacheError = async ( @@ -20,7 +20,7 @@ export const seedModelsCacheError = async ( error: NonNullable, ): Promise => { const token = crypto.randomUUID(); - const claim = await repo.claimModelsRefresh({ id, generation, token, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, force: true, observedActiveToken: null }); + const claim = await repo.claimModelsRefresh({ id, generation, token, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, bypassBackoff: true, observedActiveToken: null }); if (claim.kind !== 'claimed') return false; - return await repo.finalizeModelsRefreshFailure(id, generation, token, error, 0, 0); + return await repo.finalizeModelsRefreshFailure({ id, generation, token, error, previousFailureCount: 0, failedAt: -60_000 }); }; diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index f04e7fb23..4a84a1437 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from 'vitest'; import { InMemoryRepo } from './memory.ts'; import { createSqliteTestDb } from './test-sqlite.ts'; -import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; +import { MODEL_CATALOG_REVISION, modelsCacheGeneration } from '../../src/repo/models-cache-contract.ts'; import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import type { ModelsCacheGeneration, Repo } from '../../src/repo/types.ts'; @@ -26,7 +26,7 @@ const record: UpstreamRecord = { hue: 210, }; -const generation: ModelsCacheGeneration = { updatedAt: record.updatedAt, config: record.config }; +const generation: ModelsCacheGeneration = modelsCacheGeneration(record); const factories: [string, () => Promise][] = [ ['memory', async () => new InMemoryRepo()], @@ -39,9 +39,9 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.save(record); let now = 1_800_000_000_000; - const first = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'claim-0', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null }); + const first = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'claim-0', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); expect(first).toEqual({ kind: 'claimed', failureCount: 0 }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'racer', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'claim-0' }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'racer', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'claim-0' }); const delays = [1, 2, 4, 8, 16, 32, 60, 60].map(minutes => minutes * 60_000); if (first.kind !== 'claimed') throw new Error('expected refresh claim'); @@ -49,20 +49,19 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = for (const [index, delay] of delays.entries()) { const retryAt = modelsRefreshRetryAt(now, claim.failureCount); expect(retryAt - now).toBe(delay); - await repo.upstreams.finalizeModelsRefreshFailure(record.id, generation, `claim-${index}`, { message: 'failure', at: now }, claim.failureCount + 1, retryAt); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: `early-${index}`, now: retryAt - 1, staleClaimedBefore: retryAt - 900_001, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'backoff' }); + await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: `claim-${index}`, error: { message: 'failure', at: now }, previousFailureCount: claim.failureCount, failedAt: now }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: `early-${index}`, now: retryAt - 1, staleClaimedBefore: retryAt - 900_001, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'backoff' }); now = retryAt; - const nextClaim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: `claim-${index + 1}`, now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null }); + const nextClaim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: `claim-${index + 1}`, now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); if (nextClaim.kind !== 'claimed') throw new Error('expected refresh claim'); claim = nextClaim; expect(claim.failureCount).toBe(index + 1); } - const blockedUntil = modelsRefreshRetryAt(now, claim.failureCount); - await repo.upstreams.finalizeModelsRefreshFailure(record.id, generation, `claim-${delays.length}`, { message: 'failure', at: now }, claim.failureCount + 1, blockedUntil); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'forced', now: now + 1, staleClaimedBefore: now - 899_999, force: true, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: delays.length + 1 }); - await repo.upstreams.finalizeModelsRefreshSuccess(record.id, generation, 'forced', { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [] }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'after-success', now: now + 2, staleClaimedBefore: now - 899_998, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: `claim-${delays.length}`, error: { message: 'failure', at: now }, previousFailureCount: claim.failureCount, failedAt: now }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'forced', now: now + 1, staleClaimedBefore: now - 899_999, bypassBackoff: true, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: delays.length + 1 }); + await repo.upstreams.finalizeModelsRefreshSuccess({ id: record.id, generation, token: 'forced', cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [] } }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'after-success', now: now + 2, staleClaimedBefore: now - 899_998, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); }); test('recovers abandoned claims and fences tokens, timestamps, and config', async () => { @@ -70,39 +69,104 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.save(record); const now = 1_800_000_000_000; - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'abandoned', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'replacement', now: now + 900_001, staleClaimedBefore: now + 1, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - await repo.upstreams.finalizeModelsRefreshSuccess(record.id, generation, 'abandoned', { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 900_001, models: [] }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'racer', now: now + 900_002, staleClaimedBefore: now + 2, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'replacement' }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'abandoned', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'replacement', now: now + 900_001, staleClaimedBefore: now + 1, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await repo.upstreams.finalizeModelsRefreshSuccess({ id: record.id, generation, token: 'abandoned', cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 900_001, models: [] } }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'racer', now: now + 900_002, staleClaimedBefore: now + 2, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'replacement' }); const next = { ...record, config: { tenant: 'next' } }; - await repo.upstreams.saveClearingModelsCache(next); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'old-config', now: now + 900_003, staleClaimedBefore: now + 3, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'generation-mismatch' }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: { updatedAt: next.updatedAt, config: next.config }, token: 'current', now: now + 900_003, staleClaimedBefore: now + 3, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await repo.upstreams.replaceForModels({ previous: record, upstream: next, cachePolicy: 'clear' }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'old-config', now: now + 900_003, staleClaimedBefore: now + 3, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'generation-mismatch' }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(next), token: 'current', now: now + 900_003, staleClaimedBefore: now + 3, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); const newer = { ...next, updatedAt: '2026-08-01T00:01:00.000Z' }; - await repo.upstreams.saveClearingModelsCache(newer); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: { updatedAt: next.updatedAt, config: next.config }, token: 'old-time', now: now + 900_004, staleClaimedBefore: now + 4, force: false, observedActiveToken: null })).resolves.toEqual({ kind: 'generation-mismatch' }); + await repo.upstreams.replaceForModels({ previous: next, upstream: newer, cachePolicy: 'clear' }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(next), token: 'old-time', now: now + 900_004, staleClaimedBefore: now + 4, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'generation-mismatch' }); }); test('metadata saves preserve backoff while invalidating an active owner', async () => { const repo = await createRepo(); await repo.upstreams.save(record); const now = 1_800_000_000_000; - const claim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'failed', now, staleClaimedBefore: now - 900_000, force: false, observedActiveToken: null }); + const claim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'failed', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); - await repo.upstreams.finalizeModelsRefreshFailure(record.id, generation, 'failed', { message: 'failure', at: now }, 1, modelsRefreshRetryAt(now, 0)); + await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: 'failed', error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); const next = { ...record, name: 'Renamed', updatedAt: '2026-08-01T00:01:00.000Z' }; - await repo.upstreams.save(next); + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next, cachePolicy: 'preserve' })).resolves.toBe(true); await expect(repo.upstreams.claimModelsRefresh({ id: record.id, - generation: { updatedAt: next.updatedAt, config: next.config }, + generation: modelsCacheGeneration(next), token: 'next-generation', now: now + 1, staleClaimedBefore: now - 899_999, - force: false, + bypassBackoff: false, observedActiveToken: null, })).resolves.toEqual({ kind: 'backoff' }); }); + + test('operator credential changes preserve the snapshot while resetting refresh cooldown', async () => { + const repo = await createRepo(); + await repo.upstreams.save(record); + const now = 1_800_000_000_000; + const claim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'failed', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); + if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); + await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: 'failed', error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); + + const next = { ...record, state: { credential: 'rotated' }, updatedAt: '2026-08-01T00:01:00.000Z' }; + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next, cachePolicy: 'reset-refresh' })).resolves.toBe(true); + expect((await repo.upstreams.getById(record.id))?.modelsCache?.lastError?.message).toBe('failure'); + await expect(repo.upstreams.claimModelsRefresh({ + id: record.id, + generation: modelsCacheGeneration(next), + token: 'new-credential', + now: now + 1, + staleClaimedBefore: now - 899_999, + bypassBackoff: false, + observedActiveToken: null, + })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + }); + + test('provider-managed credential state can rotate without invalidating its own owner', async () => { + const repo = await createRepo(); + await repo.upstreams.save(record); + const claim = await repo.upstreams.claimModelsRefresh({ + id: record.id, + generation, + token: 'state-owner', + now: 1_800_000_000_000, + staleClaimedBefore: 1_799_999_100_000, + bypassBackoff: false, + observedActiveToken: null, + }); + expect(claim.kind).toBe('claimed'); + await repo.upstreams.saveState(record.id, () => ({ credential: 'rotated' })); + + await expect(repo.upstreams.finalizeModelsRefreshSuccess({ + id: record.id, + generation, + token: 'state-owner', + cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_800_000_000_001, models: [] }, + })).resolves.toBe(true); + await expect(repo.upstreams.claimModelsRefresh({ + id: record.id, + generation, + token: 'stale-generation', + now: 1_800_000_000_002, + staleClaimedBefore: 1_799_999_100_002, + bypassBackoff: false, + observedActiveToken: null, + })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + }); + + test('catalog-aware replacement rejects a stale control-plane writer', async () => { + const repo = await createRepo(); + await repo.upstreams.save(record); + const winner = { ...record, name: 'Winner', updatedAt: '2026-08-01T00:01:00.000Z' }; + const stale = { ...record, name: 'Stale', updatedAt: '2026-08-01T00:02:00.000Z' }; + + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: winner, cachePolicy: 'preserve' })).resolves.toBe(true); + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: stale, cachePolicy: 'clear' })).resolves.toBe(false); + expect((await repo.upstreams.getById(record.id))?.name).toBe('Winner'); + }); }); diff --git a/packages/gateway/__tests__/repo/proxy-fallback-list_test.ts b/packages/gateway/__tests__/repo/proxy-fallback-list_test.ts index fe3240fa5..53837cfe2 100644 --- a/packages/gateway/__tests__/repo/proxy-fallback-list_test.ts +++ b/packages/gateway/__tests__/repo/proxy-fallback-list_test.ts @@ -13,12 +13,14 @@ describe('isDirectFallbackId', () => { describe('entryMatchesColo', () => { it('treats missing colos as "active in all colos"', () => { expect(entryMatchesColo({ id: 'a' }, 'HKG')).toBe(true); + expect(entryMatchesColo({ id: 'a' }, null)).toBe(true); }); it('matches by exact case (CF returns uppercase, normalize already upper-cased the whitelist)', () => { expect(entryMatchesColo({ id: 'a', colos: ['HKG', 'NRT'] }, 'HKG')).toBe(true); expect(entryMatchesColo({ id: 'a', colos: ['HKG', 'NRT'] }, 'NRT')).toBe(true); expect(entryMatchesColo({ id: 'a', colos: ['HKG', 'NRT'] }, 'LAX')).toBe(false); + expect(entryMatchesColo({ id: 'a', colos: ['HKG', 'NRT'] }, null)).toBe(false); // The contract is that callers feed already-normalised values; we don't // re-normalise here so a lower-case `currentColo` (a hypothetical bug // upstream) is intentionally a miss rather than a silent recovery. diff --git a/packages/gateway/__tests__/repo/sql_test.ts b/packages/gateway/__tests__/repo/sql_test.ts index 81cba9397..d7915dffa 100644 --- a/packages/gateway/__tests__/repo/sql_test.ts +++ b/packages/gateway/__tests__/repo/sql_test.ts @@ -3,6 +3,7 @@ import { test } from 'vitest'; import { seedModelsCache, seedModelsCacheError } from './models-cache-fixture.ts'; import { createSqliteTestDb } from './test-sqlite.ts'; import { MODEL_CATALOG_REVISION } from '../../src/data-plane/providers/models-cache.ts'; +import { modelsCacheGeneration } from '../../src/repo/models-cache-contract.ts'; import { SqlRepo, UPSTREAM_STATE_WRITE_ATTEMPTS } from '../../src/repo/sql.ts'; import type { SqlDatabase, SqlPreparedStatement } from '@floway-dev/platform'; import type { UpstreamRecord } from '@floway-dev/provider'; @@ -28,7 +29,7 @@ const baseRecord = (overrides: Partial = {}): UpstreamRecord => hue: 210, ...overrides, }); -const generationFor = (record: UpstreamRecord) => ({ updatedAt: record.updatedAt, config: record.config }); +const generationFor = modelsCacheGeneration; const ownValue = (value: unknown, key: string): unknown => { if (value === null || typeof value !== 'object' || !Object.hasOwn(value, key)) { @@ -153,7 +154,7 @@ test('SQL upstream repo persists an immediately-stale empty catalog on first fai }); }); -test('SQL upstream repo saveClearingModelsCache updates the row and removes the cached catalog atomically', async () => { +test('SQL upstream repo catalog-aware replacement can clear the cached catalog atomically', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { @@ -166,7 +167,7 @@ test('SQL upstream repo saveClearingModelsCache updates the row and removes the name: 'New identity', config: { accounts: [{ email: 'new@example.com', chatgptAccountId: 'new-account', chatgptUserId: 'new-user', planType: 'plus' }] }, }); - await repo.saveClearingModelsCache(newIdentity); + await repo.replaceForModels({ previous: baseRecord(), upstream: newIdentity, cachePolicy: 'clear' }); const stored = await repo.getById('up_test'); assertEquals(stored?.name, 'New identity'); @@ -221,6 +222,18 @@ test('SQL upstream repo save leaves an existing cached catalog alone', async () assertEquals(record?.modelsCache?.models.map(model => model.id), ['cached-model']); }); +test('SQL rejects malformed persisted model refresh state', async () => { + const db = await createSqliteTestDb(); + const repo = new SqlRepo(db).upstreams; + await repo.save(baseRecord()); + + await assertRejects( + () => db.prepare('UPDATE upstreams SET models_refresh_json = ? WHERE id = ?') + .bind(JSON.stringify({ failCount: 0, retryAt: 0, claimToken: 'owner', claimedAt: null }), 'up_test') + .run(), + ); +}); + test('SQL upstream repo round-trips state_json on save/list/getById', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; const original = baseRecord(); diff --git a/packages/gateway/__tests__/scheduled/models-refresh_test.ts b/packages/gateway/__tests__/scheduled/models-refresh_test.ts index 66027c87a..ad42589c8 100644 --- a/packages/gateway/__tests__/scheduled/models-refresh_test.ts +++ b/packages/gateway/__tests__/scheduled/models-refresh_test.ts @@ -2,7 +2,7 @@ import { expect, test, vi } from 'vitest'; import { initRepo } from '../../src/repo/index.ts'; import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; -import { refreshModelsCaches } from '../../src/scheduled/models-refresh.ts'; +import { scheduleModelsCacheRefreshes } from '../../src/scheduled/models-refresh.ts'; import { InMemoryRepo } from '../repo/memory.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; import { withMockedFetch } from '@floway-dev/test-utils'; @@ -46,7 +46,7 @@ test('scheduled maintenance submits enabled refreshes without waiting for model }, async () => { const background: Promise[] = []; - await refreshModelsCaches('SCHEDULED', promise => { background.push(promise); }); + await scheduleModelsCacheRefreshes('TEST', promise => { background.push(promise); }); expect(background).toHaveLength(1); await vi.waitFor(() => expect(requested).toEqual(['enabled.example.com'])); @@ -62,3 +62,50 @@ test('scheduled maintenance submits enabled refreshes without waiting for model }, ); }); + +test('one malformed upstream does not prevent later refreshes from being scheduled', async () => { + const repo = new InMemoryRepo(); + initRepo(repo); + await repo.upstreams.save({ ...custom('malformed', true), config: null }); + await repo.upstreams.save(custom('healthy', true)); + const background: Promise[] = []; + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + + try { + await withMockedFetch( + () => Response.json({ data: [{ id: 'healthy-model' }] }), + async () => { + await scheduleModelsCacheRefreshes('TEST', promise => { background.push(promise); }); + expect(background).toHaveLength(1); + await background[0]; + }, + ); + } finally { + error.mockRestore(); + } + + expect((await repo.upstreams.getById('healthy'))?.modelsCache?.models).toMatchObject([{ id: 'healthy-model' }]); +}); + +test('locationless scheduled events skip colo-scoped-only egress policies', async () => { + const repo = new InMemoryRepo(); + initRepo(repo); + await repo.upstreams.save({ ...custom('scoped', true), proxyFallbackList: [{ id: 'direct_fetch', colos: ['HKG'] }] }); + await repo.upstreams.save(custom('global', true)); + const background: Promise[] = []; + const requested: string[] = []; + + await withMockedFetch( + request => { + requested.push(new URL(request.url).hostname); + return Response.json({ data: [{ id: 'global-model' }] }); + }, + async () => { + await scheduleModelsCacheRefreshes(null, promise => { background.push(promise); }); + await Promise.all(background); + }, + ); + + expect(requested).toEqual(['global.example.com']); + expect((await repo.upstreams.getById('scoped'))?.modelsCache).toBeNull(); +}); diff --git a/packages/gateway/__tests__/scheduled_test.ts b/packages/gateway/__tests__/scheduled_test.ts index 0e56554f4..252795c40 100644 --- a/packages/gateway/__tests__/scheduled_test.ts +++ b/packages/gateway/__tests__/scheduled_test.ts @@ -18,7 +18,7 @@ test('scheduled maintenance isolates the shared expiration driver from later col const error = vi.spyOn(console, 'error').mockImplementation(() => {}); try { - await runScheduledMaintenance(); + await runScheduledMaintenance('TEST', () => {}); } finally { error.mockRestore(); } @@ -38,7 +38,7 @@ test('scheduled maintenance collects exact spilled files after expiration work', vi.spyOn(repo.spilledFiles, 'claimCollectible').mockResolvedValue([key]); vi.spyOn(repo.spilledFiles, 'acknowledge').mockResolvedValue(1); - await runScheduledMaintenance(); + await runScheduledMaintenance('TEST', () => {}); expect(await files.get(key)).toBeNull(); }); @@ -55,7 +55,7 @@ test('scheduled maintenance does not collect spilled files before expiration wor }); const collect = vi.spyOn(repo.spilledFiles, 'claimCollectible').mockResolvedValue([]); - const maintenance = runScheduledMaintenance(); + const maintenance = runScheduledMaintenance('TEST', () => {}); await vi.waitFor(() => expect(releaseExpiration).not.toBeNull()); expect(collect).not.toHaveBeenCalled(); releaseExpiration!(); diff --git a/packages/gateway/migrations/0078_upstream_models_refresh.sql b/packages/gateway/migrations/0078_upstream_models_refresh.sql index 5719349e8..b69a4b17e 100644 --- a/packages/gateway/migrations/0078_upstream_models_refresh.sql +++ b/packages/gateway/migrations/0078_upstream_models_refresh.sql @@ -1,5 +1,25 @@ --- Refresh coordination stays on the upstream row beside the catalog it --- protects. Stale catalog reads need one atomic claim before upstream I/O; --- keeping that claim inline avoids restoring the serial side-table read that --- migration 0072 removed from every catalog access. -ALTER TABLE upstreams ADD COLUMN models_refresh_json TEXT NULL; +-- Refresh ownership and the catalog it protects share one row so claims and +-- catalog publication can be fenced by a single atomic update. +ALTER TABLE upstreams ADD COLUMN models_refresh_json TEXT NULL CHECK ( + models_refresh_json IS NULL OR coalesce(( + json_valid(models_refresh_json) = 1 + AND json_type(models_refresh_json, '$.failCount') IN ('integer', 'real') + AND json_extract(models_refresh_json, '$.failCount') >= 0 + AND json_extract(models_refresh_json, '$.failCount') = CAST(json_extract(models_refresh_json, '$.failCount') AS INTEGER) + AND json_type(models_refresh_json, '$.retryAt') IN ('integer', 'real') + AND json_extract(models_refresh_json, '$.retryAt') >= 0 + AND json_extract(models_refresh_json, '$.retryAt') = CAST(json_extract(models_refresh_json, '$.retryAt') AS INTEGER) + AND ( + ( + json_type(models_refresh_json, '$.claimToken') = 'null' + AND json_type(models_refresh_json, '$.claimedAt') = 'null' + ) OR ( + json_type(models_refresh_json, '$.claimToken') = 'text' + AND length(json_extract(models_refresh_json, '$.claimToken')) > 0 + AND json_type(models_refresh_json, '$.claimedAt') IN ('integer', 'real') + AND json_extract(models_refresh_json, '$.claimedAt') >= 0 + AND json_extract(models_refresh_json, '$.claimedAt') = CAST(json_extract(models_refresh_json, '$.claimedAt') AS INTEGER) + ) + ) + ), 0) = 1 +); diff --git a/packages/gateway/src/control-plane/data-transfer/routes.ts b/packages/gateway/src/control-plane/data-transfer/routes.ts index dbe717b83..8f5e4c084 100644 --- a/packages/gateway/src/control-plane/data-transfer/routes.ts +++ b/packages/gateway/src/control-plane/data-transfer/routes.ts @@ -17,8 +17,7 @@ import { getRepo } from '../../repo/index.ts'; import { DIRECT_FALLBACK_IDS } from '../../repo/proxy-fallback-list.ts'; import type { ApiKey, PerformanceTelemetryRecord, UsageRecord, User, WebSearchUsageRecord } from '../../repo/types.ts'; import { type exportQuery, type importBody } from '../schemas.ts'; -import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; -import { warmModelsCache } from '../shared/warm-models-cache.ts'; +import { saveAndWarmUpstreamsForModels } from '../shared/save-upstream-for-models.ts'; import { type FullSerializedUpstreamRecord, upstreamRecordToFullJson } from '../upstreams/serialize.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; @@ -190,8 +189,10 @@ export const importData = async (c: CtxWithJson) => { } for (const record of usage) await repo.usage.set(record); for (const record of searchUsage) await repo.webSearchUsage.set(record); - for (const upstream of upstreams) await saveUpstreamForModels(await repo.upstreams.getById(upstream.id), upstream); - await Promise.all(upstreams.map(upstream => warmModelsCache(upstream, c))); + await saveAndWarmUpstreamsForModels(await Promise.all(upstreams.map(async next => ({ + previous: await repo.upstreams.getById(next.id), + next, + }))), c); for (const record of performance) await repo.performance.set(record); await repo.webSearchConfig.save(searchConfig); diff --git a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts index 1ac5dcc5d..a8da1f382 100644 --- a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts +++ b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts @@ -1,15 +1,61 @@ -import { modelsFetchIdentity } from '../../data-plane/providers/registry.ts'; +import type { Context } from 'hono'; + +import { warmUpstreamModels } from '../../data-plane/providers/models-cache.ts'; +import { modelsCatalogIdentity, createProvider } from '../../data-plane/providers/registry.ts'; +import { createPerRequestFetcher } from '../../dial/per-request.ts'; import { getRepo } from '../../repo/index.ts'; -import type { UpstreamRecord } from '@floway-dev/provider'; +import { modelsOperatorRefreshIdentity } from '../../repo/models-cache-contract.ts'; +import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; +import type { UpstreamModelsCache, UpstreamRecord } from '@floway-dev/provider'; +import { logInfo } from '@floway-dev/provider-claude-code'; + +export interface UpstreamModelsChange { + previous: UpstreamRecord | null; + next: UpstreamRecord; +} -export const saveUpstreamForModels = async ( - previous: UpstreamRecord | null, - next: UpstreamRecord, -): Promise => { +const errorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error); + +const saveUpstreamForModels = async ({ previous, next }: UpstreamModelsChange): Promise => { const upstreams = getRepo().upstreams; - if (previous !== null && modelsFetchIdentity(previous) === modelsFetchIdentity(next)) { + if (previous === null) { await upstreams.save(next); - } else { - await upstreams.saveClearingModelsCache(next); + return; } + let cachePolicy: 'preserve' | 'reset-refresh' | 'clear'; + if (modelsCatalogIdentity(previous) !== modelsCatalogIdentity(next)) cachePolicy = 'clear'; + else if (modelsOperatorRefreshIdentity(previous) !== modelsOperatorRefreshIdentity(next)) cachePolicy = 'reset-refresh'; + else cachePolicy = 'preserve'; + const saved = await upstreams.replaceForModels({ previous, upstream: next, cachePolicy }); + if (!saved) throw new Error(`Upstream ${next.id} changed concurrently`); +}; + +export const saveAndWarmUpstreamsForModels = async ( + changes: readonly UpstreamModelsChange[], + c: Context, +): Promise> => { + for (const change of changes) await saveUpstreamForModels(change); + if (changes.length === 0) return new Map(); + + const records = changes.map(change => change.next); + const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw), records); + const entries = await Promise.all(records.map(async record => { + try { + await warmUpstreamModels(createProvider(record), fetcherForUpstream(record.id)); + } catch (error) { + logInfo('warm_models_cache_failed', { upstream_id: record.id, error: errorMessage(error) }); + } + const cache = (await getRepo().upstreams.getById(record.id))?.modelsCache ?? null; + return [record.id, cache] as const; + })); + return new Map(entries); +}; + +export const saveAndWarmUpstreamForModels = async ( + change: UpstreamModelsChange, + c: Context, +): Promise => { + const result = (await saveAndWarmUpstreamsForModels([change], c)).get(change.next.id); + if (result === undefined) throw new Error(`Missing models cache result for ${change.next.id}`); + return result; }; diff --git a/packages/gateway/src/control-plane/shared/warm-models-cache.ts b/packages/gateway/src/control-plane/shared/warm-models-cache.ts deleted file mode 100644 index aa29774fe..000000000 --- a/packages/gateway/src/control-plane/shared/warm-models-cache.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { Context } from 'hono'; - -import { warmUpstreamModels } from '../../data-plane/providers/models-cache.ts'; -import { createProvider } from '../../data-plane/providers/registry.ts'; -import { createPerRequestFetcher } from '../../dial/per-request.ts'; -import { getRepo } from '../../repo/index.ts'; -import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; -import type { UpstreamModelsCache, UpstreamRecord } from '@floway-dev/provider'; -import { logInfo } from '@floway-dev/provider-claude-code'; - -const errorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error); - -// Wait synchronously for an eligible refresh or an active owner. A persisted -// cooldown suppresses a new attempt and leaves the current snapshot in place. -// Refresh failures persist in `lastError`; errors escaping the cache layer are -// internal and remain observable without aborting the control-plane write. -// -// Returns what the row holds afterwards so the caller can answer with the -// freshness this warm produced rather than the snapshot it read before saving. -// A persisted cold failure is an empty error-bearing cache; null means the row -// disappeared or its generation was superseded before readback. -export const warmModelsCache = async (record: UpstreamRecord, c: Context): Promise => { - const provider = createProvider(record); - const fetcher = (await createPerRequestFetcher(getRuntimeLocation(c.req.raw)))(record.id); - try { - await warmUpstreamModels(provider, fetcher); - } catch (error) { - logInfo('warm_models_cache_failed', { upstream_id: record.id, error: errorMessage(error) }); - } - // Read back rather than reconstructing: on the failure path the row keeps - // whatever catalog it already had, annotated with this attempt's error. - return (await getRepo().upstreams.getById(record.id))?.modelsCache ?? null; -}; diff --git a/packages/gateway/src/control-plane/upstreams/claude-code.ts b/packages/gateway/src/control-plane/upstreams/claude-code.ts index c62cb0cb7..6195bd63a 100644 --- a/packages/gateway/src/control-plane/upstreams/claude-code.ts +++ b/packages/gateway/src/control-plane/upstreams/claude-code.ts @@ -5,8 +5,7 @@ import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { claudeCodeOAuthAuthorizeUrlBody, claudeCodeOAuthExchangeBody, claudeCodeOAuthRefreshBody, claudeCodeProbeBody, claudeCodeSetupTokenAuthorizeUrlBody, claudeCodeSetupTokenExchangeBody } from '../schemas.ts'; -import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; -import { warmModelsCache } from '../shared/warm-models-cache.ts'; +import { saveAndWarmUpstreamForModels } from '../shared/save-upstream-for-models.ts'; import type { Fetcher, UpstreamRecord } from '@floway-dev/provider'; import { type ClaudeCodeAccountCredential, @@ -79,8 +78,7 @@ export const claudeCodeOAuthExchange = async (c: CtxWithJson }; }; -// Unified model catalog fetch for both draft preview and saved-record -// refresh. Always live-fetches on the control plane; when -// record.id !== '' the request also warms/refreshes the SWR cache via -// `fetchUpstreamModelsCached` so a subsequent data-plane call picks up -// the fresh catalog. Custom's response stays the raw upstream row shape -// (dashboard translates through the draft's endpoints); every other -// kind returns UpstreamModelConfig-shaped rows. +// Unified model catalog fetch for draft previews and saved records. A request +// matching the saved fetch inputs atomically publishes its result to the +// persisted snapshot; an unsaved draft is fetched without touching that row. +// Custom keeps the raw upstream response shape for the dashboard; every other +// provider returns its ProviderModel projection. export const listModels = async (c: CtxWithJson) => { const { record } = c.req.valid('json'); if (!isValidProviderKind(record.kind)) { @@ -51,7 +50,6 @@ export const listModels = async (c: CtxWithJson) => { const persisted = record.id === '' ? null : await getRepo().upstreams.getById(record.id); if (record.id !== '' && persisted === null) return c.json({ error: 'Upstream not found' }, 404); - const scheduler = backgroundSchedulerFromContext(c); const now = new Date().toISOString(); const synthRecord: UpstreamRecord = { id: record.id || 'draft', @@ -73,9 +71,9 @@ export const listModels = async (c: CtxWithJson) => { // never carries a cached catalog. modelsCache: null, }; - const cacheGeneration = persisted === null - ? { updatedAt: synthRecord.updatedAt, config: synthRecord.config } - : { updatedAt: persisted.updatedAt, config: persisted.config }; + const canRefreshPersistedCache = persisted !== null + && modelsCatalogIdentity(persisted) === modelsCatalogIdentity(synthRecord) + && serializeStoredConfig(persisted.proxyFallbackList) === serializeStoredConfig(synthRecord.proxyFallbackList); let fetcher: Fetcher; try { @@ -91,19 +89,14 @@ export const listModels = async (c: CtxWithJson) => { try { if (kind === 'custom') { const assertedConfig = assertCustomUpstreamRecord(synthRecord).config; - const provider = createProvider(synthRecord, cacheGeneration); + const provider = createProvider(synthRecord, persisted === null ? undefined : modelsCacheGeneration(persisted)); let result: Awaited> | undefined; - if (record.id === '') { + if (!canRefreshPersistedCache) { result = await fetchCustomModels(assertedConfig, fetcher); } else { - await fetchUpstreamModelsCached(provider, { - scheduler, - fetcher, - force: true, - loadProvidedModels: async () => { - result = await fetchCustomModels(assertedConfig, fetcher); - return projectCustomModels(synthRecord, result); - }, + await fetchUpstreamModels(provider, fetcher, async () => { + result = await fetchCustomModels(assertedConfig, fetcher); + return projectCustomModels(synthRecord, result); }); // A concurrent refresh may already own the cache's in-flight slot, in // which case our raw-shape loader was not invoked. The dashboard still @@ -113,13 +106,10 @@ export const listModels = async (c: CtxWithJson) => { } return c.json({ kind, data: result.data }); } - // Copilot / codex / claude-code / azure / ollama — use the provider factory. - // Force through the SWR cache when the record is persisted so the - // side-effect refresh keeps the data-plane cache in step; otherwise - // live-fetch without any caching. - const provider = createProvider(synthRecord, cacheGeneration); - const models = record.id !== '' - ? await fetchUpstreamModelsCached(provider, { scheduler, fetcher, force: true }) + // Copilot / codex / claude-code / azure / ollama use the provider factory. + const provider = createProvider(synthRecord, persisted === null ? undefined : modelsCacheGeneration(persisted)); + const models = canRefreshPersistedCache + ? await fetchUpstreamModels(provider, fetcher) : await provider.instance.getProvidedModels(fetcher); return c.json({ kind, data: models.map(reshapeModelForDashboard) }); } catch (e) { diff --git a/packages/gateway/src/control-plane/upstreams/routes.ts b/packages/gateway/src/control-plane/upstreams/routes.ts index ea7f5aaaf..f598f22a5 100644 --- a/packages/gateway/src/control-plane/upstreams/routes.ts +++ b/packages/gateway/src/control-plane/upstreams/routes.ts @@ -11,9 +11,8 @@ import { isDirectFallbackId, normalizeProxyFallbackList } from '../../repo/proxy import { shortId } from '../../shared/short-id.ts'; import type { createUpstreamBody, updateUpstreamBody } from '../schemas.ts'; import { isRecord } from '../shared/field-validators.ts'; -import { saveUpstreamForModels } from '../shared/save-upstream-for-models.ts'; +import { saveAndWarmUpstreamForModels } from '../shared/save-upstream-for-models.ts'; import { nextSortOrder } from '../shared/sort-order.ts'; -import { warmModelsCache } from '../shared/warm-models-cache.ts'; import { normalizeModelPrefix, ALL_PROVIDER_KINDS, @@ -279,10 +278,9 @@ export const createUpstream = async (c: CtxWithJson) } const record = { ...upstream, config: config.value }; - await getRepo().upstreams.save(record); // Answer with the catalog status this warm produced, not the one the record // was built with — the dashboard re-seeds its draft from this body. - const modelsCache = await warmModelsCache(record, c); + const modelsCache = await saveAndWarmUpstreamForModels({ previous: null, next: record }, c); return c.json(await serializeForResponse({ ...record, modelsCache }, knownProxyIds), 201); }; @@ -336,8 +334,7 @@ export const updateUpstream = async (c: CtxWithJson - fetchUpstreamModelsCached(instance, { + const fetchOne = (instance: GatewayProvider) => { + const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler, fetcher: fetcherForUpstream(instance.upstreamId), - }).then(models => ({ instance, models, lastError: instance.modelsCache?.lastError ?? null })); + }); + return { instance, models: snapshot.models, lastError: snapshot.lastError }; + }; - const settled = await Promise.allSettled(providers.map(fetchOne)); + const settled = await Promise.allSettled(providers.map(async provider => fetchOne(provider))); for (const [index, result] of settled.entries()) { if (result.status === 'rejected') { diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index a6a7d0539..106ed9a38 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -1,30 +1,28 @@ import type { GatewayProvider } from './registry.ts'; import { getRepo } from '../../repo/index.ts'; -import { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; -import { MODELS_REFRESH_CLAIM_LEASE_MS, modelsRefreshRetryAt } from '../../repo/models-refresh-contract.ts'; -import { serializeStoredConfig } from '../../repo/upstream-json.ts'; +import { MODEL_CATALOG_REVISION, modelsFetchIdentity } from '../../repo/models-cache-contract.ts'; +import { MODELS_REFRESH_CLAIM_LEASE_MS } from '../../repo/models-refresh-contract.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; -import type { Fetcher, ProviderModel } from '@floway-dev/provider'; +import type { Fetcher, ProviderModel, UpstreamModelsCache } from '@floway-dev/provider'; // Soft-fresh rows need no refresh. Every older row remains usable forever; // access only triggers a background attempt guarded by the persisted refresh // claim/backoff state. const SOFT_MS = 10 * 60 * 1000; const ACTIVE_REFRESH_POLL_MS = 100; +const ACTIVE_REFRESH_POLL_CAP_MS = 1_000; +const ACTIVE_REFRESH_WAIT_MS = 60_000; export { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; -export interface ModelsCacheFetchOptions { +interface ModelsSnapshotReadOptions { scheduler: BackgroundScheduler; fetcher: Fetcher; - // The upstream editor's explicit Fetch Models action is the sole caller of - // this option. It waits for an actual fetch and bypasses refresh backoff. - force?: boolean; - // Some control-plane callers also need the upstream's raw catalog shape. - // Their loader projects that already-fetched response into the exact - // ProviderModel catalog the provider would otherwise return, avoiding a - // second upstream request while keeping cache writes in this module. - loadProvidedModels?: () => Promise; +} + +interface ModelsSnapshot { + readonly models: readonly ProviderModel[]; + readonly lastError: UpstreamModelsCache['lastError']; } // L1: per-isolate in-flight memoization. Callers join only when both their @@ -32,10 +30,10 @@ export interface ModelsCacheFetchOptions { // and superseded rows remain isolated. Not a TTL cache — the entry is removed // when the promise settles. The conditional delete defends against a stale // removal racing a later replacement. -type RefreshMode = 'fetch' | 'warm' | 'trigger'; +type RefreshIntent = 'explicit' | 'warm' | 'background'; interface InFlightRefresh { - kind: 'fetch' | 'wait'; + kind: 'refresh' | 'owner-wait'; promise: Promise; } @@ -71,17 +69,17 @@ const runFetch = async ( loadProvidedModels?: () => Promise, ): Promise => [...await (loadProvidedModels?.() ?? instance.instance.getProvidedModels(fetcher))]; -const runClaimedFetch = async ( +const runClaimedRefresh = async ( instance: GatewayProvider, fetcher: Fetcher, - mode: RefreshMode, + intent: RefreshIntent, loadProvidedModels?: () => Promise, - initialObservedActiveToken: string | null = null, ): Promise => { const repo = getRepo(); const token = crypto.randomUUID(); - let observedActiveToken = initialObservedActiveToken; - let claimed: Extract>, { kind: 'claimed' }>; + let observedActiveToken: string | null = null; + let pollMs = ACTIVE_REFRESH_POLL_MS; + const waitDeadline = Date.now() + ACTIVE_REFRESH_WAIT_MS; while (true) { const now = Date.now(); const outcome = await repo.upstreams.claimModelsRefresh({ @@ -90,74 +88,78 @@ const runClaimedFetch = async ( token, now, staleClaimedBefore: now - MODELS_REFRESH_CLAIM_LEASE_MS, - force: mode === 'fetch', + bypassBackoff: intent === 'explicit', observedActiveToken, }); - if (outcome.kind === 'claimed') { - claimed = outcome; - break; - } - if (mode !== 'warm' || outcome.kind === 'backoff' || outcome.kind === 'generation-mismatch') return null; + if (outcome.kind === 'backoff' || outcome.kind === 'generation-mismatch') return null; if (outcome.kind === 'completed') { const current = await repo.upstreams.getById(instance.upstreamId); if (current !== null && current.updatedAt === instance.modelsCacheGeneration.updatedAt - && serializeStoredConfig(current.config) === serializeStoredConfig(instance.modelsCacheGeneration.config)) instance.modelsCache = current.modelsCache; - return null; + && modelsFetchIdentity(current) === instance.modelsCacheGeneration.fetchIdentity) instance.modelsCache = current.modelsCache; + if (intent === 'explicit' && instance.modelsCache?.lastError !== null && instance.modelsCache?.lastError !== undefined) { + throw new Error(instance.modelsCache.lastError.message); + } + return instance.modelsCache?.models ?? []; + } + if (outcome.kind === 'active') { + if (intent === 'background') return null; + if (now >= waitDeadline) throw new Error(`Timed out waiting for models refresh owner for ${instance.upstreamId}`); + observedActiveToken = outcome.token; + await new Promise(resolve => setTimeout(resolve, pollMs)); + pollMs = Math.min(pollMs * 2, ACTIVE_REFRESH_POLL_CAP_MS); + continue; } - observedActiveToken = outcome.token; - await new Promise(resolve => setTimeout(resolve, ACTIVE_REFRESH_POLL_MS)); - } - let models: ProviderModel[]; - try { - models = await runFetch(instance, fetcher, loadProvidedModels); - } catch (error) { - const failureCount = claimed.failureCount + 1; - const now = Date.now(); - const lastError = { message: errorMessage(error), at: now }; + let models: ProviderModel[]; try { - const finalized = await repo.upstreams.finalizeModelsRefreshFailure( - instance.upstreamId, - instance.modelsCacheGeneration, - token, - lastError, - failureCount, - modelsRefreshRetryAt(now, claimed.failureCount), - ); + models = await runFetch(instance, fetcher, loadProvidedModels); + } catch (error) { + const failedAt = Date.now(); + const lastError = { message: errorMessage(error), at: failedAt }; + let finalized: boolean; + try { + finalized = await repo.upstreams.finalizeModelsRefreshFailure({ + id: instance.upstreamId, + generation: instance.modelsCacheGeneration, + token, + error: lastError, + previousFailureCount: outcome.failureCount, + failedAt, + }); + } catch (backoffError) { + throw new AggregateError([error, backoffError], errorMessage(error)); + } if (finalized) { if (instance.modelsCache) instance.modelsCache.lastError = lastError; else instance.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError }; + throw error; } - if (!finalized && mode === 'warm') { - const winner = await runClaimedFetch(instance, fetcher, 'warm', loadProvidedModels, token); - return winner ?? instance.modelsCache?.models ?? []; - } - } catch (backoffError) { - throw new AggregateError([error, backoffError], errorMessage(error)); + if (intent === 'background') throw error; + observedActiveToken = token; + continue; } - throw error; - } - const entry = { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models, lastError: null }; - const finalized = await repo.upstreams.finalizeModelsRefreshSuccess( - instance.upstreamId, - instance.modelsCacheGeneration, - token, - entry, - ); - // The instance is reused across alias targets in one request, so publish the - // finalized snapshot locally as well as durably. - if (finalized) instance.modelsCache = entry; - else if (mode === 'warm') { - const winner = await runClaimedFetch(instance, fetcher, 'warm', loadProvidedModels, token); - return winner ?? instance.modelsCache?.models ?? []; + const entry = { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models, lastError: null }; + const finalized = await repo.upstreams.finalizeModelsRefreshSuccess({ + id: instance.upstreamId, + generation: instance.modelsCacheGeneration, + token, + cache: entry, + }); + if (finalized) { + // The instance is reused across alias targets in one request, so publish + // the finalized snapshot locally as well as durably. + instance.modelsCache = entry; + return models; + } + if (intent === 'background') return models; + observedActiveToken = token; } - return models; }; const inFlightKey = (instance: GatewayProvider): string => { const generation = instance.modelsCacheGeneration; - return `${instance.upstreamId}\0${instance.modelsFetchIdentity}\0${generation.updatedAt}\0${serializeStoredConfig(generation.config)}`; + return `${instance.upstreamId}\0${generation.updatedAt}\0${generation.fetchIdentity}`; }; export const fetchUpstreamModels = async ( @@ -166,64 +168,60 @@ export const fetchUpstreamModels = async ( loadProvidedModels?: () => Promise, ): Promise => { const key = inFlightKey(instance); - const existing = inFlight.get(key); - if (existing?.kind === 'fetch') { - const joined = await existing.promise; - if (joined !== null) return joined; - if (inFlight.get(key) === existing) inFlight.delete(key); + while (true) { + const existing = inFlight.get(key); + if (existing?.kind === 'refresh') { + const joined = await existing.promise; + if (joined !== null) return joined; + if (inFlight.get(key) === existing) inFlight.delete(key); + continue; + } + const models = await startInFlight(key, 'refresh', () => runClaimedRefresh(instance, fetcher, 'explicit', loadProvidedModels)); + if (models === null) throw new Error(`Failed to acquire models refresh for ${instance.upstreamId}`); + return models; } - - const models = await startInFlight(key, 'fetch', () => runClaimedFetch(instance, fetcher, 'fetch', loadProvidedModels)); - if (models === null) throw new Error(`Failed to force-claim models refresh for ${instance.upstreamId}`); - return models; }; export const warmUpstreamModels = async ( instance: GatewayProvider, fetcher: Fetcher, - loadProvidedModels?: () => Promise, ): Promise => { const key = inFlightKey(instance); const existing = inFlight.get(key); if (existing) { const joined = await existing.promise; if (joined !== null) return joined; + if (existing.kind === 'owner-wait') return instance.modelsCache?.models ?? []; if (inFlight.get(key) === existing) inFlight.delete(key); } - const models = await memoInFlight(key, 'wait', () => runClaimedFetch(instance, fetcher, 'warm', loadProvidedModels)); + const models = await memoInFlight(key, 'owner-wait', () => runClaimedRefresh(instance, fetcher, 'warm')); return models ?? instance.modelsCache?.models ?? []; }; -export const triggerUpstreamModelsFetch = ( +export const scheduleUpstreamModelsRefresh = ( instance: GatewayProvider, scheduler: BackgroundScheduler, fetcher: Fetcher, - loadProvidedModels?: () => Promise, ): void => { const key = inFlightKey(instance); - scheduler(memoInFlight(key, 'fetch', () => runClaimedFetch(instance, fetcher, 'trigger', loadProvidedModels)).then(() => {})); + scheduler(memoInFlight(key, 'refresh', () => runClaimedRefresh(instance, fetcher, 'background')).then(() => {})); }; -export const fetchUpstreamModelsCached = async ( +export const readUpstreamModelsSnapshotAndScheduleRefresh = ( instance: GatewayProvider, - opts: ModelsCacheFetchOptions, -): Promise => { - const { scheduler, fetcher, force, loadProvidedModels } = opts; + opts: ModelsSnapshotReadOptions, +): ModelsSnapshot => { + const { scheduler, fetcher } = opts; const now = Date.now(); - - if (force) { - return await fetchUpstreamModels(instance, fetcher, loadProvidedModels); - } - - // Read off the instance rather than queried: the row that produced this - // provider carried its catalog, so the SWR check costs nothing. const cached = instance.modelsCache?.revision === MODEL_CATALOG_REVISION ? instance.modelsCache : null; + const snapshot = { + models: cached?.models ?? [], + lastError: cached?.lastError ?? null, + }; - if (cached && now - cached.fetchedAt < SOFT_MS) return cached.models; - - triggerUpstreamModelsFetch(instance, scheduler, fetcher, loadProvidedModels); - return cached?.models ?? []; + if (!cached || now - cached.fetchedAt >= SOFT_MS) scheduleUpstreamModelsRefresh(instance, scheduler, fetcher); + return snapshot; }; // Test-only: drop the L1 map so a test's setup is independent of any diff --git a/packages/gateway/src/data-plane/providers/registry.ts b/packages/gateway/src/data-plane/providers/registry.ts index e68a4291e..ce263eadf 100644 --- a/packages/gateway/src/data-plane/providers/registry.ts +++ b/packages/gateway/src/data-plane/providers/registry.ts @@ -1,4 +1,5 @@ import { getRepo } from '../../repo/index.ts'; +import { modelsCacheGeneration } from '../../repo/models-cache-contract.ts'; import type { ModelsCacheGeneration } from '../../repo/types.ts'; import { serializeStoredConfig } from '../../repo/upstream-json.ts'; import type { FlagDefaults, Provider, ProviderModule, UpstreamProviderKind, UpstreamRecord } from '@floway-dev/provider'; @@ -20,21 +21,19 @@ const providersByKind: Record = { export type GatewayProvider = Provider & { readonly modelsCacheGeneration: ModelsCacheGeneration; - readonly modelsFetchIdentity: string; }; -export const modelsFetchIdentity = (record: Pick): string => - serializeStoredConfig({ kind: record.kind, config: record.config, state: record.state, proxyFallbackList: record.proxyFallbackList }); +export const modelsCatalogIdentity = (record: UpstreamRecord): string => + serializeStoredConfig({ kind: record.kind, identity: providersByKind[record.kind].modelCatalogIdentity(record) }); export const createProvider = ( record: UpstreamRecord, - cacheGeneration: ModelsCacheGeneration = { updatedAt: record.updatedAt, config: record.config }, + cacheGeneration: ModelsCacheGeneration = modelsCacheGeneration(record), ): GatewayProvider => { const provider = providersByKind[record.kind].create(record); return { ...provider, modelsCacheGeneration: cacheGeneration, - modelsFetchIdentity: modelsFetchIdentity(record), }; }; diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index 1990c2280..b13beb692 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -1,7 +1,7 @@ import { isEqual, uniqWith } from 'es-toolkit'; import { internalModelFromProviderModel } from './catalog.ts'; -import { fetchUpstreamModelsCached } from './models-cache.ts'; +import { readUpstreamModelsSnapshotAndScheduleRefresh } from './models-cache.ts'; import { listModelProviders, type GatewayProvider } from './registry.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; import { getRepo } from '../../repo/index.ts'; @@ -46,19 +46,19 @@ const enumerateOneUpstreamCandidates = async ( } if (lookupIds.length === 0) return { candidates: [], sawAnyId: false, modelsError: false }; - const providedModels = await fetchUpstreamModelsCached(provider, { scheduler, fetcher }); + const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(provider, { scheduler, fetcher }); const disabled = new Set(provider.disabledPublicModelIds); const candidates: ModelCandidate[] = []; let sawAnyId = false; for (const lookupId of lookupIds) { - const match = providedModels.find(m => m.id === lookupId && !disabled.has(m.id)); + const match = snapshot.models.find(m => m.id === lookupId && !disabled.has(m.id)); if (!match) continue; sawAnyId = true; if (match.kind === kind) { candidates.push({ provider, model: internalModelFromProviderModel(match, provider.upstreamId), fetcher }); } } - return { candidates, sawAnyId, modelsError: provider.modelsCache?.lastError != null }; + return { candidates, sawAnyId, modelsError: snapshot.lastError !== null }; }; // Walk every visible upstream in configured order. Snapshot reads never wait @@ -209,8 +209,6 @@ export const enumerateModelCandidates = async ({ upstreamIds: readonly string[] | null; model: string; kind: ModelKind; - // Threaded into `enumerateRealModelCandidates` so stale snapshot access can - // submit or join a separate background refresh trigger. scheduler: BackgroundScheduler; // Runtime location tag for this request — see GatewayCtx.runtimeLocation. // Threaded into the per-request fetcher so colo-scoped fallback entries diff --git a/packages/gateway/src/data-plane/shared/listing/addressable.ts b/packages/gateway/src/data-plane/shared/listing/addressable.ts index 7194aa0ad..60748c889 100644 --- a/packages/gateway/src/data-plane/shared/listing/addressable.ts +++ b/packages/gateway/src/data-plane/shared/listing/addressable.ts @@ -12,7 +12,7 @@ // a second registry round trip. import { compareModelIds, getModelsFromProviders } from '../../providers/catalog.ts'; -import { fetchUpstreamModelsCached } from '../../providers/models-cache.ts'; +import { readUpstreamModelsSnapshotAndScheduleRefresh } from '../../providers/models-cache.ts'; import { listModelProviders } from '../../providers/registry.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; import { isAbortError, type Fetcher, type InternalModel, type Provider, type UpstreamRecord } from '@floway-dev/provider'; @@ -87,7 +87,7 @@ export const enumerateAddressableModelIds = async ( const addressableOnly = cfg !== null ? cfg.addressable.filter(form => !cfg.listed.includes(form)) : []; if (cfg === null || addressableOnly.length === 0) return [] as AddressableIdEntry[]; - const upstreamModels = await fetchUpstreamModelsCached(provider, { scheduler, fetcher: fetcherForUpstream(provider.upstreamId) }); + const upstreamModels = readUpstreamModelsSnapshotAndScheduleRefresh(provider, { scheduler, fetcher: fetcherForUpstream(provider.upstreamId) }).models; const disabled = new Set(provider.disabledPublicModelIds); const out: AddressableIdEntry[] = []; diff --git a/packages/gateway/src/dial/fetcher.ts b/packages/gateway/src/dial/fetcher.ts index c7772e3db..3aecf8b3a 100644 --- a/packages/gateway/src/dial/fetcher.ts +++ b/packages/gateway/src/dial/fetcher.ts @@ -13,8 +13,9 @@ interface CreateFetcherInput { fallbackList: ProxyFallbackEntry[]; proxyById: Map; // Location tag the request landed in, used to apply each entry's optional - // `colos` whitelist via `entryMatchesColo`. See `getRuntimeLocation`. - runtimeLocation: string; + // `colos` whitelist via `entryMatchesColo`. Null is reserved for platform + // events that expose no colo. See `getRuntimeLocation`. + runtimeLocation: string | null; // Injected so the fetcher stays runtime-agnostic — the composition root // chooses the concrete dial/fetch implementations. runProxied: ( diff --git a/packages/gateway/src/dial/per-request.ts b/packages/gateway/src/dial/per-request.ts index a5b218d4d..dc9b60d35 100644 --- a/packages/gateway/src/dial/per-request.ts +++ b/packages/gateway/src/dial/per-request.ts @@ -15,7 +15,7 @@ import { runDirectConnectRequest, runProxiedRequest } from '@floway-dev/proxy'; // `preFetchedUpstreams` lets a caller reuse a list it already loaded on // this request instead of paying a second `upstreams.list()` round-trip. export const createPerRequestFetcher = async ( - runtimeLocation: string, + runtimeLocation: string | null, preFetchedUpstreams?: readonly UpstreamRecord[], ): Promise<(upstreamId: string) => Fetcher> => { const repo = getRepo(); diff --git a/packages/gateway/src/repo/models-cache-contract.ts b/packages/gateway/src/repo/models-cache-contract.ts index 5d3e19906..215fb15f9 100644 --- a/packages/gateway/src/repo/models-cache-contract.ts +++ b/packages/gateway/src/repo/models-cache-contract.ts @@ -1,4 +1,36 @@ +import type { ModelsCacheGeneration } from './types.ts'; +import { serializeStoredConfig } from './upstream-json.ts'; +import type { UpstreamRecord } from '@floway-dev/provider'; + // Persisted ProviderModel rows contain code-derived metadata as well as the // upstream response. Increment this whenever that derived catalog contract or // its serialization changes so older rows become cold across deployments. export const MODEL_CATALOG_REVISION = 5; + +// Fetch ownership survives provider-managed state writes such as token +// rotation, but changes whenever static request inputs or egress policy do. +export const modelsFetchIdentity = ( + record: Pick, +): string => serializeStoredConfig({ + kind: record.kind, + config: record.config, + proxyFallbackList: record.proxyFallbackList, +}); + +// Control-plane replacements reset cooldown when any operator-owned fetch +// input changes, including credential state updated by OAuth/import flows. +export const modelsOperatorRefreshIdentity = ( + record: Pick, +): string => serializeStoredConfig({ + kind: record.kind, + config: record.config, + state: record.state ?? null, + proxyFallbackList: record.proxyFallbackList, +}); + +export const modelsCacheGeneration = ( + record: Pick, +): ModelsCacheGeneration => ({ + updatedAt: record.updatedAt, + fetchIdentity: modelsFetchIdentity(record), +}); diff --git a/packages/gateway/src/repo/proxy-fallback-list.ts b/packages/gateway/src/repo/proxy-fallback-list.ts index 04bdd2f91..b47518c4e 100644 --- a/packages/gateway/src/repo/proxy-fallback-list.ts +++ b/packages/gateway/src/repo/proxy-fallback-list.ts @@ -42,9 +42,13 @@ const normalizeColos = (colos: readonly string[] | undefined): string[] | undefi return out.length === 0 ? undefined : out; }; -// True when the entry is active under the request's current colo. `colos` +// True when the entry is active under the request's current colo. Null means +// the runtime cannot identify its colo, so only unscoped entries match. `colos` // is either absent (all colos) or non-empty — the wire schema rejects an // empty array and `normalizeProxyFallbackList` strips one before storage, so // we don't defend the "empty means all colos" interpretation here. -export const entryMatchesColo = (entry: ProxyFallbackEntry, currentColo: string): boolean => - entry.colos === undefined || entry.colos.includes(currentColo); +export const entryMatchesColo = (entry: ProxyFallbackEntry, currentColo: string | null): boolean => + entry.colos === undefined || (currentColo !== null && entry.colos.includes(currentColo)); + +export const hasLocationIndependentEgress = (entries: readonly ProxyFallbackEntry[]): boolean => + entries.length === 0 || entries.some(entry => entry.colos === undefined); diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index 285117228..9454e961b 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -2,7 +2,8 @@ import { normalizeDisabledPublicModelIds } from './disabled-public-models.ts'; import { SqlExpirationSweepsRepo } from './expiration-sweeps-sql.ts'; import { normalizeFlagOverrides } from './flag-overrides.ts'; import { decodeAliasTargets, decodeAnnouncedMetadata, encodeAliasTargets, encodeAnnouncedMetadata } from './model-alias-codecs.ts'; -import { MODEL_CATALOG_REVISION } from './models-cache-contract.ts'; +import { MODEL_CATALOG_REVISION, modelsFetchIdentity } from './models-cache-contract.ts'; +import { modelsRefreshRetryAt } from './models-refresh-contract.ts'; import { querySqlPerformanceOverview } from './performance-overview-sql.ts'; import { normalizeProxyFallbackList } from './proxy-fallback-list.ts'; import { SqlResponsesItemsRepo, SqlResponsesSnapshotsRepo } from './responses-state-sql.ts'; @@ -22,6 +23,8 @@ import type { ModelsCacheGeneration, ModelsRefreshClaimInput, ModelsRefreshClaimResult, + ModelsRefreshFailureInput, + ModelsRefreshSuccessInput, ModelAliasesRepo, ModelAliasRecord, PerformanceBucketRow, @@ -892,17 +895,81 @@ class SqlUpstreamRepo implements UpstreamRepo { } save(upstream: UpstreamRecord): Promise { - return this.saveRecord(upstream, false); - } - - saveClearingModelsCache(upstream: UpstreamRecord): Promise { - return this.saveRecord(upstream, true); + return this.saveRecord(upstream); + } + + async replaceForModels(input: { + previous: UpstreamRecord; + upstream: UpstreamRecord; + cachePolicy: 'preserve' | 'reset-refresh' | 'clear'; + }): Promise { + const { previous, upstream, cachePolicy } = input; + const modelsRefreshUpdate = cachePolicy === 'preserve' + ? "CASE WHEN models_refresh_json IS NULL THEN NULL ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) END" + : 'NULL'; + const modelsCacheUpdate = cachePolicy === 'clear' ? ', models_cache_json = NULL' : ''; + const result = await this.db + .prepare( + `UPDATE upstreams SET + provider = ?, + name = ?, + enabled = ?, + sort_order = ?, + updated_at = ?, + config_json = ?, + state_json = ?, + flag_overrides = ?, + disabled_public_model_ids = ?, + proxy_fallback_list_json = ?, + model_prefix_json = ?, + hue = ?, + models_refresh_json = ${modelsRefreshUpdate}${modelsCacheUpdate} + WHERE id = ? + AND provider = ? + AND name = ? + AND enabled = ? + AND sort_order = ? + AND updated_at = ? + AND config_json = ? + AND state_json IS ? + AND flag_overrides = ? + AND disabled_public_model_ids = ? + AND proxy_fallback_list_json = ? + AND model_prefix_json IS ? + AND hue = ?`, + ) + .bind( + upstream.kind, + upstream.name, + upstream.enabled ? 1 : 0, + upstream.sortOrder, + upstream.updatedAt, + serializeStoredConfig(upstream.config), + serializeStoredState(upstream.state), + JSON.stringify(normalizeFlagOverrides(upstream.flagOverrides)), + JSON.stringify(normalizeDisabledPublicModelIds(upstream.disabledPublicModelIds)), + JSON.stringify(normalizeProxyFallbackList(upstream.proxyFallbackList)), + upstream.modelPrefix === null ? null : JSON.stringify(upstream.modelPrefix), + upstream.hue, + upstream.id, + previous.kind, + previous.name, + previous.enabled ? 1 : 0, + previous.sortOrder, + previous.updatedAt, + serializeStoredConfig(previous.config), + serializeStoredState(previous.state), + JSON.stringify(normalizeFlagOverrides(previous.flagOverrides)), + JSON.stringify(normalizeDisabledPublicModelIds(previous.disabledPublicModelIds)), + JSON.stringify(normalizeProxyFallbackList(previous.proxyFallbackList)), + previous.modelPrefix === null ? null : JSON.stringify(previous.modelPrefix), + previous.hue, + ) + .run(); + return (result.meta.changes ?? 0) > 0; } - private async saveRecord(upstream: UpstreamRecord, clearModelsCache: boolean): Promise { - const modelsRefreshUpdate = clearModelsCache - ? 'NULL' - : "CASE WHEN models_refresh_json IS NULL THEN NULL ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) END"; + private async saveRecord(upstream: UpstreamRecord): Promise { // created_at is deliberately not in the ON CONFLICT update list: the row's first INSERT // wins, and re-saves preserve that timestamp regardless of what the caller passes. await this.db @@ -921,7 +988,7 @@ class SqlUpstreamRepo implements UpstreamRepo { proxy_fallback_list_json = excluded.proxy_fallback_list_json, model_prefix_json = excluded.model_prefix_json, hue = excluded.hue, - models_refresh_json = ${modelsRefreshUpdate}${clearModelsCache ? ', models_cache_json = NULL' : ''}`, + models_refresh_json = CASE WHEN models_refresh_json IS NULL THEN NULL ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) END`, ) .bind( upstream.id, @@ -951,19 +1018,23 @@ class SqlUpstreamRepo implements UpstreamRepo { await this.db.prepare('DELETE FROM upstreams').run(); } - async finalizeModelsRefreshSuccess(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise { - const rawConfig = await this.modelsCacheWriteConfig(id, generation); - if (rawConfig === null) return false; + async finalizeModelsRefreshSuccess(input: ModelsRefreshSuccessInput): Promise { + const { id, generation, token, cache } = input; + const fence = await this.modelsRefreshWriteFence(id, generation); + if (fence === null) return false; const result = await this.db - .prepare("UPDATE upstreams SET models_cache_json = ?, models_refresh_json = NULL WHERE id = ? AND updated_at = ? AND config_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") - .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, generation.updatedAt, rawConfig, token) + .prepare("UPDATE upstreams SET models_cache_json = ?, models_refresh_json = NULL WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") + .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, token) .run(); return (result.meta.changes ?? 0) > 0; } - async finalizeModelsRefreshFailure(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable, failureCount: number, retryAt: number): Promise { - const rawConfig = await this.modelsCacheWriteConfig(id, generation); - if (rawConfig === null) return false; + async finalizeModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise { + const { id, generation, token, error, previousFailureCount, failedAt } = input; + const failureCount = previousFailureCount + 1; + const retryAt = modelsRefreshRetryAt(failedAt, previousFailureCount); + const fence = await this.modelsRefreshWriteFence(id, generation); + if (fence === null) return false; // A cold failure remains immediately stale while preserving the error for // the next request and dashboard read. const coldFailure = encodeUpstreamModelsCache({ revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }); @@ -972,17 +1043,17 @@ class SqlUpstreamRepo implements UpstreamRepo { `UPDATE upstreams SET models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END, models_refresh_json = json_object('failCount', ?, 'retryAt', ?, 'claimToken', NULL, 'claimedAt', NULL) - WHERE id = ? AND updated_at = ? AND config_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?`, + WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?`, ) - .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, generation.updatedAt, rawConfig, token) + .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, token) .run(); return (result.meta.changes ?? 0) > 0; } async claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { - const { id, generation, token, now, staleClaimedBefore, force, observedActiveToken } = input; - const rawConfig = await this.modelsCacheWriteConfig(id, generation); - if (rawConfig === null) return { kind: 'generation-mismatch' }; + const { id, generation, token, now, staleClaimedBefore, bypassBackoff, observedActiveToken } = input; + const fence = await this.modelsRefreshWriteFence(id, generation); + if (fence === null) return { kind: 'generation-mismatch' }; while (true) { const row = await this.db .prepare( @@ -993,19 +1064,16 @@ class SqlUpstreamRepo implements UpstreamRepo { 'claimToken', ?, 'claimedAt', ? ) - WHERE id = ? AND updated_at = ? AND config_json = ? AND ( - ? = 1 OR ( - ? IS NULL AND ( - models_refresh_json IS NULL - OR ( - coalesce(json_extract(models_refresh_json, '$.retryAt'), 0) <= ? - AND ( - json_extract(models_refresh_json, '$.claimToken') IS NULL - OR json_extract(models_refresh_json, '$.claimedAt') <= ? - ) - ) + WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ? AND ( + ? IS NULL AND ( + models_refresh_json IS NULL + OR ( + json_extract(models_refresh_json, '$.claimToken') IS NULL + AND (? = 1 OR coalesce(json_extract(models_refresh_json, '$.retryAt'), 0) <= ?) ) - ) OR ( + OR json_extract(models_refresh_json, '$.claimedAt') <= ? + ) + OR ( ? IS NOT NULL AND json_extract(models_refresh_json, '$.claimToken') = ? AND json_extract(models_refresh_json, '$.claimedAt') <= ? @@ -1013,7 +1081,7 @@ class SqlUpstreamRepo implements UpstreamRepo { ) RETURNING json_extract(models_refresh_json, '$.failCount') AS fail_count`, ) - .bind(token, now, id, generation.updatedAt, rawConfig, sqliteBoolean(force), observedActiveToken, now, staleClaimedBefore, observedActiveToken, observedActiveToken, staleClaimedBefore) + .bind(token, now, id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, observedActiveToken, sqliteBoolean(bypassBackoff), now, staleClaimedBefore, observedActiveToken, observedActiveToken, staleClaimedBefore) .first<{ fail_count: number }>(); if (row !== null) return { kind: 'claimed', failureCount: row.fail_count }; @@ -1023,9 +1091,9 @@ class SqlUpstreamRepo implements UpstreamRepo { json_extract(models_refresh_json, '$.retryAt') AS retry_at, json_extract(models_refresh_json, '$.claimToken') AS claim_token, json_extract(models_refresh_json, '$.claimedAt') AS claimed_at - FROM upstreams WHERE id = ? AND updated_at = ? AND config_json = ?`, + FROM upstreams WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ?`, ) - .bind(id, generation.updatedAt, rawConfig) + .bind(id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList) .first<{ models_refresh_json: string | null; retry_at: number | null; claim_token: string | null; claimed_at: number | null }>(); if (state === null) return { kind: 'generation-mismatch' }; if (state.models_refresh_json === null) { @@ -1033,19 +1101,28 @@ class SqlUpstreamRepo implements UpstreamRepo { continue; } if (state.claim_token !== null && state.claimed_at !== null && state.claimed_at > staleClaimedBefore) return { kind: 'active', token: state.claim_token }; - if (state.retry_at !== null && state.retry_at > now) return { kind: 'backoff' }; - if (observedActiveToken !== null) return { kind: 'completed' }; + if (observedActiveToken !== null && state.claim_token === null) return { kind: 'completed' }; + if (!bypassBackoff && state.retry_at !== null && state.retry_at > now) return { kind: 'backoff' }; } } - private async modelsCacheWriteConfig(id: string, generation: ModelsCacheGeneration): Promise { + private async modelsRefreshWriteFence(id: string, generation: ModelsCacheGeneration): Promise<{ + provider: string; + config: string; + proxyFallbackList: string; + } | null> { const row = await this.db - .prepare('SELECT updated_at, config_json FROM upstreams WHERE id = ?') + .prepare('SELECT updated_at, provider, config_json, proxy_fallback_list_json FROM upstreams WHERE id = ?') .bind(id) - .first<{ updated_at: string; config_json: string }>(); + .first<{ updated_at: string; provider: string; config_json: string; proxy_fallback_list_json: string }>(); if (row === null || row.updated_at !== generation.updatedAt) return null; - return serializeStoredConfig(JSON.parse(row.config_json)) === serializeStoredConfig(generation.config) - ? row.config_json + const identity = modelsFetchIdentity({ + kind: parseUpstreamKind(id, row.provider), + config: decodeUpstreamConfig(row.config_json, id), + proxyFallbackList: parseProxyFallbackList(id, row.proxy_fallback_list_json), + }); + return identity === generation.fetchIdentity + ? { provider: row.provider, config: row.config_json, proxyFallbackList: row.proxy_fallback_list_json } : null; } diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index c8567cb82..034e257f7 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -344,7 +344,11 @@ export interface UpstreamRepo { list(): Promise; getById(id: string): Promise; save(upstream: UpstreamRecord): Promise; - saveClearingModelsCache(upstream: UpstreamRecord): Promise; + replaceForModels(input: { + previous: UpstreamRecord; + upstream: UpstreamRecord; + cachePolicy: 'preserve' | 'reset-refresh' | 'clear'; + }): Promise; delete(id: string): Promise; deleteAll(): Promise; // Upstream state write with optimistic concurrency, used both by the @@ -357,8 +361,8 @@ export interface UpstreamRepo { // Catalog-cache writes are conditional on the row generation that started // the fetch. A superseded provider can finish serving its own request, but // cannot publish models or errors under newer credentials/configuration. - finalizeModelsRefreshSuccess(id: string, generation: ModelsCacheGeneration, token: string, cache: Omit): Promise; - finalizeModelsRefreshFailure(id: string, generation: ModelsCacheGeneration, token: string, error: NonNullable, failureCount: number, retryAt: number): Promise; + finalizeModelsRefreshSuccess(input: ModelsRefreshSuccessInput): Promise; + finalizeModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise; claimModelsRefresh(input: ModelsRefreshClaimInput): Promise; } @@ -368,10 +372,26 @@ export interface ModelsRefreshClaimInput { token: string; now: number; staleClaimedBefore: number; - force: boolean; + bypassBackoff: boolean; observedActiveToken: string | null; } +export interface ModelsRefreshSuccessInput { + id: string; + generation: ModelsCacheGeneration; + token: string; + cache: Omit; +} + +export interface ModelsRefreshFailureInput { + id: string; + generation: ModelsCacheGeneration; + token: string; + error: NonNullable; + previousFailureCount: number; + failedAt: number; +} + export interface ModelsRefreshClaim { kind: 'claimed'; failureCount: number; @@ -385,7 +405,7 @@ export type ModelsRefreshClaimResult = ModelsRefreshClaim export interface ModelsCacheGeneration { updatedAt: string; - config: unknown; + fetchIdentity: string; } export interface ProxyRecord { diff --git a/packages/gateway/src/scheduled.ts b/packages/gateway/src/scheduled.ts index f366bb6cd..483e5a6bf 100644 --- a/packages/gateway/src/scheduled.ts +++ b/packages/gateway/src/scheduled.ts @@ -1,5 +1,5 @@ import { sweepExpirations } from './scheduled/expiration-sweeps.ts'; -import { refreshModelsCaches } from './scheduled/models-refresh.ts'; +import { scheduleModelsCacheRefreshes } from './scheduled/models-refresh.ts'; import { collectSpilledFiles } from './scheduled/spilled-files.ts'; import { getImageCacheStore, type BackgroundScheduler } from '@floway-dev/platform'; @@ -13,13 +13,9 @@ const runSweep = async (name: string, fn: () => Promise): Promise { - promise.catch(error => console.error('[scheduled] background task failed', error)); -}; - export const runScheduledMaintenance = async ( - runtimeLocation = 'SCHEDULED', - backgroundScheduler: BackgroundScheduler = defaultBackgroundScheduler, + runtimeLocation: string | null, + backgroundScheduler: BackgroundScheduler, ): Promise => { const nowMs = Date.now(); const storageMaintenance = async (): Promise => { @@ -27,7 +23,7 @@ export const runScheduledMaintenance = async ( await runSweep('spilledFiles.collect', () => collectSpilledFiles(nowMs)); }; await Promise.all([ - runSweep('models.refresh', () => refreshModelsCaches(runtimeLocation, backgroundScheduler)), + runSweep('models.refresh', () => scheduleModelsCacheRefreshes(runtimeLocation, backgroundScheduler)), storageMaintenance(), runSweep('imageCacheStore.sweepExpired', () => getImageCacheStore().sweepExpired(nowMs)), ]); diff --git a/packages/gateway/src/scheduled/models-refresh.ts b/packages/gateway/src/scheduled/models-refresh.ts index c441f175b..40370d57d 100644 --- a/packages/gateway/src/scheduled/models-refresh.ts +++ b/packages/gateway/src/scheduled/models-refresh.ts @@ -1,17 +1,20 @@ -import { fetchUpstreamModelsCached } from '../data-plane/providers/models-cache.ts'; +import { scheduleUpstreamModelsRefresh } from '../data-plane/providers/models-cache.ts'; import { createProvider } from '../data-plane/providers/registry.ts'; import { createPerRequestFetcher } from '../dial/per-request.ts'; import { getRepo } from '../repo/index.ts'; +import { hasLocationIndependentEgress } from '../repo/proxy-fallback-list.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; -export const refreshModelsCaches = async (runtimeLocation: string, scheduler: BackgroundScheduler): Promise => { - const upstreams = (await getRepo().upstreams.list()).filter(upstream => upstream.enabled); +export const scheduleModelsCacheRefreshes = async (runtimeLocation: string | null, scheduler: BackgroundScheduler): Promise => { + const upstreams = (await getRepo().upstreams.list()).filter(upstream => + upstream.enabled && (runtimeLocation !== null || hasLocationIndependentEgress(upstream.proxyFallbackList))); const fetcherForUpstream = await createPerRequestFetcher(runtimeLocation, upstreams); for (const upstream of upstreams) { - await fetchUpstreamModelsCached(createProvider(upstream), { - scheduler, - fetcher: fetcherForUpstream(upstream.id), - }); + try { + scheduleUpstreamModelsRefresh(createProvider(upstream), scheduler, fetcherForUpstream(upstream.id)); + } catch (error) { + console.error(`[scheduled] models.refresh failed for ${upstream.id}`, error); + } } }; diff --git a/packages/provider-azure/src/index.ts b/packages/provider-azure/src/index.ts index 744129dbd..b8b526bad 100644 --- a/packages/provider-azure/src/index.ts +++ b/packages/provider-azure/src/index.ts @@ -1,9 +1,11 @@ +import { assertAzureUpstreamRecord } from './config.ts'; import { AZURE_DEFAULT_FLAGS } from './defaults.ts'; import { createAzureProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const azureProviderModule: ProviderModule = { create: createAzureProvider, + modelCatalogIdentity: record => assertAzureUpstreamRecord(record).config, defaultFlags: AZURE_DEFAULT_FLAGS, }; export { assertAzureUpstreamRecord, type AzureUpstreamConfig } from './config.ts'; diff --git a/packages/provider-claude-code/src/index.ts b/packages/provider-claude-code/src/index.ts index 13fe12d88..9d6a862ea 100644 --- a/packages/provider-claude-code/src/index.ts +++ b/packages/provider-claude-code/src/index.ts @@ -1,9 +1,14 @@ +import { assertClaudeCodeUpstreamRecord } from './config.ts'; import { CLAUDE_CODE_DEFAULT_FLAGS } from './defaults.ts'; import { createClaudeCodeProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const claudeCodeProviderModule: ProviderModule = { create: createClaudeCodeProvider, + modelCatalogIdentity: record => { + assertClaudeCodeUpstreamRecord(record); + return record.config; + }, defaultFlags: CLAUDE_CODE_DEFAULT_FLAGS, }; diff --git a/packages/provider-codex/src/index.ts b/packages/provider-codex/src/index.ts index d9e52b55d..9f9f29015 100644 --- a/packages/provider-codex/src/index.ts +++ b/packages/provider-codex/src/index.ts @@ -1,9 +1,14 @@ +import { assertCodexUpstreamRecord } from './config.ts'; import { CODEX_DEFAULT_FLAGS } from './defaults.ts'; import { createCodexProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const codexProviderModule: ProviderModule = { create: createCodexProvider, + modelCatalogIdentity: record => { + assertCodexUpstreamRecord(record); + return record.config; + }, defaultFlags: CODEX_DEFAULT_FLAGS, }; diff --git a/packages/provider-copilot/__tests__/provider_test.ts b/packages/provider-copilot/__tests__/provider_test.ts index 4e03b0f1e..0411ce1b9 100644 --- a/packages/provider-copilot/__tests__/provider_test.ts +++ b/packages/provider-copilot/__tests__/provider_test.ts @@ -887,7 +887,7 @@ test('Copilot provider throws "disappeared mid-request" when the upstream row va test('Copilot provider swallows a saveState throw so a transient persistence hiccup does not invalidate the fetched models', async () => { // Persistence is best-effort: the fetched models are the user-facing // payload, and a storage-level error on the write must not propagate out of - // getProvidedModels. Mirrors the gateway SWR layer's persistence policy. + // getProvidedModels. Mirrors the gateway's persisted catalog policy. const harness = await setupCopilotTest(); harness.overrideSaveState(() => Promise.reject(new Error('D1 hiccup'))); diff --git a/packages/provider-copilot/src/index.ts b/packages/provider-copilot/src/index.ts index c97adaedb..d508b90f1 100644 --- a/packages/provider-copilot/src/index.ts +++ b/packages/provider-copilot/src/index.ts @@ -1,9 +1,14 @@ +import { assertCopilotUpstreamRecord } from './config.ts'; import { COPILOT_DEFAULT_FLAGS } from './defaults.ts'; import { createCopilotProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const copilotProviderModule: ProviderModule = { create: createCopilotProvider, + modelCatalogIdentity: record => { + const upstream = assertCopilotUpstreamRecord(record); + return { githubHost: upstream.config.githubHost, userId: upstream.config.user.id }; + }, defaultFlags: COPILOT_DEFAULT_FLAGS, }; diff --git a/packages/provider-custom/src/index.ts b/packages/provider-custom/src/index.ts index 6c3fc8264..5a79c37e2 100644 --- a/packages/provider-custom/src/index.ts +++ b/packages/provider-custom/src/index.ts @@ -1,9 +1,11 @@ +import { assertCustomUpstreamRecord } from './config.ts'; import { CUSTOM_DEFAULT_FLAGS } from './defaults.ts'; import { createCustomProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const customProviderModule: ProviderModule = { create: createCustomProvider, + modelCatalogIdentity: record => assertCustomUpstreamRecord(record).config, defaultFlags: CUSTOM_DEFAULT_FLAGS, }; diff --git a/packages/provider-ollama/src/fetch-models.ts b/packages/provider-ollama/src/fetch-models.ts index 659c4f2bb..fd4a4109d 100644 --- a/packages/provider-ollama/src/fetch-models.ts +++ b/packages/provider-ollama/src/fetch-models.ts @@ -133,7 +133,7 @@ const fetchShowForTag = async ( export const fetchOllamaCatalog = async (config: OllamaUpstreamConfig, fetcher: Fetcher): Promise => { // /api/tags through the shared scaffold so network / non-2xx / shape errors // surface as ProviderModelsUnavailableError — same envelope every other - // provider's catalog fetch produces, which the control-plane and SWR cache + // provider's catalog fetch produces, which the control-plane and persisted cache // both branch on. const tags = await fetchUpstreamModels( () => ollamaFetchTags(config, { method: 'GET' }, { fetcher, wrapUpstreamCall: identityWrapUpstreamCall }), diff --git a/packages/provider-ollama/src/index.ts b/packages/provider-ollama/src/index.ts index 49dfb890b..ccdab62bc 100644 --- a/packages/provider-ollama/src/index.ts +++ b/packages/provider-ollama/src/index.ts @@ -1,9 +1,11 @@ +import { assertOllamaUpstreamRecord } from './config.ts'; import { OLLAMA_DEFAULT_FLAGS } from './defaults.ts'; import { createOllamaProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const ollamaProviderModule: ProviderModule = { create: createOllamaProvider, + modelCatalogIdentity: record => assertOllamaUpstreamRecord(record).config, defaultFlags: OLLAMA_DEFAULT_FLAGS, }; diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index 3b5164eab..3f6f34fdb 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -38,9 +38,8 @@ export interface Provider { // record so registry helpers — routing and listing — read it from the // instance instead of re-fetching the row. `null` keeps the bare-id behavior. modelPrefix: ModelPrefixConfig | null; - // The row's cached catalog, mirrored for the same reason: the SWR layer - // reads it from the instance instead of paying a second round trip that the - // row read already covered. + // The row's persisted catalog snapshot, mirrored so resolution does not pay + // a second round trip after the row has already been loaded. modelsCache: UpstreamModelsCache | null; instance: ProviderInstance; } @@ -167,6 +166,9 @@ export interface ProviderModule { // fetch) happens on demand inside the per-request methods on the // returned ProviderInstance. create: (record: UpstreamRecord) => Provider; + // Stable identity of the upstream account/catalog namespace. Each provider + // decides which of its configuration changes can preserve a snapshot. + modelCatalogIdentity: (record: UpstreamRecord) => unknown; // Exhaustive default map over every catalog flag id for a fresh // upstream of this kind; see each provider package's `defaults.ts`. defaultFlags: FlagDefaults; From fe6f666c26f87da9c9e17e8a6743e55b1deb5057 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 17:51:39 +0800 Subject: [PATCH 40/46] refactor(gateway): separate model snapshots from refresh coordination Keep snapshot capture and freshness in a small cache facade, with durable/L1 ownership isolated in a refresh coordinator. Explicit fetches join successful work but retry through their own transport after background failure.\n\nMake catalog-aware creates insert-only, merge unrelated provider-state races during metadata replacement, fence failure counts, total the stale-owner transition, and retry uncertain finalization. Provider modules now own normalized catalog and refresh identities.\n\nFilter egress by runtime location before proxy parsing and cover cold inference convergence, remote-owner recovery, locationless scheduling, stale writers, and cross-owner lease recovery. --- .../upstreams/copilot-device-login_test.ts | 4 +- .../chat/shared/target-picker_test.ts | 2 - .../data-plane/completions/http_test.ts | 41 +++- .../data-plane/providers/catalog_test.ts | 6 +- .../data-plane/providers/models-cache_test.ts | 54 +++-- .../data-plane/providers/resolution_test.ts | 24 +- .../shared/listing/addressable_test.ts | 8 +- packages/gateway/__tests__/repo/memory.ts | 16 +- .../__tests__/repo/models-refresh_test.ts | 44 ++++ .../scheduled/models-refresh_test.ts | 9 +- packages/gateway/__tests__/test-utils/app.ts | 4 +- .../shared/save-upstream-for-models.ts | 8 +- .../src/control-plane/upstreams/models.ts | 8 +- .../src/data-plane/providers/models-cache.ts | 220 +---------------- .../data-plane/providers/models-refresh.ts | 222 ++++++++++++++++++ .../src/data-plane/providers/registry.ts | 7 + packages/gateway/src/dial/per-request.ts | 7 +- .../gateway/src/repo/models-cache-contract.ts | 11 - packages/gateway/src/repo/sql.ts | 40 +++- packages/gateway/src/repo/types.ts | 1 + .../gateway/src/scheduled/models-refresh.ts | 2 +- packages/provider-azure/src/index.ts | 1 + packages/provider-claude-code/src/index.ts | 4 + packages/provider-codex/src/index.ts | 4 + packages/provider-copilot/src/index.ts | 4 + packages/provider-custom/src/index.ts | 1 + packages/provider-ollama/src/index.ts | 1 + packages/provider/src/provider.ts | 3 + 28 files changed, 475 insertions(+), 281 deletions(-) create mode 100644 packages/gateway/src/data-plane/providers/models-refresh.ts diff --git a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts index 0604f55a4..ce0777636 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts @@ -5,14 +5,14 @@ import { afterEach, expect, test, vi } from 'vitest'; // exchange and persistence. const modelsCacheMock = vi.hoisted<{ calls: number; error: Error | null; pending: Promise | null }>(() => ({ calls: 0, error: null, pending: null })); -vi.mock('../../../src/data-plane/providers/models-cache.ts', () => ({ +vi.mock('../../../src/data-plane/providers/models-refresh.ts', () => ({ warmUpstreamModels: async () => { modelsCacheMock.calls++; if (modelsCacheMock.pending) await modelsCacheMock.pending; if (modelsCacheMock.error) throw modelsCacheMock.error; return []; }, - clearInFlightForTesting: () => {}, + clearModelsRefreshesForTesting: () => {}, })); import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; diff --git a/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts index 82e6e466b..f69c4a467 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts @@ -7,8 +7,6 @@ import type { ModelEndpoints } from '@floway-dev/protocols/common'; import type { UpstreamRecord } from '@floway-dev/provider'; import { assertEquals } from '@floway-dev/test-utils'; -// Drains the separately scheduled snapshot refresh so a rejection surfaces in the runner -// instead of being swallowed. const testScheduler = (promise: Promise): void => { promise.catch(err => console.error('[background]', err)); }; diff --git a/packages/gateway/__tests__/data-plane/completions/http_test.ts b/packages/gateway/__tests__/data-plane/completions/http_test.ts index 207d9c750..4f040e8df 100644 --- a/packages/gateway/__tests__/data-plane/completions/http_test.ts +++ b/packages/gateway/__tests__/data-plane/completions/http_test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { tokenCountsFromUsage } from '../../../src/repo/usage-metrics.ts'; import { installDumpStubs } from '../../dump/test-fixtures.ts'; -import { buildCustomUpstreamRecord, flushAsyncWork, requestAppWithWarmModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; +import { buildCustomUpstreamRecord, flushAsyncWork, requestApp as requestAppCold, requestAppWithWarmModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -45,6 +45,45 @@ const completionStream = (): Response => { }); }; +test('/v1/completions cold resolution schedules the catalog and a later request dispatches', async () => { + const { apiKey, repo } = await setupAppTest(); + await registerCompletionsUpstream(repo); + let upstreamCalls = 0; + const request = { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ model: 'davinci-002', prompt: 'hello' }), + }; + + await withMockedFetch( + request => { + const url = new URL(request.url); + if (url.hostname !== 'passthrough.example.com' || url.pathname !== '/v1/completions') { + throw new Error(`Unhandled fetch ${request.url}`); + } + upstreamCalls++; + return jsonResponse({ + id: 'cmpl_resp', + object: 'text_completion', + created: 1, + model: 'davinci-002', + choices: [{ index: 0, text: ' world', finish_reason: 'stop' }], + usage: { prompt_tokens: 5, completion_tokens: 1, total_tokens: 6 }, + }); + }, + async () => { + const cold = await requestAppCold('/v1/completions', request); + assertEquals(cold.status, 404); + assertEquals(upstreamCalls, 0); + + await flushAsyncWork(); + const warm = await requestAppCold('/v1/completions', request); + assertEquals(warm.status, 200); + assertEquals(upstreamCalls, 1); + }, + ); +}); + test('/v1/completions non-streaming forwards body to upstream /v1/completions and records usage', async () => { const { apiKey, repo } = await setupAppTest(); await registerCompletionsUpstream(repo); diff --git a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts index 49396d899..b6dbaade0 100644 --- a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts @@ -1,7 +1,7 @@ import { describe, expect, test, vi } from 'vitest'; import { compareModelIds, getModelsFromProviders } from '../../../src/data-plane/providers/catalog.ts'; -import { clearInFlightForTesting } from '../../../src/data-plane/providers/models-cache.ts'; +import { clearModelsRefreshesForTesting } from '../../../src/data-plane/providers/models-refresh.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { buildCustomUpstreamRecord, copilotModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; @@ -254,7 +254,7 @@ test('disabledPublicModelIds hides models from the catalog and routing, per upst // directly observes concurrency without a wall-clock threshold that load can // satisfy or violate independently of execution order. test('catalog refresh triggers fan out per upstream in parallel', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); @@ -302,7 +302,7 @@ test('catalog refresh triggers fan out per upstream in parallel', async () => { // recorded against `sawSuccess === true`; the public catalog still includes // every successful upstream's models. test('catalog assembly: a rejected provider does not block other providers', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 0621b3b53..6b9b13780 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; -import { clearInFlightForTesting, fetchUpstreamModels, readUpstreamModelsSnapshotAndScheduleRefresh, MODEL_CATALOG_REVISION, warmUpstreamModels } from '../../../src/data-plane/providers/models-cache.ts'; +import { readUpstreamModelsSnapshotAndScheduleRefresh, MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; +import { clearModelsRefreshesForTesting, fetchUpstreamModels, warmUpstreamModels } from '../../../src/data-plane/providers/models-refresh.ts'; import type { GatewayProvider } from '../../../src/data-plane/providers/registry.ts'; import { initRepo } from '../../../src/repo/index.ts'; import { modelsFetchIdentity } from '../../../src/repo/models-cache-contract.ts'; @@ -89,7 +90,7 @@ const captureScheduled = () => { beforeEach(() => { vi.restoreAllMocks(); - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); }); describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { @@ -188,7 +189,7 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: firstScheduled.scheduler, fetcher: directFetcher }).models.map(model => model.id)).toEqual(['stale']); await expect(firstScheduled.promises[0]).rejects.toThrow('boom'); - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const secondScheduled = captureScheduled(); expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: secondScheduled.scheduler, fetcher: directFetcher }).models.map(model => model.id)).toEqual(['stale']); await expect(secondScheduled.promises[0]).resolves.toBeUndefined(); @@ -208,14 +209,14 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { await expect(firstScheduled.promises[0]).rejects.toThrow('boom'); expect(await storedCache(repo)).toMatchObject({ fetchedAt: 0, models: [], lastError: { message: 'boom' } }); - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); now += 59_999; const backedOff = captureScheduled(); expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: backedOff.scheduler, fetcher: directFetcher }).models).toEqual([]); await expect(backedOff.promises[0]).resolves.toBeUndefined(); expect(fetchFn).toHaveBeenCalledTimes(1); - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); now += 1; const retry = captureScheduled(); expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: retry.scheduler, fetcher: directFetcher }).models).toEqual([]); @@ -231,7 +232,7 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { const scheduled = captureScheduled(); readUpstreamModelsSnapshotAndScheduleRefresh(failing, { scheduler: scheduled.scheduler, fetcher: directFetcher }); await expect(scheduled.promises[0]).rejects.toThrow('boom'); - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const fetchFn = vi.fn(async () => [aModel('recovered')]); const cache = await storedCache(repo); @@ -290,6 +291,27 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { expect((await warming).map(model => model.id)).toEqual(['remote-model']); }); + test('explicit fetch retries after a remote owner records failure', async () => { + const repo = await setupRepo(); + const now = Date.now(); + await repo.upstreams.claimModelsRefresh({ id: UPSTREAM_ID, generation: CACHE_GENERATION, token: 'remote-owner', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); + const fetchFn = vi.fn(async () => [aModel('explicit-recovery-model')]); + const explicit = fetchUpstreamModels(stubInstance(fetchFn), directFetcher); + await new Promise(resolve => setTimeout(resolve, 20)); + + await repo.upstreams.finalizeModelsRefreshFailure({ + id: UPSTREAM_ID, + generation: CACHE_GENERATION, + token: 'remote-owner', + error: { message: 'remote failure', at: now + 1 }, + previousFailureCount: 0, + failedAt: now + 1, + }); + + await expect(explicit).resolves.toEqual([aModel('explicit-recovery-model')]); + expect(fetchFn).toHaveBeenCalledOnce(); + }); + test('explicit fetch joins a warm that already owns the durable refresh', async () => { await setupRepo(); let resolveWarm: ((models: ProviderModel[]) => void) | null = null; @@ -304,16 +326,16 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { expect(fetchFn).toHaveBeenCalledTimes(1); }); - test('an atomic success-finalize failure does not install upstream failure backoff', async () => { + test('a transient success-finalize error is retried without installing failure backoff', async () => { const repo = await setupRepo(); const finalizeFailure = vi.spyOn(repo.upstreams, 'finalizeModelsRefreshFailure'); - vi.spyOn(repo.upstreams, 'finalizeModelsRefreshSuccess').mockRejectedValueOnce(new Error('finalize failed')); + const finalizeSuccess = vi.spyOn(repo.upstreams, 'finalizeModelsRefreshSuccess').mockRejectedValueOnce(new Error('finalize failed')); const instance = stubInstance(async () => [aModel('published-model')]); - await expect(fetchUpstreamModels(instance, directFetcher)) - .rejects.toThrow('finalize failed'); + await expect(fetchUpstreamModels(instance, directFetcher)).resolves.toEqual([aModel('published-model')]); + expect(finalizeSuccess).toHaveBeenCalledTimes(2); expect(finalizeFailure).not.toHaveBeenCalled(); - expect(await storedCache(repo)).toBeNull(); + expect((await storedCache(repo))?.models).toEqual([aModel('published-model')]); }); test('a superseded generation neither joins nor overwrites the current catalog', async () => { @@ -363,7 +385,7 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['late-old-model']); }); - test('explicit fetch surfaces failure from an older background owner', async () => { + test('explicit fetch retries with its own transport after an older background owner fails', async () => { const repo = await setupRepo(); let rejectOld: ((error: Error) => void) | null = null; const oldFetch = vi.fn(() => new Promise((_resolve, reject) => { rejectOld = reject; })); @@ -374,13 +396,13 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { ); await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); - const explicitFetch = vi.fn(async () => [aModel('duplicate-explicit-model')]); + const explicitFetch = vi.fn(async () => [aModel('explicit-recovery-model')]); const explicit = fetchUpstreamModels(stubInstance(explicitFetch), directFetcher); rejectOld!(new Error('late old failure')); await expect(oldScheduled.promises[0]).rejects.toThrow('late old failure'); - await expect(explicit).rejects.toThrow('late old failure'); - expect(explicitFetch).not.toHaveBeenCalled(); - expect(await storedCache(repo)).toMatchObject({ models: [], lastError: { message: 'late old failure' } }); + await expect(explicit).resolves.toEqual([aModel('explicit-recovery-model')]); + expect(explicitFetch).toHaveBeenCalledOnce(); + expect(await storedCache(repo)).toMatchObject({ models: [{ id: 'explicit-recovery-model' }], lastError: null }); }); test('catalog revision mismatch is cold and refreshes without blocking', async () => { diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index 877e3781f..59843307b 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -1,6 +1,6 @@ import { describe, expect, test, vi } from 'vitest'; -import { clearInFlightForTesting, fetchUpstreamModels } from '../../../src/data-plane/providers/models-cache.ts'; +import { clearModelsRefreshesForTesting, fetchUpstreamModels } from '../../../src/data-plane/providers/models-refresh.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; import { enumerateModelCandidates, enumerateRealModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { buildCustomUpstreamRecord, copilotModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; @@ -352,7 +352,7 @@ test('a recorded refresh failure is irrelevant when the prefix policy cannot add // upstream's display name flows back via `failedUpstreams` while its empty // or last-known-good snapshot stays independent of the current request. test('enumerateModelCandidates: healthy upstream still resolves alongside a rejecting one, with failedUpstreams reported', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); @@ -404,7 +404,7 @@ test('enumerateModelCandidates: healthy upstream still resolves alongside a reje // attempt, so the resolver returns immediately rather than walking the // stripped form. test('enumerateModelCandidates does NOT trigger the dated-suffix retry on a wrong-kind sawAnyId match', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(buildCustomUpstreamRecord({ @@ -445,7 +445,7 @@ test('enumerateModelCandidates does NOT trigger the dated-suffix retry on a wron // failedUpstreams across the two retry attempts must dedupe: a single broken // upstream that rejects both walks reports its name once, not twice. test('enumerateModelCandidates deduplicates failedUpstreams across the dated-suffix retry attempts', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(buildCustomUpstreamRecord({ @@ -481,7 +481,7 @@ test('enumerateModelCandidates deduplicates failedUpstreams across the dated-suf }); test('an AbortError from background catalog refresh does not abort model resolution', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(buildCustomUpstreamRecord({ @@ -519,7 +519,7 @@ test('an AbortError from background catalog refresh does not abort model resolut // upstream fetch. The failure renderer surfaces this as a model-missing 404 // without re-deriving the empty-cap branch. test('enumerateModelCandidates returns the empty triple when the visible upstream list is empty', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); // A populated catalog is the case under test: the empty cap, not an empty @@ -577,7 +577,7 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }; test('flattens across targets in declaration order for first-available', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ @@ -605,7 +605,7 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }); test('shuffles the outer walk for random selection but keeps intra-target order', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ @@ -638,7 +638,7 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }); test('dedups (model, upstream, rules) when two targets hit the same binding with identical rules', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ @@ -665,7 +665,7 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }); test('keeps the first representative in its original position when duplicate bindings are interleaved', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ @@ -697,7 +697,7 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }); test('keeps two entries for the same (model, upstream) with distinct rules', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ @@ -723,7 +723,7 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }); test('falls through to a later target when an earlier one has no kind-matching binding', async () => { - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ diff --git a/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts b/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts index 4f9e92b1d..fa442b546 100644 --- a/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { clearInFlightForTesting } from '../../../../src/data-plane/providers/models-cache.ts'; +import { clearModelsRefreshesForTesting } from '../../../../src/data-plane/providers/models-refresh.ts'; import { enumerateAddressableModelIds } from '../../../../src/data-plane/shared/listing/addressable.ts'; import { buildCustomUpstreamRecord, setupAppTest, warmModelsForTest } from '../../../test-utils/app.ts'; import { directFetcher } from '@floway-dev/provider'; @@ -15,7 +15,7 @@ describe('enumerateAddressableModelIds', () => { const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(buildCustomUpstreamRecord()); - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); await withMockedFetch( request => { @@ -45,7 +45,7 @@ describe('enumerateAddressableModelIds', () => { // public id. modelPrefix: { prefix: 'cust/', addressable: ['unprefixed', 'prefixed'], listed: ['prefixed'] }, })); - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); await withMockedFetch( request => { @@ -71,7 +71,7 @@ describe('enumerateAddressableModelIds', () => { test('throws "no upstream configured" when the upstream cap is empty — surfacing the same hint /v1/models has always raised', async () => { const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); - clearInFlightForTesting(); + clearModelsRefreshesForTesting(); await expect(enumerateAddressableModelIds(null, () => directFetcher, noBackground)) .rejects.toThrow('No upstream provider configured'); diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index e787030e1..c1fd090eb 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -762,6 +762,12 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(); } + insertForModels(upstream: UpstreamRecord): Promise { + if (this.store.has(upstream.id)) return Promise.resolve(false); + this.store.set(upstream.id, cloneUpstreamRecord({ ...upstream, modelsCache: null })); + return Promise.resolve(true); + } + replaceForModels(input: { previous: UpstreamRecord; upstream: UpstreamRecord; @@ -769,10 +775,15 @@ class MemoryUpstreamRepo implements UpstreamRepo { }): Promise { const { previous, upstream, cachePolicy } = input; const existing = this.store.get(upstream.id); - if (existing === undefined || serializeStoredConfig({ ...existing, modelsCache: null }) !== serializeStoredConfig({ ...previous, modelsCache: null })) return Promise.resolve(false); + if (existing === undefined) return Promise.resolve(false); + const replaceState = serializeStoredState(previous.state) !== serializeStoredState(upstream.state); + const comparableExisting = { ...existing, modelsCache: null, state: replaceState ? existing.state : null }; + const comparablePrevious = { ...previous, modelsCache: null, state: replaceState ? previous.state : null }; + if (serializeStoredConfig(comparableExisting) !== serializeStoredConfig(comparablePrevious)) return Promise.resolve(false); const next = cloneUpstreamRecord({ ...upstream, createdAt: existing.createdAt, + state: replaceState ? upstream.state : existing.state, modelsCache: cachePolicy === 'clear' ? null : existing.modelsCache, }); this.store.set(upstream.id, next); @@ -823,7 +834,8 @@ class MemoryUpstreamRepo implements UpstreamRepo { const { id, generation, token, error, previousFailureCount, failedAt } = input; const failureCount = previousFailureCount + 1; const retryAt = modelsRefreshRetryAt(failedAt, previousFailureCount); - if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); + const refresh = this.modelsRefreshes.get(id); + if (refresh?.claimToken !== token || refresh.failCount !== previousFailureCount) return Promise.resolve(false); const existing = this.store.get(id); if (!existing || existing.updatedAt !== generation.updatedAt || modelsFetchIdentity(existing) !== generation.fetchIdentity) return Promise.resolve(false); if (existing.modelsCache) existing.modelsCache.lastError = error; diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index 4a84a1437..de54110ca 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -162,11 +162,55 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = test('catalog-aware replacement rejects a stale control-plane writer', async () => { const repo = await createRepo(); await repo.upstreams.save(record); + await repo.upstreams.saveState(record.id, () => ({ providerManaged: 'newer' })); const winner = { ...record, name: 'Winner', updatedAt: '2026-08-01T00:01:00.000Z' }; const stale = { ...record, name: 'Stale', updatedAt: '2026-08-01T00:02:00.000Z' }; await expect(repo.upstreams.replaceForModels({ previous: record, upstream: winner, cachePolicy: 'preserve' })).resolves.toBe(true); await expect(repo.upstreams.replaceForModels({ previous: record, upstream: stale, cachePolicy: 'clear' })).resolves.toBe(false); expect((await repo.upstreams.getById(record.id))?.name).toBe('Winner'); + expect((await repo.upstreams.getById(record.id))?.state).toEqual({ providerManaged: 'newer' }); + }); + + test('catalog-aware insertion never overwrites a concurrent winner', async () => { + const repo = await createRepo(); + await expect(repo.upstreams.insertForModels(record)).resolves.toBe(true); + await expect(repo.upstreams.insertForModels({ ...record, name: 'Loser' })).resolves.toBe(false); + expect((await repo.upstreams.getById(record.id))?.name).toBe(record.name); + }); + + test('a waiter can reclaim a replacement owner after its lease also expires', async () => { + const repo = await createRepo(); + await repo.upstreams.save(record); + const firstNow = 1_800_000_000_000; + await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'owner-a', now: firstNow, staleClaimedBefore: firstNow - 900_000, bypassBackoff: false, observedActiveToken: null }); + const secondNow = firstNow + 900_001; + await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'owner-b', now: secondNow, staleClaimedBefore: firstNow + 1, bypassBackoff: false, observedActiveToken: null }); + + await expect(repo.upstreams.claimModelsRefresh({ + id: record.id, + generation, + token: 'waiter', + now: secondNow + 900_001, + staleClaimedBefore: secondNow + 1, + bypassBackoff: false, + observedActiveToken: 'owner-a', + })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + }); + + test('failure finalization rejects a count not issued with the claim', async () => { + const repo = await createRepo(); + await repo.upstreams.save(record); + const now = 1_800_000_000_000; + await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'owner', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); + + await expect(repo.upstreams.finalizeModelsRefreshFailure({ + id: record.id, + generation, + token: 'owner', + error: { message: 'failure', at: now }, + previousFailureCount: 99, + failedAt: now, + })).resolves.toBe(false); }); }); diff --git a/packages/gateway/__tests__/scheduled/models-refresh_test.ts b/packages/gateway/__tests__/scheduled/models-refresh_test.ts index ad42589c8..828437910 100644 --- a/packages/gateway/__tests__/scheduled/models-refresh_test.ts +++ b/packages/gateway/__tests__/scheduled/models-refresh_test.ts @@ -90,8 +90,15 @@ test('one malformed upstream does not prevent later refreshes from being schedul test('locationless scheduled events skip colo-scoped-only egress policies', async () => { const repo = new InMemoryRepo(); initRepo(repo); + await repo.proxies.save({ id: 'bad-scoped', name: 'Bad scoped proxy', url: 'not a URL', dialTimeoutSeconds: null }); await repo.upstreams.save({ ...custom('scoped', true), proxyFallbackList: [{ id: 'direct_fetch', colos: ['HKG'] }] }); - await repo.upstreams.save(custom('global', true)); + await repo.upstreams.save({ + ...custom('global', true), + proxyFallbackList: [ + { id: 'bad-scoped', colos: ['HKG'] }, + { id: 'direct_fetch' }, + ], + }); const background: Promise[] = []; const requested: string[] = []; diff --git a/packages/gateway/__tests__/test-utils/app.ts b/packages/gateway/__tests__/test-utils/app.ts index 9a74fa8b0..fc4387c34 100644 --- a/packages/gateway/__tests__/test-utils/app.ts +++ b/packages/gateway/__tests__/test-utils/app.ts @@ -1,6 +1,6 @@ import { trackBackground } from './background-tracker.ts'; import { app } from '../../src/app.ts'; -import { clearInFlightForTesting, warmUpstreamModels } from '../../src/data-plane/providers/models-cache.ts'; +import { clearModelsRefreshesForTesting, warmUpstreamModels } from '../../src/data-plane/providers/models-refresh.ts'; import { listModelProviders } from '../../src/data-plane/providers/registry.ts'; import type { WebSearchConfig } from '../../src/data-plane/tools/web-search/types.ts'; import { createPerRequestFetcher } from '../../src/dial/per-request.ts'; @@ -140,7 +140,7 @@ export async function setupAppTest(options: SetupOptions = {}): Promise error instanceof Error ? error. const saveUpstreamForModels = async ({ previous, next }: UpstreamModelsChange): Promise => { const upstreams = getRepo().upstreams; if (previous === null) { - await upstreams.save(next); + const inserted = await upstreams.insertForModels(next); + if (!inserted) throw new Error(`Upstream ${next.id} changed concurrently`); return; } let cachePolicy: 'preserve' | 'reset-refresh' | 'clear'; diff --git a/packages/gateway/src/control-plane/upstreams/models.ts b/packages/gateway/src/control-plane/upstreams/models.ts index 9cbb418e3..03c537c9f 100644 --- a/packages/gateway/src/control-plane/upstreams/models.ts +++ b/packages/gateway/src/control-plane/upstreams/models.ts @@ -2,12 +2,11 @@ import { resolveControlPlaneFetcher } from './proxy-resolution.ts'; import { isValidProviderKind, upstreamErrorMessage as errorMessage } from './shared.ts'; import type { ListedUpstreamModel } from './types.ts'; import { MODEL_LISTING_FAILURE_CODE, MODEL_LISTING_FAILURE_MESSAGE } from '../../data-plane/models/shared.ts'; -import { fetchUpstreamModels } from '../../data-plane/providers/models-cache.ts'; -import { createProvider, modelsCatalogIdentity } from '../../data-plane/providers/registry.ts'; +import { fetchUpstreamModels } from '../../data-plane/providers/models-refresh.ts'; +import { createProvider, modelsOperatorRefreshIdentity } from '../../data-plane/providers/registry.ts'; import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { modelsCacheGeneration } from '../../repo/models-cache-contract.ts'; -import { serializeStoredConfig } from '../../repo/upstream-json.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { listModelsBody } from '../schemas.ts'; import { ProviderModelsUnavailableError, type Fetcher, type ProviderModel, type ProxyFallbackEntry, type UpstreamRecord } from '@floway-dev/provider'; @@ -72,8 +71,7 @@ export const listModels = async (c: CtxWithJson) => { modelsCache: null, }; const canRefreshPersistedCache = persisted !== null - && modelsCatalogIdentity(persisted) === modelsCatalogIdentity(synthRecord) - && serializeStoredConfig(persisted.proxyFallbackList) === serializeStoredConfig(synthRecord.proxyFallbackList); + && modelsOperatorRefreshIdentity(persisted) === modelsOperatorRefreshIdentity(synthRecord); let fetcher: Fetcher; try { diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index 106ed9a38..df3e2cb1c 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -1,231 +1,37 @@ +import { scheduleUpstreamModelsRefresh } from './models-refresh.ts'; import type { GatewayProvider } from './registry.ts'; -import { getRepo } from '../../repo/index.ts'; -import { MODEL_CATALOG_REVISION, modelsFetchIdentity } from '../../repo/models-cache-contract.ts'; -import { MODELS_REFRESH_CLAIM_LEASE_MS } from '../../repo/models-refresh-contract.ts'; +import { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { Fetcher, ProviderModel, UpstreamModelsCache } from '@floway-dev/provider'; -// Soft-fresh rows need no refresh. Every older row remains usable forever; -// access only triggers a background attempt guarded by the persisted refresh -// claim/backoff state. const SOFT_MS = 10 * 60 * 1000; -const ACTIVE_REFRESH_POLL_MS = 100; -const ACTIVE_REFRESH_POLL_CAP_MS = 1_000; -const ACTIVE_REFRESH_WAIT_MS = 60_000; export { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; -interface ModelsSnapshotReadOptions { - scheduler: BackgroundScheduler; - fetcher: Fetcher; -} - -interface ModelsSnapshot { +export interface ModelsSnapshot { readonly models: readonly ProviderModel[]; readonly lastError: UpstreamModelsCache['lastError']; } -// L1: per-isolate in-flight memoization. Callers join only when both their -// actual fetch inputs and persisted-cache ownership match; different drafts -// and superseded rows remain isolated. Not a TTL cache — the entry is removed -// when the promise settles. The conditional delete defends against a stale -// removal racing a later replacement. -type RefreshIntent = 'explicit' | 'warm' | 'background'; - -interface InFlightRefresh { - kind: 'refresh' | 'owner-wait'; - promise: Promise; +interface ModelsSnapshotReadOptions { + scheduler: BackgroundScheduler; + fetcher: Fetcher; } -const inFlight = new Map(); - -const startInFlight = ( - key: string, - kind: InFlightRefresh['kind'], - fn: () => Promise, -): Promise => { - const entry: InFlightRefresh = { kind, promise: fn() }; - inFlight.set(key, entry); - entry.promise.finally(() => { - if (inFlight.get(key) === entry) inFlight.delete(key); - }).catch(() => {}); - return entry.promise; -}; - -const memoInFlight = ( - key: string, - kind: InFlightRefresh['kind'], - fn: () => Promise, -): Promise => { - const existing = inFlight.get(key); - return existing?.promise ?? startInFlight(key, kind, fn); -}; - -const errorMessage = (err: unknown): string => err instanceof Error ? err.message : String(err); - -const runFetch = async ( - instance: GatewayProvider, - fetcher: Fetcher, - loadProvidedModels?: () => Promise, -): Promise => [...await (loadProvidedModels?.() ?? instance.instance.getProvidedModels(fetcher))]; - -const runClaimedRefresh = async ( - instance: GatewayProvider, - fetcher: Fetcher, - intent: RefreshIntent, - loadProvidedModels?: () => Promise, -): Promise => { - const repo = getRepo(); - const token = crypto.randomUUID(); - let observedActiveToken: string | null = null; - let pollMs = ACTIVE_REFRESH_POLL_MS; - const waitDeadline = Date.now() + ACTIVE_REFRESH_WAIT_MS; - while (true) { - const now = Date.now(); - const outcome = await repo.upstreams.claimModelsRefresh({ - id: instance.upstreamId, - generation: instance.modelsCacheGeneration, - token, - now, - staleClaimedBefore: now - MODELS_REFRESH_CLAIM_LEASE_MS, - bypassBackoff: intent === 'explicit', - observedActiveToken, - }); - if (outcome.kind === 'backoff' || outcome.kind === 'generation-mismatch') return null; - if (outcome.kind === 'completed') { - const current = await repo.upstreams.getById(instance.upstreamId); - if (current !== null - && current.updatedAt === instance.modelsCacheGeneration.updatedAt - && modelsFetchIdentity(current) === instance.modelsCacheGeneration.fetchIdentity) instance.modelsCache = current.modelsCache; - if (intent === 'explicit' && instance.modelsCache?.lastError !== null && instance.modelsCache?.lastError !== undefined) { - throw new Error(instance.modelsCache.lastError.message); - } - return instance.modelsCache?.models ?? []; - } - if (outcome.kind === 'active') { - if (intent === 'background') return null; - if (now >= waitDeadline) throw new Error(`Timed out waiting for models refresh owner for ${instance.upstreamId}`); - observedActiveToken = outcome.token; - await new Promise(resolve => setTimeout(resolve, pollMs)); - pollMs = Math.min(pollMs * 2, ACTIVE_REFRESH_POLL_CAP_MS); - continue; - } - - let models: ProviderModel[]; - try { - models = await runFetch(instance, fetcher, loadProvidedModels); - } catch (error) { - const failedAt = Date.now(); - const lastError = { message: errorMessage(error), at: failedAt }; - let finalized: boolean; - try { - finalized = await repo.upstreams.finalizeModelsRefreshFailure({ - id: instance.upstreamId, - generation: instance.modelsCacheGeneration, - token, - error: lastError, - previousFailureCount: outcome.failureCount, - failedAt, - }); - } catch (backoffError) { - throw new AggregateError([error, backoffError], errorMessage(error)); - } - if (finalized) { - if (instance.modelsCache) instance.modelsCache.lastError = lastError; - else instance.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError }; - throw error; - } - if (intent === 'background') throw error; - observedActiveToken = token; - continue; - } - const entry = { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models, lastError: null }; - const finalized = await repo.upstreams.finalizeModelsRefreshSuccess({ - id: instance.upstreamId, - generation: instance.modelsCacheGeneration, - token, - cache: entry, - }); - if (finalized) { - // The instance is reused across alias targets in one request, so publish - // the finalized snapshot locally as well as durably. - instance.modelsCache = entry; - return models; - } - if (intent === 'background') return models; - observedActiveToken = token; - } -}; - -const inFlightKey = (instance: GatewayProvider): string => { - const generation = instance.modelsCacheGeneration; - return `${instance.upstreamId}\0${generation.updatedAt}\0${generation.fetchIdentity}`; -}; - -export const fetchUpstreamModels = async ( - instance: GatewayProvider, - fetcher: Fetcher, - loadProvidedModels?: () => Promise, -): Promise => { - const key = inFlightKey(instance); - while (true) { - const existing = inFlight.get(key); - if (existing?.kind === 'refresh') { - const joined = await existing.promise; - if (joined !== null) return joined; - if (inFlight.get(key) === existing) inFlight.delete(key); - continue; - } - const models = await startInFlight(key, 'refresh', () => runClaimedRefresh(instance, fetcher, 'explicit', loadProvidedModels)); - if (models === null) throw new Error(`Failed to acquire models refresh for ${instance.upstreamId}`); - return models; - } -}; - -export const warmUpstreamModels = async ( - instance: GatewayProvider, - fetcher: Fetcher, -): Promise => { - const key = inFlightKey(instance); - const existing = inFlight.get(key); - if (existing) { - const joined = await existing.promise; - if (joined !== null) return joined; - if (existing.kind === 'owner-wait') return instance.modelsCache?.models ?? []; - if (inFlight.get(key) === existing) inFlight.delete(key); - } - - const models = await memoInFlight(key, 'owner-wait', () => runClaimedRefresh(instance, fetcher, 'warm')); - return models ?? instance.modelsCache?.models ?? []; -}; - -export const scheduleUpstreamModelsRefresh = ( - instance: GatewayProvider, - scheduler: BackgroundScheduler, - fetcher: Fetcher, -): void => { - const key = inFlightKey(instance); - scheduler(memoInFlight(key, 'refresh', () => runClaimedRefresh(instance, fetcher, 'background')).then(() => {})); -}; - +// Capture one immutable snapshot before scheduling any refresh work so its +// models and error metadata always describe the same durable generation. export const readUpstreamModelsSnapshotAndScheduleRefresh = ( instance: GatewayProvider, - opts: ModelsSnapshotReadOptions, + options: ModelsSnapshotReadOptions, ): ModelsSnapshot => { - const { scheduler, fetcher } = opts; - const now = Date.now(); + const { scheduler, fetcher } = options; const cached = instance.modelsCache?.revision === MODEL_CATALOG_REVISION ? instance.modelsCache : null; const snapshot = { models: cached?.models ?? [], lastError: cached?.lastError ?? null, }; - - if (!cached || now - cached.fetchedAt >= SOFT_MS) scheduleUpstreamModelsRefresh(instance, scheduler, fetcher); + if (!cached || Date.now() - cached.fetchedAt >= SOFT_MS) { + scheduleUpstreamModelsRefresh(instance, scheduler, fetcher); + } return snapshot; }; - -// Test-only: drop the L1 map so a test's setup is independent of any -// promise the previous test left mid-settle. -export const clearInFlightForTesting = (): void => { - inFlight.clear(); -}; diff --git a/packages/gateway/src/data-plane/providers/models-refresh.ts b/packages/gateway/src/data-plane/providers/models-refresh.ts new file mode 100644 index 000000000..55725aaed --- /dev/null +++ b/packages/gateway/src/data-plane/providers/models-refresh.ts @@ -0,0 +1,222 @@ +import type { GatewayProvider } from './registry.ts'; +import { getRepo } from '../../repo/index.ts'; +import { MODEL_CATALOG_REVISION, modelsFetchIdentity } from '../../repo/models-cache-contract.ts'; +import { MODELS_REFRESH_CLAIM_LEASE_MS } from '../../repo/models-refresh-contract.ts'; +import type { BackgroundScheduler } from '@floway-dev/platform'; +import type { Fetcher, ProviderModel } from '@floway-dev/provider'; + +const ACTIVE_REFRESH_POLL_MS = 100; +const ACTIVE_REFRESH_POLL_CAP_MS = 1_000; +const ACTIVE_REFRESH_WAIT_MS = 60_000; + +// L1: per-isolate in-flight memoization. Callers join only when both their +// actual fetch inputs and persisted-cache ownership match; different drafts +// and superseded rows remain isolated. Not a TTL cache — the entry is removed +// when the promise settles. The conditional delete defends against a stale +// removal racing a later replacement. +type RefreshIntent = 'explicit' | 'warm' | 'background'; + +interface InFlightRefresh { + kind: 'background-refresh' | 'explicit-refresh' | 'owner-wait'; + promise: Promise; +} + +const inFlight = new Map(); + +const startInFlight = ( + key: string, + kind: InFlightRefresh['kind'], + fn: () => Promise, +): Promise => { + const entry: InFlightRefresh = { kind, promise: fn() }; + inFlight.set(key, entry); + entry.promise.finally(() => { + if (inFlight.get(key) === entry) inFlight.delete(key); + }).catch(() => {}); + return entry.promise; +}; + +const memoInFlight = ( + key: string, + kind: InFlightRefresh['kind'], + fn: () => Promise, +): Promise => { + const existing = inFlight.get(key); + return existing?.promise ?? startInFlight(key, kind, fn); +}; + +const errorMessage = (err: unknown): string => err instanceof Error ? err.message : String(err); + +const finalizeRefresh = async (finalize: () => Promise): Promise => { + const errors: unknown[] = []; + for (let attempt = 0; attempt < 3; attempt++) { + try { + return await finalize(); + } catch (error) { + errors.push(error); + } + } + throw new AggregateError(errors, 'Failed to finalize models refresh'); +}; + +const runFetch = async ( + instance: GatewayProvider, + fetcher: Fetcher, + loadProvidedModels?: () => Promise, +): Promise => [...await (loadProvidedModels?.() ?? instance.instance.getProvidedModels(fetcher))]; + +const runClaimedRefresh = async ( + instance: GatewayProvider, + fetcher: Fetcher, + intent: RefreshIntent, + loadProvidedModels?: () => Promise, +): Promise => { + const repo = getRepo(); + const token = crypto.randomUUID(); + let observedActiveToken: string | null = null; + let pollMs = ACTIVE_REFRESH_POLL_MS; + const waitDeadline = Date.now() + ACTIVE_REFRESH_WAIT_MS; + while (true) { + const now = Date.now(); + const outcome = await repo.upstreams.claimModelsRefresh({ + id: instance.upstreamId, + generation: instance.modelsCacheGeneration, + token, + now, + staleClaimedBefore: now - MODELS_REFRESH_CLAIM_LEASE_MS, + bypassBackoff: intent === 'explicit', + observedActiveToken, + }); + if (outcome.kind === 'backoff' || outcome.kind === 'generation-mismatch') return null; + if (outcome.kind === 'completed') { + const current = await repo.upstreams.getById(instance.upstreamId); + if (current === null + || current.updatedAt !== instance.modelsCacheGeneration.updatedAt + || modelsFetchIdentity(current) !== instance.modelsCacheGeneration.fetchIdentity) return null; + instance.modelsCache = current.modelsCache; + if (intent === 'explicit' && current.modelsCache?.lastError !== null && current.modelsCache?.lastError !== undefined) { + observedActiveToken = null; + continue; + } + return current.modelsCache?.models ?? []; + } + if (outcome.kind === 'active') { + if (intent === 'background') return null; + if (now >= waitDeadline) throw new Error(`Timed out waiting for models refresh owner for ${instance.upstreamId}`); + observedActiveToken = outcome.token; + await new Promise(resolve => setTimeout(resolve, pollMs)); + pollMs = Math.min(pollMs * 2, ACTIVE_REFRESH_POLL_CAP_MS); + continue; + } + + let models: ProviderModel[]; + try { + models = await runFetch(instance, fetcher, loadProvidedModels); + } catch (error) { + const failedAt = Date.now(); + const lastError = { message: errorMessage(error), at: failedAt }; + let finalized: boolean; + try { + finalized = await finalizeRefresh(async () => await repo.upstreams.finalizeModelsRefreshFailure({ + id: instance.upstreamId, + generation: instance.modelsCacheGeneration, + token, + error: lastError, + previousFailureCount: outcome.failureCount, + failedAt, + })); + } catch (backoffError) { + throw new AggregateError([error, backoffError], errorMessage(error)); + } + if (finalized) { + if (instance.modelsCache) instance.modelsCache.lastError = lastError; + else instance.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError }; + throw error; + } + if (intent === 'background') throw error; + observedActiveToken = token; + continue; + } + const entry = { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models, lastError: null }; + const finalized = await finalizeRefresh(async () => await repo.upstreams.finalizeModelsRefreshSuccess({ + id: instance.upstreamId, + generation: instance.modelsCacheGeneration, + token, + cache: entry, + })); + if (finalized) { + // The instance is reused across alias targets in one request, so publish + // the finalized snapshot locally as well as durably. + instance.modelsCache = entry; + return models; + } + if (intent === 'background') return models; + observedActiveToken = token; + } +}; + +const inFlightKey = (instance: GatewayProvider): string => { + const generation = instance.modelsCacheGeneration; + return `${instance.upstreamId}\0${generation.updatedAt}\0${generation.fetchIdentity}`; +}; + +export const fetchUpstreamModels = async ( + instance: GatewayProvider, + fetcher: Fetcher, + loadProvidedModels?: () => Promise, +): Promise => { + const key = inFlightKey(instance); + while (true) { + const existing = inFlight.get(key); + if (existing?.kind === 'explicit-refresh') { + const joined = await existing.promise; + if (joined === null) throw new Error(`Models refresh generation changed for ${instance.upstreamId}`); + return joined; + } + if (existing?.kind === 'background-refresh') { + try { + const joined = await existing.promise; + if (joined !== null) return joined; + } catch { + // The operator request owns a distinct attempt after a background + // failure, and bypasses the cooldown that failure just established. + } + if (inFlight.get(key) === existing) inFlight.delete(key); + } + const models = await startInFlight(key, 'explicit-refresh', () => runClaimedRefresh(instance, fetcher, 'explicit', loadProvidedModels)); + if (models === null) throw new Error(`Failed to acquire models refresh for ${instance.upstreamId}`); + return models; + } +}; + +export const warmUpstreamModels = async ( + instance: GatewayProvider, + fetcher: Fetcher, +): Promise => { + const key = inFlightKey(instance); + const existing = inFlight.get(key); + if (existing) { + const joined = await existing.promise; + if (joined !== null) return joined; + if (existing.kind === 'owner-wait') return instance.modelsCache?.models ?? []; + if (inFlight.get(key) === existing) inFlight.delete(key); + } + + const models = await memoInFlight(key, 'owner-wait', () => runClaimedRefresh(instance, fetcher, 'warm')); + return models ?? instance.modelsCache?.models ?? []; +}; + +export const scheduleUpstreamModelsRefresh = ( + instance: GatewayProvider, + scheduler: BackgroundScheduler, + fetcher: Fetcher, +): void => { + const key = inFlightKey(instance); + scheduler(memoInFlight(key, 'background-refresh', () => runClaimedRefresh(instance, fetcher, 'background')).then(() => {})); +}; + +// Test-only: drop the L1 map so a test's setup is independent of any +// promise the previous test left mid-settle. +export const clearModelsRefreshesForTesting = (): void => { + inFlight.clear(); +}; diff --git a/packages/gateway/src/data-plane/providers/registry.ts b/packages/gateway/src/data-plane/providers/registry.ts index ce263eadf..9e0bfdcdf 100644 --- a/packages/gateway/src/data-plane/providers/registry.ts +++ b/packages/gateway/src/data-plane/providers/registry.ts @@ -26,6 +26,13 @@ export type GatewayProvider = Provider & { export const modelsCatalogIdentity = (record: UpstreamRecord): string => serializeStoredConfig({ kind: record.kind, identity: providersByKind[record.kind].modelCatalogIdentity(record) }); +export const modelsOperatorRefreshIdentity = (record: UpstreamRecord): string => + serializeStoredConfig({ + kind: record.kind, + identity: providersByKind[record.kind].modelRefreshIdentity(record), + proxyFallbackList: record.proxyFallbackList, + }); + export const createProvider = ( record: UpstreamRecord, cacheGeneration: ModelsCacheGeneration = modelsCacheGeneration(record), diff --git a/packages/gateway/src/dial/per-request.ts b/packages/gateway/src/dial/per-request.ts index dc9b60d35..bbb6ae58d 100644 --- a/packages/gateway/src/dial/per-request.ts +++ b/packages/gateway/src/dial/per-request.ts @@ -1,7 +1,7 @@ import { createFetcher } from './fetcher.ts'; import { loadProxyCatalog } from './proxy-catalog.ts'; import { getRepo } from '../repo/index.ts'; -import { isDirectFallbackId } from '../repo/proxy-fallback-list.ts'; +import { entryMatchesColo, isDirectFallbackId } from '../repo/proxy-fallback-list.ts'; import { getSocketDial } from '@floway-dev/platform'; import { directFetcher, type Fetcher, type UpstreamRecord } from '@floway-dev/provider'; import { runDirectConnectRequest, runProxiedRequest } from '@floway-dev/proxy'; @@ -20,7 +20,10 @@ export const createPerRequestFetcher = async ( ): Promise<(upstreamId: string) => Fetcher> => { const repo = getRepo(); const upstreams = preFetchedUpstreams ?? await repo.upstreams.list(); - const fallbackById = new Map(upstreams.map(u => [u.id, u.proxyFallbackList] as const)); + const fallbackById = new Map(upstreams.map(u => [ + u.id, + u.proxyFallbackList.filter(entry => entryMatchesColo(entry, runtimeLocation)), + ] as const)); const referencedProxyIds = new Set(); for (const list of fallbackById.values()) { diff --git a/packages/gateway/src/repo/models-cache-contract.ts b/packages/gateway/src/repo/models-cache-contract.ts index 215fb15f9..89435a77a 100644 --- a/packages/gateway/src/repo/models-cache-contract.ts +++ b/packages/gateway/src/repo/models-cache-contract.ts @@ -17,17 +17,6 @@ export const modelsFetchIdentity = ( proxyFallbackList: record.proxyFallbackList, }); -// Control-plane replacements reset cooldown when any operator-owned fetch -// input changes, including credential state updated by OAuth/import flows. -export const modelsOperatorRefreshIdentity = ( - record: Pick, -): string => serializeStoredConfig({ - kind: record.kind, - config: record.config, - state: record.state ?? null, - proxyFallbackList: record.proxyFallbackList, -}); - export const modelsCacheGeneration = ( record: Pick, ): ModelsCacheGeneration => ({ diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index 9454e961b..5b368fbaf 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -898,12 +898,36 @@ class SqlUpstreamRepo implements UpstreamRepo { return this.saveRecord(upstream); } + async insertForModels(upstream: UpstreamRecord): Promise { + const result = await this.db + .prepare('INSERT INTO upstreams (id, provider, name, enabled, sort_order, created_at, updated_at, config_json, state_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING') + .bind( + upstream.id, + upstream.kind, + upstream.name, + upstream.enabled ? 1 : 0, + upstream.sortOrder, + upstream.createdAt, + upstream.updatedAt, + serializeStoredConfig(upstream.config), + serializeStoredState(upstream.state), + JSON.stringify(normalizeFlagOverrides(upstream.flagOverrides)), + JSON.stringify(normalizeDisabledPublicModelIds(upstream.disabledPublicModelIds)), + JSON.stringify(normalizeProxyFallbackList(upstream.proxyFallbackList)), + upstream.modelPrefix === null ? null : JSON.stringify(upstream.modelPrefix), + upstream.hue, + ) + .run(); + return (result.meta.changes ?? 0) > 0; + } + async replaceForModels(input: { previous: UpstreamRecord; upstream: UpstreamRecord; cachePolicy: 'preserve' | 'reset-refresh' | 'clear'; }): Promise { const { previous, upstream, cachePolicy } = input; + const replaceState = serializeStoredState(previous.state) !== serializeStoredState(upstream.state); const modelsRefreshUpdate = cachePolicy === 'preserve' ? "CASE WHEN models_refresh_json IS NULL THEN NULL ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) END" : 'NULL'; @@ -917,7 +941,7 @@ class SqlUpstreamRepo implements UpstreamRepo { sort_order = ?, updated_at = ?, config_json = ?, - state_json = ?, + state_json = CASE WHEN ? THEN ? ELSE state_json END, flag_overrides = ?, disabled_public_model_ids = ?, proxy_fallback_list_json = ?, @@ -931,7 +955,7 @@ class SqlUpstreamRepo implements UpstreamRepo { AND sort_order = ? AND updated_at = ? AND config_json = ? - AND state_json IS ? + AND (? = 0 OR state_json IS ?) AND flag_overrides = ? AND disabled_public_model_ids = ? AND proxy_fallback_list_json = ? @@ -945,6 +969,7 @@ class SqlUpstreamRepo implements UpstreamRepo { upstream.sortOrder, upstream.updatedAt, serializeStoredConfig(upstream.config), + sqliteBoolean(replaceState), serializeStoredState(upstream.state), JSON.stringify(normalizeFlagOverrides(upstream.flagOverrides)), JSON.stringify(normalizeDisabledPublicModelIds(upstream.disabledPublicModelIds)), @@ -958,6 +983,7 @@ class SqlUpstreamRepo implements UpstreamRepo { previous.sortOrder, previous.updatedAt, serializeStoredConfig(previous.config), + sqliteBoolean(replaceState), serializeStoredState(previous.state), JSON.stringify(normalizeFlagOverrides(previous.flagOverrides)), JSON.stringify(normalizeDisabledPublicModelIds(previous.disabledPublicModelIds)), @@ -1043,9 +1069,11 @@ class SqlUpstreamRepo implements UpstreamRepo { `UPDATE upstreams SET models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END, models_refresh_json = json_object('failCount', ?, 'retryAt', ?, 'claimToken', NULL, 'claimedAt', NULL) - WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?`, + WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ? + AND json_extract(models_refresh_json, '$.claimToken') = ? + AND coalesce(json_extract(models_refresh_json, '$.failCount'), 0) = ?`, ) - .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, token) + .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, token, previousFailureCount) .run(); return (result.meta.changes ?? 0) > 0; } @@ -1075,13 +1103,13 @@ class SqlUpstreamRepo implements UpstreamRepo { ) OR ( ? IS NOT NULL - AND json_extract(models_refresh_json, '$.claimToken') = ? + AND json_extract(models_refresh_json, '$.claimToken') IS NOT NULL AND json_extract(models_refresh_json, '$.claimedAt') <= ? ) ) RETURNING json_extract(models_refresh_json, '$.failCount') AS fail_count`, ) - .bind(token, now, id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, observedActiveToken, sqliteBoolean(bypassBackoff), now, staleClaimedBefore, observedActiveToken, observedActiveToken, staleClaimedBefore) + .bind(token, now, id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, observedActiveToken, sqliteBoolean(bypassBackoff), now, staleClaimedBefore, observedActiveToken, staleClaimedBefore) .first<{ fail_count: number }>(); if (row !== null) return { kind: 'claimed', failureCount: row.fail_count }; diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index 034e257f7..6c89b6246 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -344,6 +344,7 @@ export interface UpstreamRepo { list(): Promise; getById(id: string): Promise; save(upstream: UpstreamRecord): Promise; + insertForModels(upstream: UpstreamRecord): Promise; replaceForModels(input: { previous: UpstreamRecord; upstream: UpstreamRecord; diff --git a/packages/gateway/src/scheduled/models-refresh.ts b/packages/gateway/src/scheduled/models-refresh.ts index 40370d57d..65466992d 100644 --- a/packages/gateway/src/scheduled/models-refresh.ts +++ b/packages/gateway/src/scheduled/models-refresh.ts @@ -1,4 +1,4 @@ -import { scheduleUpstreamModelsRefresh } from '../data-plane/providers/models-cache.ts'; +import { scheduleUpstreamModelsRefresh } from '../data-plane/providers/models-refresh.ts'; import { createProvider } from '../data-plane/providers/registry.ts'; import { createPerRequestFetcher } from '../dial/per-request.ts'; import { getRepo } from '../repo/index.ts'; diff --git a/packages/provider-azure/src/index.ts b/packages/provider-azure/src/index.ts index b8b526bad..29f836ed8 100644 --- a/packages/provider-azure/src/index.ts +++ b/packages/provider-azure/src/index.ts @@ -6,6 +6,7 @@ import type { ProviderModule } from '@floway-dev/provider'; export const azureProviderModule: ProviderModule = { create: createAzureProvider, modelCatalogIdentity: record => assertAzureUpstreamRecord(record).config, + modelRefreshIdentity: record => assertAzureUpstreamRecord(record).config, defaultFlags: AZURE_DEFAULT_FLAGS, }; export { assertAzureUpstreamRecord, type AzureUpstreamConfig } from './config.ts'; diff --git a/packages/provider-claude-code/src/index.ts b/packages/provider-claude-code/src/index.ts index 9d6a862ea..17d512b1e 100644 --- a/packages/provider-claude-code/src/index.ts +++ b/packages/provider-claude-code/src/index.ts @@ -9,6 +9,10 @@ export const claudeCodeProviderModule: ProviderModule = { assertClaudeCodeUpstreamRecord(record); return record.config; }, + modelRefreshIdentity: record => { + assertClaudeCodeUpstreamRecord(record); + return { config: record.config, state: record.state }; + }, defaultFlags: CLAUDE_CODE_DEFAULT_FLAGS, }; diff --git a/packages/provider-codex/src/index.ts b/packages/provider-codex/src/index.ts index 9f9f29015..233417396 100644 --- a/packages/provider-codex/src/index.ts +++ b/packages/provider-codex/src/index.ts @@ -9,6 +9,10 @@ export const codexProviderModule: ProviderModule = { assertCodexUpstreamRecord(record); return record.config; }, + modelRefreshIdentity: record => { + assertCodexUpstreamRecord(record); + return { config: record.config, state: record.state }; + }, defaultFlags: CODEX_DEFAULT_FLAGS, }; diff --git a/packages/provider-copilot/src/index.ts b/packages/provider-copilot/src/index.ts index d508b90f1..883c47959 100644 --- a/packages/provider-copilot/src/index.ts +++ b/packages/provider-copilot/src/index.ts @@ -9,6 +9,10 @@ export const copilotProviderModule: ProviderModule = { const upstream = assertCopilotUpstreamRecord(record); return { githubHost: upstream.config.githubHost, userId: upstream.config.user.id }; }, + modelRefreshIdentity: record => { + const upstream = assertCopilotUpstreamRecord(record); + return { config: upstream.config, state: upstream.state }; + }, defaultFlags: COPILOT_DEFAULT_FLAGS, }; diff --git a/packages/provider-custom/src/index.ts b/packages/provider-custom/src/index.ts index 5a79c37e2..ec62e620f 100644 --- a/packages/provider-custom/src/index.ts +++ b/packages/provider-custom/src/index.ts @@ -6,6 +6,7 @@ import type { ProviderModule } from '@floway-dev/provider'; export const customProviderModule: ProviderModule = { create: createCustomProvider, modelCatalogIdentity: record => assertCustomUpstreamRecord(record).config, + modelRefreshIdentity: record => assertCustomUpstreamRecord(record).config, defaultFlags: CUSTOM_DEFAULT_FLAGS, }; diff --git a/packages/provider-ollama/src/index.ts b/packages/provider-ollama/src/index.ts index ccdab62bc..3d2ddb159 100644 --- a/packages/provider-ollama/src/index.ts +++ b/packages/provider-ollama/src/index.ts @@ -6,6 +6,7 @@ import type { ProviderModule } from '@floway-dev/provider'; export const ollamaProviderModule: ProviderModule = { create: createOllamaProvider, modelCatalogIdentity: record => assertOllamaUpstreamRecord(record).config, + modelRefreshIdentity: record => assertOllamaUpstreamRecord(record).config, defaultFlags: OLLAMA_DEFAULT_FLAGS, }; diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index 3f6f34fdb..efa43b358 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -169,6 +169,9 @@ export interface ProviderModule { // Stable identity of the upstream account/catalog namespace. Each provider // decides which of its configuration changes can preserve a snapshot. modelCatalogIdentity: (record: UpstreamRecord) => unknown; + // Normalized inputs an operator-controlled refresh would use. Changes reset + // refresh cooldown even when the last-known-good catalog remains valid. + modelRefreshIdentity: (record: UpstreamRecord) => unknown; // Exhaustive default map over every catalog flag id for a fresh // upstream of this kind; see each provider package's `defaults.ts`. defaultFlags: FlagDefaults; From a69983a285e766360003b641ed0e25aed330d68e Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 18:11:37 +0800 Subject: [PATCH 41/46] fix(gateway): close catalog coordination races Reconcile uncertain finalization by releasing claims that never committed, retry explicit recovery exactly once across concurrent waiters, and make stale-owner and failure-count transitions total across SQL and memory.\n\nUse authoritative post-save rows for warming, preserve unrelated provider state during metadata writes, make catalog-aware inserts conflict-safe, and reject duplicate import ids before mutation. Normalize request identity per provider while excluding passive observation state.\n\nFilter proxy catalogs after location selection and update the merged Node repository fixture to the current refresh contract. --- .../__tests__/node-sqlite-repo_test.ts | 27 ++++++----- .../data-transfer/routes_test.ts | 13 +++++- .../upstreams/copilot-device-login_test.ts | 2 +- .../data-plane/providers/models-cache_test.ts | 24 +++++++++- packages/gateway/__tests__/repo/memory.ts | 16 ++++++- .../data-transfer/import-schema.ts | 8 ++++ .../shared/save-upstream-for-models.ts | 12 ++++- .../src/control-plane/upstreams/models.ts | 9 ++-- .../data-plane/providers/models-refresh.ts | 45 ++++++++++++------- .../src/data-plane/providers/registry.ts | 4 +- packages/gateway/src/index.ts | 2 +- .../gateway/src/repo/models-cache-contract.ts | 11 +++++ packages/gateway/src/repo/sql.ts | 12 +++++ packages/gateway/src/repo/types.ts | 7 +++ packages/provider-azure/src/index.ts | 2 +- packages/provider-claude-code/src/index.ts | 4 +- packages/provider-codex/src/index.ts | 4 +- packages/provider-copilot/src/index.ts | 4 +- packages/provider-custom/src/index.ts | 2 +- packages/provider-ollama/src/index.ts | 2 +- packages/provider/src/provider.ts | 6 +-- 21 files changed, 162 insertions(+), 54 deletions(-) diff --git a/apps/platform-node/__tests__/node-sqlite-repo_test.ts b/apps/platform-node/__tests__/node-sqlite-repo_test.ts index cf92c2802..38942de7e 100644 --- a/apps/platform-node/__tests__/node-sqlite-repo_test.ts +++ b/apps/platform-node/__tests__/node-sqlite-repo_test.ts @@ -2,7 +2,7 @@ import { test } from 'vitest'; import { applyMigrations } from '../src/migrate.ts'; import { createNodeSqliteDatabase } from '../src/node-sqlite-database.ts'; -import { MODEL_CATALOG_REVISION, SqlRepo } from '@floway-dev/gateway'; +import { MODEL_CATALOG_REVISION, modelsCacheGeneration, SqlRepo } from '@floway-dev/gateway'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; // The repo layer's own suite runs against sql.js, which — like D1 — coerces a @@ -77,9 +77,9 @@ test('expiration sweep completion lands on both discriminants', () => withRepo(a test('repository JSON codecs round-trip upstream, alias, and Responses state through node:sqlite', () => withRepo(async repo => { await seedKey(repo); - await repo.upstreams.save({ + const upstreamRecord = { id: 'up_node', - kind: 'custom', + kind: 'custom' as const, name: 'Node upstream', enabled: true, sortOrder: 0, @@ -93,18 +93,21 @@ test('repository JSON codecs round-trip upstream, alias, and Responses state thr proxyFallbackList: [], modelPrefix: null, hue: 210, - }); - const cacheGeneration = { - updatedAt: '2026-08-05T00:00:00.000Z', - config: { opaque: { value: true } }, }; + await repo.upstreams.save(upstreamRecord); + const cacheGeneration = modelsCacheGeneration(upstreamRecord); const cacheToken = 'node-cache-fixture'; - const cacheClaim = await repo.upstreams.claimModelsRefresh({ id: 'up_node', generation: cacheGeneration, token: cacheToken, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, force: true, observedActiveToken: null }); + const cacheClaim = await repo.upstreams.claimModelsRefresh({ id: 'up_node', generation: cacheGeneration, token: cacheToken, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, bypassBackoff: true, observedActiveToken: null }); if (cacheClaim.kind !== 'claimed') throw new Error('expected model-cache fixture claim'); - await repo.upstreams.finalizeModelsRefreshSuccess('up_node', cacheGeneration, cacheToken, { - revision: MODEL_CATALOG_REVISION, - fetchedAt: 1_786_000_000_000, - models: [stubProviderModel({ id: 'node-model', enabledFlags: new Set(['vendor-kimi'] as const) })], + await repo.upstreams.finalizeModelsRefreshSuccess({ + id: 'up_node', + generation: cacheGeneration, + token: cacheToken, + cache: { + revision: MODEL_CATALOG_REVISION, + fetchedAt: 1_786_000_000_000, + models: [stubProviderModel({ id: 'node-model', enabledFlags: new Set(['vendor-kimi'] as const) })], + }, }); await repo.modelAliases.insert({ id: 'alias_node', diff --git a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts index a36e11fe5..f5f6b8912 100644 --- a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts @@ -7,7 +7,7 @@ import { expect, test, vi } from 'vitest'; // until the vitest timeout. Stub the cache layer to a no-op so the import // path's own behavior (upserts, identity validation, etc.) is what the tests // exercise — the warm itself has dedicated coverage in models-cache_test.ts. -vi.mock('../../../src/data-plane/providers/models-cache.ts', () => ({ +vi.mock('../../../src/data-plane/providers/models-refresh.ts', () => ({ warmUpstreamModels: () => Promise.resolve([]), })); @@ -1035,6 +1035,17 @@ test('import reports the earliest duplicate before later malformed records', asy assertEquals(duplicateBucket.body.error, 'invalid performance record at index 0: duplicate bucket entry for {metric: ttft_ms, lower: 0}'); }); +test('import rejects duplicate upstream ids before applying any records', async () => { + const { app, repo } = setup(); + const upstream = upstreamRecordToFullJson(CUSTOM_UPSTREAM); + const result = await doImport(app, 'replace', latestImportData({ + upstreams: [upstream, { ...upstream, name: 'Duplicate' }], + })); + + assertEquals(result.body.error, `invalid upstreams: duplicate upstream id ${upstream.id} at indexes 0 and 1`); + assertEquals(await repo.upstreams.list(), []); +}); + test('import preserves staged intra-record error precedence', async () => { const { app } = setup(); const badUpstream = await doImport(app, 'replace', latestImportData({ diff --git a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts index ce0777636..7dec9d0cb 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts @@ -15,8 +15,8 @@ vi.mock('../../../src/data-plane/providers/models-refresh.ts', () => ({ clearModelsRefreshesForTesting: () => {}, })); -import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; +import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; import { buildCopilotUpstreamRecord, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, assertStringIncludes, jsonResponse, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 6b9b13780..0c37d8207 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -338,6 +338,23 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { expect((await storedCache(repo))?.models).toEqual([aModel('published-model')]); }); + test('persistent finalize errors release the claim for a later refresh', async () => { + const repo = await setupRepo(); + vi.spyOn(repo.upstreams, 'finalizeModelsRefreshSuccess').mockRejectedValue(new Error('storage unavailable')); + + await expect(fetchUpstreamModels(stubInstance(async () => [aModel('unpublished')]), directFetcher)) + .rejects.toThrow('Failed to finalize models refresh'); + await expect(repo.upstreams.claimModelsRefresh({ + id: UPSTREAM_ID, + generation: CACHE_GENERATION, + token: 'next-owner', + now: Date.now(), + staleClaimedBefore: Number.MIN_SAFE_INTEGER, + bypassBackoff: false, + observedActiveToken: null, + })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + }); + test('a superseded generation neither joins nor overwrites the current catalog', async () => { const repo = await setupRepo(); let resolveOld: ((models: ProviderModel[]) => void) | null = null; @@ -397,10 +414,13 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); const explicitFetch = vi.fn(async () => [aModel('explicit-recovery-model')]); - const explicit = fetchUpstreamModels(stubInstance(explicitFetch), directFetcher); + const explicitInstance = stubInstance(explicitFetch); + const firstExplicit = fetchUpstreamModels(explicitInstance, directFetcher); + const secondExplicit = fetchUpstreamModels(explicitInstance, directFetcher); rejectOld!(new Error('late old failure')); await expect(oldScheduled.promises[0]).rejects.toThrow('late old failure'); - await expect(explicit).resolves.toEqual([aModel('explicit-recovery-model')]); + await expect(firstExplicit).resolves.toEqual([aModel('explicit-recovery-model')]); + await expect(secondExplicit).resolves.toEqual([aModel('explicit-recovery-model')]); expect(explicitFetch).toHaveBeenCalledOnce(); expect(await storedCache(repo)).toMatchObject({ models: [{ id: 'explicit-recovery-model' }], lastError: null }); }); diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index c1fd090eb..4ef773f05 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -29,10 +29,10 @@ import type { AgentSetupRenewal, AgentSetupRepository, BackoffRow, - ModelsCacheGeneration, ModelsRefreshClaimInput, ModelsRefreshClaimResult, ModelsRefreshFailureInput, + ModelsRefreshOwnerInput, ModelsRefreshSuccessInput, ModelAliasesRepo, ModelAliasRecord, @@ -74,7 +74,7 @@ import { bucketForTtftMs, bucketForTpotUs } from '../../src/shared/performance-h import { assertWebSearchProviderName, type WebSearchConfig } from '../../src/shared/web-search-providers.ts'; import { AgentSetupTokenCollisionError } from '@floway-dev/agent-setup'; import { addDecimalStrings, canonicalPricingSelectorKey, canonicalizePricingSelector, multiplyDecimalStrings, tokenUsageUnattributedUserId, usageUpstreamDimensionValue, type BillingMetric, type DecimalString, type PricingSelector } from '@floway-dev/protocols/common'; -import { UpstreamGoneError, type UpstreamModelsCache, type UpstreamRecord } from '@floway-dev/provider'; +import { UpstreamGoneError, type UpstreamRecord } from '@floway-dev/provider'; const SEED_ADMIN_USER: User = { id: SEED_ADMIN_USER_ID, @@ -844,6 +844,18 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(true); } + abandonModelsRefresh(input: ModelsRefreshOwnerInput): Promise { + const { id, generation, token } = input; + const existing = this.store.get(id); + const refresh = this.modelsRefreshes.get(id); + if (!existing + || existing.updatedAt !== generation.updatedAt + || modelsFetchIdentity(existing) !== generation.fetchIdentity + || refresh?.claimToken !== token) return Promise.resolve(false); + this.modelsRefreshes.set(id, { ...refresh, claimToken: null, claimedAt: null }); + return Promise.resolve(true); + } + claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { const { id, generation, token, now, staleClaimedBefore, bypassBackoff, observedActiveToken } = input; const stored = this.store.get(id); diff --git a/packages/gateway/src/control-plane/data-transfer/import-schema.ts b/packages/gateway/src/control-plane/data-transfer/import-schema.ts index be5453257..0ebf9f7d5 100644 --- a/packages/gateway/src/control-plane/data-transfer/import-schema.ts +++ b/packages/gateway/src/control-plane/data-transfer/import-schema.ts @@ -487,6 +487,14 @@ export const parseImportData = (value: unknown): ImportDataParseResult => { if (usage.type === 'invalid') return usage; const upstreams = parseCollection('upstreams', upstreamWireSchema, value.upstreams, { arrayError: 'upstreams must be an array' }); if (upstreams.type === 'invalid') return upstreams; + const upstreamIds = new Map(); + for (let index = 0; index < upstreams.records.length; index++) { + const prior = upstreamIds.get(upstreams.records[index].id); + if (prior !== undefined) { + return { type: 'invalid', error: `invalid upstreams: duplicate upstream id ${upstreams.records[index].id} at indexes ${prior} and ${index}` }; + } + upstreamIds.set(upstreams.records[index].id, index); + } const proxies = parseCollection('proxies', proxySchema, value.proxies, { arrayError: 'proxies must be an array', optional: true }); if (proxies.type === 'invalid') return proxies; const proxyIds = new Map(); diff --git a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts index 01605f608..f0f3e80b0 100644 --- a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts +++ b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts @@ -1,9 +1,10 @@ import type { Context } from 'hono'; import { warmUpstreamModels } from '../../data-plane/providers/models-refresh.ts'; -import { modelsCatalogIdentity, modelsOperatorRefreshIdentity, createProvider } from '../../data-plane/providers/registry.ts'; +import { modelsCatalogIdentity, createProvider } from '../../data-plane/providers/registry.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; import { getRepo } from '../../repo/index.ts'; +import { modelsOperatorRefreshIdentity } from '../../repo/models-cache-contract.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { UpstreamModelsCache, UpstreamRecord } from '@floway-dev/provider'; import { logInfo } from '@floway-dev/provider-claude-code'; @@ -34,10 +35,17 @@ export const saveAndWarmUpstreamsForModels = async ( changes: readonly UpstreamModelsChange[], c: Context, ): Promise> => { + if (new Set(changes.map(change => change.next.id)).size !== changes.length) { + throw new Error('Duplicate upstream ids in models save batch'); + } for (const change of changes) await saveUpstreamForModels(change); if (changes.length === 0) return new Map(); - const records = changes.map(change => change.next); + const records = await Promise.all(changes.map(async change => { + const record = await getRepo().upstreams.getById(change.next.id); + if (record === null) throw new Error(`Upstream ${change.next.id} disappeared after save`); + return record; + })); const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw), records); const entries = await Promise.all(records.map(async record => { try { diff --git a/packages/gateway/src/control-plane/upstreams/models.ts b/packages/gateway/src/control-plane/upstreams/models.ts index 03c537c9f..ef7602d6f 100644 --- a/packages/gateway/src/control-plane/upstreams/models.ts +++ b/packages/gateway/src/control-plane/upstreams/models.ts @@ -3,7 +3,7 @@ import { isValidProviderKind, upstreamErrorMessage as errorMessage } from './sha import type { ListedUpstreamModel } from './types.ts'; import { MODEL_LISTING_FAILURE_CODE, MODEL_LISTING_FAILURE_MESSAGE } from '../../data-plane/models/shared.ts'; import { fetchUpstreamModels } from '../../data-plane/providers/models-refresh.ts'; -import { createProvider, modelsOperatorRefreshIdentity } from '../../data-plane/providers/registry.ts'; +import { createProvider, modelsRequestIdentity } from '../../data-plane/providers/registry.ts'; import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { modelsCacheGeneration } from '../../repo/models-cache-contract.ts'; @@ -48,6 +48,7 @@ export const listModels = async (c: CtxWithJson) => { const kind = record.kind; const persisted = record.id === '' ? null : await getRepo().upstreams.getById(record.id); if (record.id !== '' && persisted === null) return c.json({ error: 'Upstream not found' }, 404); + const effectiveProxyFallbackList = (record.proxy_fallback_list ?? persisted?.proxyFallbackList ?? []) as ProxyFallbackEntry[]; const now = new Date().toISOString(); const synthRecord: UpstreamRecord = { @@ -60,7 +61,7 @@ export const listModels = async (c: CtxWithJson) => { updatedAt: persisted?.updatedAt ?? now, flagOverrides: {}, disabledPublicModelIds: [], - proxyFallbackList: (record.proxy_fallback_list ?? []) as ProxyFallbackEntry[], + proxyFallbackList: effectiveProxyFallbackList, modelPrefix: null, // A draft only lists models; nothing renders its badge. hue: 0, @@ -71,12 +72,12 @@ export const listModels = async (c: CtxWithJson) => { modelsCache: null, }; const canRefreshPersistedCache = persisted !== null - && modelsOperatorRefreshIdentity(persisted) === modelsOperatorRefreshIdentity(synthRecord); + && modelsRequestIdentity(persisted) === modelsRequestIdentity(synthRecord); let fetcher: Fetcher; try { fetcher = await resolveControlPlaneFetcher({ - override: record.proxy_fallback_list, + override: effectiveProxyFallbackList, upstreamId: record.id || undefined, runtimeLocation: getRuntimeLocation(c.req.raw), }); diff --git a/packages/gateway/src/data-plane/providers/models-refresh.ts b/packages/gateway/src/data-plane/providers/models-refresh.ts index 55725aaed..e6dc89354 100644 --- a/packages/gateway/src/data-plane/providers/models-refresh.ts +++ b/packages/gateway/src/data-plane/providers/models-refresh.ts @@ -47,7 +47,10 @@ const memoInFlight = ( const errorMessage = (err: unknown): string => err instanceof Error ? err.message : String(err); -const finalizeRefresh = async (finalize: () => Promise): Promise => { +const finalizeRefresh = async ( + finalize: () => Promise, + abandon: () => Promise, +): Promise => { const errors: unknown[] = []; for (let attempt = 0; attempt < 3; attempt++) { try { @@ -56,6 +59,11 @@ const finalizeRefresh = async (finalize: () => Promise): Promise await repo.upstreams.finalizeModelsRefreshFailure({ - id: instance.upstreamId, - generation: instance.modelsCacheGeneration, - token, - error: lastError, - previousFailureCount: outcome.failureCount, - failedAt, - })); + finalized = await finalizeRefresh( + async () => await repo.upstreams.finalizeModelsRefreshFailure({ + id: instance.upstreamId, + generation: instance.modelsCacheGeneration, + token, + error: lastError, + previousFailureCount: outcome.failureCount, + failedAt, + }), + async () => await repo.upstreams.abandonModelsRefresh({ id: instance.upstreamId, generation: instance.modelsCacheGeneration, token }), + ); } catch (backoffError) { throw new AggregateError([error, backoffError], errorMessage(error)); } @@ -138,12 +149,15 @@ const runClaimedRefresh = async ( continue; } const entry = { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models, lastError: null }; - const finalized = await finalizeRefresh(async () => await repo.upstreams.finalizeModelsRefreshSuccess({ - id: instance.upstreamId, - generation: instance.modelsCacheGeneration, - token, - cache: entry, - })); + const finalized = await finalizeRefresh( + async () => await repo.upstreams.finalizeModelsRefreshSuccess({ + id: instance.upstreamId, + generation: instance.modelsCacheGeneration, + token, + cache: entry, + }), + async () => await repo.upstreams.abandonModelsRefresh({ id: instance.upstreamId, generation: instance.modelsCacheGeneration, token }), + ); if (finalized) { // The instance is reused across alias targets in one request, so publish // the finalized snapshot locally as well as durably. @@ -182,6 +196,7 @@ export const fetchUpstreamModels = async ( // failure, and bypasses the cooldown that failure just established. } if (inFlight.get(key) === existing) inFlight.delete(key); + continue; } const models = await startInFlight(key, 'explicit-refresh', () => runClaimedRefresh(instance, fetcher, 'explicit', loadProvidedModels)); if (models === null) throw new Error(`Failed to acquire models refresh for ${instance.upstreamId}`); diff --git a/packages/gateway/src/data-plane/providers/registry.ts b/packages/gateway/src/data-plane/providers/registry.ts index 9e0bfdcdf..04d786b0a 100644 --- a/packages/gateway/src/data-plane/providers/registry.ts +++ b/packages/gateway/src/data-plane/providers/registry.ts @@ -26,10 +26,10 @@ export type GatewayProvider = Provider & { export const modelsCatalogIdentity = (record: UpstreamRecord): string => serializeStoredConfig({ kind: record.kind, identity: providersByKind[record.kind].modelCatalogIdentity(record) }); -export const modelsOperatorRefreshIdentity = (record: UpstreamRecord): string => +export const modelsRequestIdentity = (record: UpstreamRecord): string => serializeStoredConfig({ kind: record.kind, - identity: providersByKind[record.kind].modelRefreshIdentity(record), + identity: providersByKind[record.kind].modelRequestIdentity(record), proxyFallbackList: record.proxyFallbackList, }); diff --git a/packages/gateway/src/index.ts b/packages/gateway/src/index.ts index 256ae2ad7..25511037a 100644 --- a/packages/gateway/src/index.ts +++ b/packages/gateway/src/index.ts @@ -2,7 +2,7 @@ export { app } from './app.ts'; export { initRepo } from './repo/index.ts'; export { FileDumpStore } from './repo/dump-store.ts'; export { SqlRepo } from './repo/sql.ts'; -export { MODEL_CATALOG_REVISION } from './repo/models-cache-contract.ts'; +export { MODEL_CATALOG_REVISION, modelsCacheGeneration } from './repo/models-cache-contract.ts'; export { initBackgroundSchedulerResolver } from './runtime/background.ts'; export { initDumpBroker, initDumpStore } from './dump/registry.ts'; export { initResponsesWebSocketUpgradeResolver } from './data-plane/chat/responses/websocket.ts'; diff --git a/packages/gateway/src/repo/models-cache-contract.ts b/packages/gateway/src/repo/models-cache-contract.ts index 89435a77a..ad0f35c6e 100644 --- a/packages/gateway/src/repo/models-cache-contract.ts +++ b/packages/gateway/src/repo/models-cache-contract.ts @@ -17,6 +17,17 @@ export const modelsFetchIdentity = ( proxyFallbackList: record.proxyFallbackList, }); +// Control-plane credential and transport changes reset refresh cooldown even +// when the provider says the previous catalog remains valid. +export const modelsOperatorRefreshIdentity = ( + record: Pick, +): string => serializeStoredConfig({ + kind: record.kind, + config: record.config, + state: record.state ?? null, + proxyFallbackList: record.proxyFallbackList, +}); + export const modelsCacheGeneration = ( record: Pick, ): ModelsCacheGeneration => ({ diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index 5b368fbaf..a5de8cbba 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -24,6 +24,7 @@ import type { ModelsRefreshClaimInput, ModelsRefreshClaimResult, ModelsRefreshFailureInput, + ModelsRefreshOwnerInput, ModelsRefreshSuccessInput, ModelAliasesRepo, ModelAliasRecord, @@ -1078,6 +1079,17 @@ class SqlUpstreamRepo implements UpstreamRepo { return (result.meta.changes ?? 0) > 0; } + async abandonModelsRefresh(input: ModelsRefreshOwnerInput): Promise { + const { id, generation, token } = input; + const fence = await this.modelsRefreshWriteFence(id, generation); + if (fence === null) return false; + const result = await this.db + .prepare("UPDATE upstreams SET models_refresh_json = json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") + .bind(id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, token) + .run(); + return (result.meta.changes ?? 0) > 0; + } + async claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { const { id, generation, token, now, staleClaimedBefore, bypassBackoff, observedActiveToken } = input; const fence = await this.modelsRefreshWriteFence(id, generation); diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index 6c89b6246..09f43e64c 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -364,6 +364,7 @@ export interface UpstreamRepo { // cannot publish models or errors under newer credentials/configuration. finalizeModelsRefreshSuccess(input: ModelsRefreshSuccessInput): Promise; finalizeModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise; + abandonModelsRefresh(input: ModelsRefreshOwnerInput): Promise; claimModelsRefresh(input: ModelsRefreshClaimInput): Promise; } @@ -384,6 +385,12 @@ export interface ModelsRefreshSuccessInput { cache: Omit; } +export interface ModelsRefreshOwnerInput { + id: string; + generation: ModelsCacheGeneration; + token: string; +} + export interface ModelsRefreshFailureInput { id: string; generation: ModelsCacheGeneration; diff --git a/packages/provider-azure/src/index.ts b/packages/provider-azure/src/index.ts index 29f836ed8..640c54fda 100644 --- a/packages/provider-azure/src/index.ts +++ b/packages/provider-azure/src/index.ts @@ -6,7 +6,7 @@ import type { ProviderModule } from '@floway-dev/provider'; export const azureProviderModule: ProviderModule = { create: createAzureProvider, modelCatalogIdentity: record => assertAzureUpstreamRecord(record).config, - modelRefreshIdentity: record => assertAzureUpstreamRecord(record).config, + modelRequestIdentity: record => assertAzureUpstreamRecord(record).config, defaultFlags: AZURE_DEFAULT_FLAGS, }; export { assertAzureUpstreamRecord, type AzureUpstreamConfig } from './config.ts'; diff --git a/packages/provider-claude-code/src/index.ts b/packages/provider-claude-code/src/index.ts index 17d512b1e..66dbbbb78 100644 --- a/packages/provider-claude-code/src/index.ts +++ b/packages/provider-claude-code/src/index.ts @@ -9,9 +9,9 @@ export const claudeCodeProviderModule: ProviderModule = { assertClaudeCodeUpstreamRecord(record); return record.config; }, - modelRefreshIdentity: record => { + modelRequestIdentity: record => { assertClaudeCodeUpstreamRecord(record); - return { config: record.config, state: record.state }; + return record.config; }, defaultFlags: CLAUDE_CODE_DEFAULT_FLAGS, }; diff --git a/packages/provider-codex/src/index.ts b/packages/provider-codex/src/index.ts index 233417396..e03e4d078 100644 --- a/packages/provider-codex/src/index.ts +++ b/packages/provider-codex/src/index.ts @@ -9,9 +9,9 @@ export const codexProviderModule: ProviderModule = { assertCodexUpstreamRecord(record); return record.config; }, - modelRefreshIdentity: record => { + modelRequestIdentity: record => { assertCodexUpstreamRecord(record); - return { config: record.config, state: record.state }; + return record.config; }, defaultFlags: CODEX_DEFAULT_FLAGS, }; diff --git a/packages/provider-copilot/src/index.ts b/packages/provider-copilot/src/index.ts index 883c47959..aee7fdd59 100644 --- a/packages/provider-copilot/src/index.ts +++ b/packages/provider-copilot/src/index.ts @@ -9,9 +9,9 @@ export const copilotProviderModule: ProviderModule = { const upstream = assertCopilotUpstreamRecord(record); return { githubHost: upstream.config.githubHost, userId: upstream.config.user.id }; }, - modelRefreshIdentity: record => { + modelRequestIdentity: record => { const upstream = assertCopilotUpstreamRecord(record); - return { config: upstream.config, state: upstream.state }; + return upstream.config; }, defaultFlags: COPILOT_DEFAULT_FLAGS, }; diff --git a/packages/provider-custom/src/index.ts b/packages/provider-custom/src/index.ts index ec62e620f..ec41e2929 100644 --- a/packages/provider-custom/src/index.ts +++ b/packages/provider-custom/src/index.ts @@ -6,7 +6,7 @@ import type { ProviderModule } from '@floway-dev/provider'; export const customProviderModule: ProviderModule = { create: createCustomProvider, modelCatalogIdentity: record => assertCustomUpstreamRecord(record).config, - modelRefreshIdentity: record => assertCustomUpstreamRecord(record).config, + modelRequestIdentity: record => assertCustomUpstreamRecord(record).config, defaultFlags: CUSTOM_DEFAULT_FLAGS, }; diff --git a/packages/provider-ollama/src/index.ts b/packages/provider-ollama/src/index.ts index 3d2ddb159..6ef118e80 100644 --- a/packages/provider-ollama/src/index.ts +++ b/packages/provider-ollama/src/index.ts @@ -6,7 +6,7 @@ import type { ProviderModule } from '@floway-dev/provider'; export const ollamaProviderModule: ProviderModule = { create: createOllamaProvider, modelCatalogIdentity: record => assertOllamaUpstreamRecord(record).config, - modelRefreshIdentity: record => assertOllamaUpstreamRecord(record).config, + modelRequestIdentity: record => assertOllamaUpstreamRecord(record).config, defaultFlags: OLLAMA_DEFAULT_FLAGS, }; diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index efa43b358..e88ef4137 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -169,9 +169,9 @@ export interface ProviderModule { // Stable identity of the upstream account/catalog namespace. Each provider // decides which of its configuration changes can preserve a snapshot. modelCatalogIdentity: (record: UpstreamRecord) => unknown; - // Normalized inputs an operator-controlled refresh would use. Changes reset - // refresh cooldown even when the last-known-good catalog remains valid. - modelRefreshIdentity: (record: UpstreamRecord) => unknown; + // Normalized request inputs captured by the provider instance. Provider- + // managed state is reread from storage when the catalog request runs. + modelRequestIdentity: (record: UpstreamRecord) => unknown; // Exhaustive default map over every catalog flag id for a fresh // upstream of this kind; see each provider package's `defaults.ts`. defaultFlags: FlagDefaults; From 552767de46b0f946b5fed2c51f6a2067cfb17b92 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 19:44:58 +0800 Subject: [PATCH 42/46] refactor(gateway): version upstream catalog configuration Replace provider-specific catalog/request identity projections with a persisted configVersion owned by the upstream repository. Split saved catalog refresh from draft preview so draft values can never publish a cache snapshot. Fence refresh coordination solely by the stored config generation while allowing runtime state and metadata writes to preserve it. --- .../__tests__/node-sqlite-repo_test.ts | 1 + .../components/upstream-editor/data_test.ts | 12 ++- apps/web/src/api/types.ts | 2 +- .../src/components/upstream-editor/data.ts | 29 +++-- .../src/components/upstream-editor/page.tsx | 6 +- .../data-transfer/routes_test.ts | 5 + .../control-plane/models/routes_test.ts | 1 + .../control-plane/upstreams/routes_test.ts | 61 +++++++---- .../control-plane/upstreams/serialize_test.ts | 3 + .../__tests__/data-plane/audio/http_test.ts | 1 + .../affinity/copilot-roundtrip_test.ts | 1 + .../image-generation-integration_test.ts | 1 + .../chat/shared/target-picker_test.ts | 1 + .../data-plane/codex/routes_images_test.ts | 1 + .../__tests__/data-plane/images/http_test.ts | 1 + .../data-plane/providers/catalog_test.ts | 1 + .../data-plane/providers/models-cache_test.ts | 23 ++-- .../data-plane/providers/registry_test.ts | 14 +-- .../data-plane/providers/resolution_test.ts | 1 + .../__tests__/dial/per-request_test.ts | 1 + packages/gateway/__tests__/repo/memory.ts | 47 +++++--- .../__tests__/repo/models-refresh_test.ts | 41 ++++--- .../gateway/__tests__/repo/proxies_test.ts | 1 + packages/gateway/__tests__/repo/sql_test.ts | 3 +- .../gateway/__tests__/repo/upstreams_test.ts | 17 ++- .../scheduled/models-refresh_test.ts | 1 + packages/gateway/__tests__/test-utils/app.ts | 2 + .../0079_upstream_config_version.sql | 4 + .../data-transfer/import-schema.ts | 3 + packages/gateway/src/control-plane/routes.ts | 7 +- packages/gateway/src/control-plane/schemas.ts | 12 +-- .../shared/save-upstream-for-models.ts | 22 ++-- .../src/control-plane/upstreams/models.ts | 102 +++++++++++------- .../src/control-plane/upstreams/routes.ts | 9 +- .../gateway/src/data-plane/models/shared.ts | 2 +- .../data-plane/providers/models-refresh.ts | 7 +- .../src/data-plane/providers/registry.ts | 11 -- .../gateway/src/repo/models-cache-contract.ts | 25 +---- packages/gateway/src/repo/sql.ts | 99 ++++++++--------- packages/gateway/src/repo/types.ts | 6 +- .../provider-azure/__tests__/config_test.ts | 1 + .../provider-azure/__tests__/fetch_test.ts | 1 + .../provider-azure/__tests__/provider_test.ts | 3 + packages/provider-azure/src/index.ts | 3 - .../__tests__/access-token_test.ts | 1 + .../__tests__/config_test.ts | 4 +- .../__tests__/fetch_test.ts | 1 + .../__tests__/provider_test.ts | 1 + packages/provider-claude-code/src/index.ts | 9 -- .../__tests__/access-token_test.ts | 1 + .../provider-codex/__tests__/config_test.ts | 4 +- .../provider-codex/__tests__/fetch_test.ts | 1 + .../responses/action-pivot_test.ts | 1 + .../provider-codex/__tests__/provider_test.ts | 1 + .../provider-codex/__tests__/quota_test.ts | 1 + packages/provider-codex/src/index.ts | 9 -- .../provider-copilot/__tests__/auth_test.ts | 2 + .../__tests__/fetch-models_test.ts | 1 + .../responses/action-pivot_test.ts | 1 + .../__tests__/provider_test.ts | 1 + packages/provider-copilot/src/index.ts | 9 -- .../provider-custom/__tests__/config_test.ts | 1 + .../__tests__/fetch-models_test.ts | 1 + .../provider-custom/__tests__/fetch_test.ts | 1 + .../__tests__/infer-endpoints_test.ts | 1 + .../__tests__/provider_test.ts | 1 + packages/provider-custom/src/index.ts | 3 - .../provider-ollama/__tests__/config_test.ts | 1 + .../__tests__/fetch-models_test.ts | 1 + .../provider-ollama/__tests__/fetch_test.ts | 1 + .../__tests__/provider_test.ts | 1 + packages/provider-ollama/src/index.ts | 3 - packages/provider/src/model-config.ts | 4 +- packages/provider/src/model.ts | 3 + packages/provider/src/provider.ts | 6 -- 75 files changed, 371 insertions(+), 300 deletions(-) create mode 100644 packages/gateway/migrations/0079_upstream_config_version.sql diff --git a/apps/platform-node/__tests__/node-sqlite-repo_test.ts b/apps/platform-node/__tests__/node-sqlite-repo_test.ts index 38942de7e..f092e0c46 100644 --- a/apps/platform-node/__tests__/node-sqlite-repo_test.ts +++ b/apps/platform-node/__tests__/node-sqlite-repo_test.ts @@ -87,6 +87,7 @@ test('repository JSON codecs round-trip upstream, alias, and Responses state thr updatedAt: '2026-08-05T00:00:00.000Z', config: { opaque: { value: true } }, state: { cursor: ['a', 1] }, + configVersion: 1, modelsCache: null, flagOverrides: {}, disabledPublicModelIds: [], diff --git a/apps/web/__tests__/components/upstream-editor/data_test.ts b/apps/web/__tests__/components/upstream-editor/data_test.ts index c3de1d985..65770035e 100644 --- a/apps/web/__tests__/components/upstream-editor/data_test.ts +++ b/apps/web/__tests__/components/upstream-editor/data_test.ts @@ -1,7 +1,7 @@ import { expect, test } from 'vitest'; import type { UpstreamRecord } from '../../../src/api/types'; -import { createBody, previewRecord, updateBody, valuesFromRecord } from '../../../src/components/upstream-editor/data'; +import { createBody, modelCatalogOperation, previewRecord, updateBody, valuesFromRecord } from '../../../src/components/upstream-editor/data'; import { upstreamRecord } from '../../api/upstream-fixture'; type CustomRecord = Extract; @@ -44,3 +44,13 @@ test('Custom editor values add one blank ingress row and never serialize it', () expect((updateBody(record, values).config as CustomRecord['config']).ingressHeadersRules).toEqual(expected); expect((previewRecord(record, values).config as CustomRecord['config']).ingressHeadersRules).toEqual(expected); }); + +test('model catalog operations write cache only for the unchanged saved config', () => { + expect(modelCatalogOperation(record, {})).toBe('saved'); + expect(modelCatalogOperation(record, { config: true })).toBe('preview'); + expect(modelCatalogOperation(record, { state: true })).toBe('preview'); + expect(modelCatalogOperation(record, { proxyFallbackList: true })).toBe('preview'); + expect(modelCatalogOperation({ ...record, id: '' }, {})).toBe('preview'); + // Metadata does not alter the provider request inputs. + expect(modelCatalogOperation(record, { name: true })).toBe('saved'); +}); diff --git a/apps/web/src/api/types.ts b/apps/web/src/api/types.ts index fbfeef223..1706fde00 100644 --- a/apps/web/src/api/types.ts +++ b/apps/web/src/api/types.ts @@ -16,7 +16,7 @@ export type { } from '@floway-dev/gateway/control-plane/upstreams/types'; export type UpstreamRecordEnvelope = InferRequestType< - typeof api.api.upstreams['list-models']['$post'] + typeof api.api.upstreams['preview-models']['$post'] >['json']['record']; export type ProxyRecord = SerializedProxyRecord; diff --git a/apps/web/src/components/upstream-editor/data.ts b/apps/web/src/components/upstream-editor/data.ts index 06d53e980..889e6e909 100644 --- a/apps/web/src/components/upstream-editor/data.ts +++ b/apps/web/src/components/upstream-editor/data.ts @@ -48,6 +48,16 @@ export type UpstreamEditorLoaderData = UpstreamEditorLoaderDataBase & ( // record. export const isPersisted = (record: UpstreamRecord): boolean => record.id !== ''; +export const modelCatalogOperation = ( + record: UpstreamRecord, + dirtyFields: Partial>, +): 'saved' | 'preview' => isPersisted(record) + && !dirtyFields.config + && !dirtyFields.state + && !dirtyFields.proxyFallbackList + ? 'saved' + : 'preview'; + // `hasAuto` says the upstream also lists the model, which is what makes // switching the row back to `auto` possible. export interface ModelRow { @@ -148,18 +158,23 @@ export interface ModelListingFailure { upstreamListingFailed: boolean; } -// Listing re-reads the upstream afterwards: the server writes its models cache -// as a side effect of the call, and the record the editor holds carries it. +interface ModelCatalogFetchOptions extends RequestInit { + operation?: 'saved' | 'preview'; +} + export const fetchModelCatalog = async ( record: UpstreamRecord, values: UpstreamEditorValues, - init?: RequestInit, + options: ModelCatalogFetchOptions = {}, ): Promise => { if (!canFetchModelCatalog(record, values.config)) return { discovered: null, modelsError: null, refreshed: null }; - const result = await callApi(() => api.api.upstreams['list-models'].$post({ - json: { record: previewRecord(record, values) }, - }, { init })); + const { operation = modelCatalogOperation(record, {}), ...init } = options; + const result = operation === 'saved' + ? await callApi(() => api.api.upstreams[':id']['list-models'].$post({ param: { id: record.id } }, { init })) + : await callApi(() => api.api.upstreams['preview-models'].$post({ + json: { record: previewRecord(record, values) }, + }, { init })); if (result.error) { return { discovered: null, @@ -175,7 +190,7 @@ export const fetchModelCatalog = async ( ? (values.config as Extract['config']).endpoints : {}; const discovered = discoveredModelsFromResponse(result.data, endpoints); - if (!isPersisted(record)) return { discovered, modelsError: null, refreshed: null }; + if (operation === 'preview') return { discovered, modelsError: null, refreshed: null }; const refreshed = await callApi(() => api.api.upstreams[':id'].$get({ param: { id: record.id } }, { init })); return refreshed.error diff --git a/apps/web/src/components/upstream-editor/page.tsx b/apps/web/src/components/upstream-editor/page.tsx index 3ed0b8d37..0474a1f31 100644 --- a/apps/web/src/components/upstream-editor/page.tsx +++ b/apps/web/src/components/upstream-editor/page.tsx @@ -10,6 +10,7 @@ import { refineCustomIngressHeaderRules } from './custom-ingress-header-rules-va import { createBody, fetchModelCatalog, + modelCatalogOperation, modelPrefixIsValid, updateBody, valuesFromRecord, @@ -137,12 +138,13 @@ export function UpstreamEditorPage({ data }: { data: UpstreamEditorLoaderData }) // reach this, so runs can overlap; `useRefresh` aborts the superseded one. const { refresh: refreshModels, refreshing: modelsLoading } = useRefresh(useCallback(async (signal: AbortSignal) => { setModelsError(null); - const catalog = await fetchModelCatalog(record, getValues(), { signal }); + const operation = modelCatalogOperation(record, formState.dirtyFields); + const catalog = await fetchModelCatalog(record, getValues(), { operation, signal }); if (signal.aborted) return; setModelsError(catalog.modelsError); if (catalog.discovered) setDiscovered(catalog.discovered); if (catalog.refreshed) updateRecord({ ...recordRef.current, modelsCache: catalog.refreshed.modelsCache } as UpstreamRecord); - }, [getValues, record, updateRecord])); + }, [formState.dirtyFields, getValues, record, updateRecord])); const applyProviderPatch = (patch: { config?: unknown; state?: unknown }, persisted = false) => { if (patch.config !== undefined) setValue('config', patch.config as UpstreamEditorValues['config'], { shouldDirty: !persisted }); diff --git a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts index f5f6b8912..ee55fa2e4 100644 --- a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts @@ -86,6 +86,7 @@ const CUSTOM_UPSTREAM: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { @@ -114,6 +115,7 @@ const COPILOT_UPSTREAM: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { @@ -141,6 +143,7 @@ const AZURE_UPSTREAM: UpstreamRecord = { disabledPublicModelIds: ['gpt-public'], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { @@ -175,6 +178,7 @@ const OLLAMA_UPSTREAM: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { @@ -203,6 +207,7 @@ const CODEX_UPSTREAM: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/control-plane/models/routes_test.ts b/packages/gateway/__tests__/control-plane/models/routes_test.ts index da31c469a..2b71d9242 100644 --- a/packages/gateway/__tests__/control-plane/models/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/models/routes_test.ts @@ -16,6 +16,7 @@ const azureUpstream = (): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts index 8127ff783..0ddbdebc9 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts @@ -260,6 +260,7 @@ test('PATCH /api/upstreams rejects kind changes and preserves the row', async () const create = await requestApp('/api/upstreams', authed(adminSession, createBody())); const created = (await create.json()) as Record; + assertEquals((await repo.upstreams.getById(created.id))?.configVersion, 1); const patch = await requestApp(`/api/upstreams/${created.id}`, { method: 'PATCH', @@ -318,6 +319,7 @@ test('PATCH /api/upstreams preserves omitted secrets and re-warms the models cac const updated = await repo.upstreams.getById(created.id); assertEquals((updated?.config as Record).apiKey, 'sk-test'); + assertEquals(updated?.configVersion, 2); assertEquals((updated?.config as Record).endpoints, { responses: {} }); assertEquals((updated?.config as Record).ingressHeadersRules, [{ key: 'x-route', value: 'patched' }]); @@ -341,6 +343,7 @@ test('PATCH /api/upstreams keeps Azure as a single endpoint config', async () => disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { @@ -388,6 +391,7 @@ test('PATCH /api/upstreams round-trips a flat per-model flagOverrides map', asyn disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { @@ -433,6 +437,7 @@ test('GET /api/upstreams attaches models-cache freshness to every row', async () disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { baseUrl: 'https://a.example.com', authStyle: 'bearer', apiKey: 'x', endpoints: { chatCompletions: {} }, ingressHeadersRules: [] }, @@ -485,6 +490,7 @@ test('GET /api/upstream-options returns the minimal picker shape to admin and no disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { baseUrl: 'https://custom.example.com', authStyle: 'bearer', apiKey: 'sk-secret', endpoints: { chatCompletions: {} } }, @@ -520,7 +526,7 @@ test('GET /api/upstream-options returns the minimal picker shape to admin and no } }); -test('POST /api/upstreams/list-models fetches a draft custom upstream model list', async () => { +test('POST /api/upstreams/preview-models fetches a draft custom upstream model list', async () => { const { adminSession } = await setupAppTest(); await withMockedFetch( @@ -533,7 +539,7 @@ test('POST /api/upstreams/list-models fetches a draft custom upstream model list throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const resp = await requestApp('/api/upstreams/list-models', authed(adminSession, { + const resp = await requestApp('/api/upstreams/preview-models', authed(adminSession, { record: blueprintEnvelope('custom', { config: customConfig }), })); assertEquals(resp.status, 200); @@ -544,7 +550,7 @@ test('POST /api/upstreams/list-models fetches a draft custom upstream model list ); }); -test('POST /api/upstreams/list-models projects an ollama draft into UpstreamModelConfig rows with capability-derived endpoints', async () => { +test('POST /api/upstreams/preview-models projects an ollama draft into UpstreamModelConfig rows with capability-derived endpoints', async () => { const { adminSession } = await setupAppTest(); await withMockedFetch( @@ -573,7 +579,7 @@ test('POST /api/upstreams/list-models projects an ollama draft into UpstreamMode throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const resp = await requestApp('/api/upstreams/list-models', authed(adminSession, { + const resp = await requestApp('/api/upstreams/preview-models', authed(adminSession, { record: blueprintEnvelope('ollama', { config: { baseUrl: 'https://ollama.com', apiKey: 'ollama_test' }, }), @@ -592,7 +598,7 @@ test('POST /api/upstreams/list-models projects an ollama draft into UpstreamMode ); }); -test('POST /api/upstreams/list-models surfaces upstream model-listing failures as 502', async () => { +test('POST /api/upstreams/preview-models surfaces upstream model-listing failures as 502', async () => { const { adminSession } = await setupAppTest(); await withMockedFetch( @@ -604,7 +610,7 @@ test('POST /api/upstreams/list-models surfaces upstream model-listing failures a throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const resp = await requestApp('/api/upstreams/list-models', authed(adminSession, { + const resp = await requestApp('/api/upstreams/preview-models', authed(adminSession, { record: blueprintEnvelope('custom', { config: customConfig }), })); assertEquals(resp.status, 502); @@ -615,7 +621,7 @@ test('POST /api/upstreams/list-models surfaces upstream model-listing failures a ); }); -test('POST /api/upstreams/list-models surfaces an ollama /api/tags failure as 502', async () => { +test('POST /api/upstreams/preview-models surfaces an ollama /api/tags failure as 502', async () => { const { adminSession } = await setupAppTest(); await withMockedFetch( @@ -627,7 +633,7 @@ test('POST /api/upstreams/list-models surfaces an ollama /api/tags failure as 50 throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const resp = await requestApp('/api/upstreams/list-models', authed(adminSession, { + const resp = await requestApp('/api/upstreams/preview-models', authed(adminSession, { record: blueprintEnvelope('ollama', { config: { baseUrl: 'https://ollama.com', apiKey: 'ollama_test' }, }), @@ -640,12 +646,12 @@ test('POST /api/upstreams/list-models surfaces an ollama /api/tags failure as 50 ); }); -test('POST /api/upstreams/list-models rejects a malformed draft config with 400', async () => { +test('POST /api/upstreams/preview-models rejects a malformed draft config with 400', async () => { const { adminSession } = await setupAppTest(); // Blank token with no id and no stored secret to substitute: the runtime // assert rejects the empty apiKey, surfaced as a 400 validation error. - const resp = await requestApp('/api/upstreams/list-models', authed(adminSession, { + const resp = await requestApp('/api/upstreams/preview-models', authed(adminSession, { record: blueprintEnvelope('custom', { config: { ...customConfig, apiKey: '' } }), })); assertEquals(resp.status, 400); @@ -653,7 +659,7 @@ test('POST /api/upstreams/list-models rejects a malformed draft config with 400' assertEquals(body.error.includes('apiKey'), true); }); -test('POST /api/upstreams/list-models with matching saved inputs fetches and publishes a fresh snapshot', async () => { +test('POST /api/upstreams/:id/list-models reads the saved config and publishes a fresh snapshot', async () => { const { repo, adminSession } = await setupAppTest(); await repo.upstreams.deleteAll(); const savedRecord: UpstreamRecord = { @@ -668,6 +674,7 @@ test('POST /api/upstreams/list-models with matching saved inputs fetches and pub disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { ...customConfig, apiKey: 'sk-refresh' }, @@ -686,9 +693,10 @@ test('POST /api/upstreams/list-models with matching saved inputs fetches and pub throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const resp = await requestApp('/api/upstreams/list-models', authed(adminSession, { - record: envelopeFromRecord(savedRecord), - })); + const resp = await requestApp(`/api/upstreams/${savedRecord.id}/list-models`, { + method: 'POST', + headers: { 'x-floway-session': adminSession }, + }); assertEquals(resp.status, 200); const body = (await resp.json()) as { data: Array<{ id?: string }> }; // Custom returns the raw upstream row shape (id-keyed), not the @@ -702,10 +710,19 @@ test('POST /api/upstreams/list-models with matching saved inputs fetches and pub ); }); -test('POST /api/upstreams/list-models rejects an invalid kind with 400', async () => { +test('POST /api/upstreams/:id/list-models rejects a missing saved upstream', async () => { + const { adminSession } = await setupAppTest(); + const response = await requestApp('/api/upstreams/up_missing/list-models', { + method: 'POST', + headers: { 'x-floway-session': adminSession }, + }); + assertEquals(response.status, 404); +}); + +test('POST /api/upstreams/preview-models rejects an invalid kind with 400', async () => { const { adminSession } = await setupAppTest(); - const resp = await requestApp('/api/upstreams/list-models', authed(adminSession, { + const resp = await requestApp('/api/upstreams/preview-models', authed(adminSession, { record: { id: '', kind: 'bogus-kind', config: {}, state: null }, })); assertEquals(resp.status, 400); @@ -791,6 +808,7 @@ test('PATCH /api/upstreams metadata warm preserves refresh backoff', async () => async () => await (await requestApp('/api/upstreams', authed(adminSession, createBody()))).json() as { id: string }, ); const generation = await getCacheGeneration(repo, created.id); + const configVersion = (await repo.upstreams.getById(created.id))?.configVersion; const now = Date.now(); const claim = await repo.upstreams.claimModelsRefresh({ id: created.id, generation, token: 'failed-refresh', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); @@ -813,10 +831,11 @@ test('PATCH /api/upstreams metadata warm preserves refresh backoff', async () => ); assertEquals(modelRequests, 0); + assertEquals((await repo.upstreams.getById(created.id))?.configVersion, configVersion); assertEquals((await repo.upstreams.getById(created.id))?.modelsCache?.models.map(model => model.id), ['cached-model']); }); -test('POST /api/upstreams/list-models without an id still serves draft preview', async () => { +test('POST /api/upstreams/preview-models without an id still serves draft preview', async () => { const { adminSession } = await setupAppTest(); await withMockedFetch( @@ -828,7 +847,7 @@ test('POST /api/upstreams/list-models without an id still serves draft preview', throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const resp = await requestApp('/api/upstreams/list-models', authed(adminSession, { + const resp = await requestApp('/api/upstreams/preview-models', authed(adminSession, { record: blueprintEnvelope('custom', { config: customConfig }), })); assertEquals(resp.status, 200); @@ -2173,7 +2192,7 @@ test('spec invariant (3): POST /api/upstreams/claude-code/probe does not persist assertEquals(stored?.proxyFallbackList, originalList); }); -test('spec invariant (3): POST /api/upstreams/list-models ignores record.name mutation on a saved row', async () => { +test('POST /api/upstreams/preview-models never writes the matching saved row', async () => { const { repo, adminSession } = await setupAppTest(); await repo.upstreams.deleteAll(); // Azure publishes through the persisted-snapshot branch alongside Copilot / Codex / @@ -2193,6 +2212,7 @@ test('spec invariant (3): POST /api/upstreams/list-models ignores record.name mu disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { @@ -2207,11 +2227,12 @@ test('spec invariant (3): POST /api/upstreams/list-models ignores record.name mu const envelope = envelopeFromRecord(savedRecord); envelope.name = 'Mutated'; - const resp = await requestApp('/api/upstreams/list-models', authed(adminSession, { record: envelope })); + const resp = await requestApp('/api/upstreams/preview-models', authed(adminSession, { record: envelope })); assertEquals(resp.status, 200); const stored = await repo.upstreams.getById(savedRecord.id); assertEquals(stored?.name, savedRecord.name); + assertEquals(stored?.modelsCache, null); }); // --- Group B: endpoint tests for surfaces with zero coverage --- diff --git a/packages/gateway/__tests__/control-plane/upstreams/serialize_test.ts b/packages/gateway/__tests__/control-plane/upstreams/serialize_test.ts index 6821c51e9..6cfc91e67 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/serialize_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/serialize_test.ts @@ -27,6 +27,7 @@ const custom: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { @@ -200,6 +201,7 @@ const claudeCodeBase = (overrides: { config?: unknown; state?: unknown }): Upstr disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: overrides.config ?? claudeCodeConfig, @@ -218,6 +220,7 @@ const codexBase = (overrides: { config?: unknown; state?: unknown }): UpstreamRe disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: overrides.config ?? { accounts: [{ email: 'a@example.com', chatgptAccountId: 'account', chatgptUserId: 'user', planType: 'plus' }] }, diff --git a/packages/gateway/__tests__/data-plane/audio/http_test.ts b/packages/gateway/__tests__/data-plane/audio/http_test.ts index c51d0a287..037f80248 100644 --- a/packages/gateway/__tests__/data-plane/audio/http_test.ts +++ b/packages/gateway/__tests__/data-plane/audio/http_test.ts @@ -25,6 +25,7 @@ const registerAudioModel = async ( disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/data-plane/chat/responses/affinity/copilot-roundtrip_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/affinity/copilot-roundtrip_test.ts index 7d039731f..a2f280971 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/affinity/copilot-roundtrip_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/affinity/copilot-roundtrip_test.ts @@ -30,6 +30,7 @@ const upstream: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts index 25bf15efd..7fc071000 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts @@ -208,6 +208,7 @@ beforeEach(async () => { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts index f69c4a467..2b1380ec8 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts @@ -30,6 +30,7 @@ const azureUpstream = (id: string, sortOrder: number, modelIds: string[], endpoi disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts b/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts index 2795372fe..b778bcd27 100644 --- a/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts +++ b/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts @@ -19,6 +19,7 @@ const saveAzureImages = async (repo: InMemoryRepo): Promise => { disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/data-plane/images/http_test.ts b/packages/gateway/__tests__/data-plane/images/http_test.ts index 67f10565d..18fb48992 100644 --- a/packages/gateway/__tests__/data-plane/images/http_test.ts +++ b/packages/gateway/__tests__/data-plane/images/http_test.ts @@ -198,6 +198,7 @@ test('/v1/images/edits forwards a multipart request through an Azure model and r disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts index b6dbaade0..1a83f6242 100644 --- a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts @@ -209,6 +209,7 @@ test('disabledPublicModelIds hides models from the catalog and routing, per upst disabledPublicModelIds: over.disabledPublicModelIds, proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 0c37d8207..63c75f211 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -4,7 +4,7 @@ import { readUpstreamModelsSnapshotAndScheduleRefresh, MODEL_CATALOG_REVISION } import { clearModelsRefreshesForTesting, fetchUpstreamModels, warmUpstreamModels } from '../../../src/data-plane/providers/models-refresh.ts'; import type { GatewayProvider } from '../../../src/data-plane/providers/registry.ts'; import { initRepo } from '../../../src/repo/index.ts'; -import { modelsFetchIdentity } from '../../../src/repo/models-cache-contract.ts'; +import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; import { SqlRepo } from '../../../src/repo/sql.ts'; import type { ModelsCacheGeneration } from '../../../src/repo/types.ts'; import { InMemoryRepo } from '../../repo/memory.ts'; @@ -15,14 +15,8 @@ import { stubProvider, stubProviderModel } from '@floway-dev/test-utils'; const UPSTREAM_ID = 'up_a'; const CACHE_CONFIG = { identity: 'old' }; -const fetchIdentityForConfig = (config: unknown): string => modelsFetchIdentity({ - kind: 'custom', - config, - proxyFallbackList: [], -}); const CACHE_GENERATION: ModelsCacheGeneration = { - updatedAt: '2026-08-01T00:00:00.000Z', - fetchIdentity: fetchIdentityForConfig(CACHE_CONFIG), + configVersion: 1, }; const aModel = (id: string): ProviderModel => stubProviderModel({ id }); @@ -31,7 +25,6 @@ const stubInstance = ( fetchFn: () => Promise, modelsCache: UpstreamModelsCache | null = null, generation: ModelsCacheGeneration = CACHE_GENERATION, - fetchIdentity = generation.fetchIdentity, ): GatewayProvider => ({ upstreamId: UPSTREAM_ID, kind: 'custom', @@ -41,7 +34,7 @@ const stubInstance = ( modelPrefix: null, modelsCache, instance: stubProvider({ getProvidedModels: fetchFn }), - modelsCacheGeneration: { ...generation, fetchIdentity }, + modelsCacheGeneration: generation, }); const setupRepo = async (): Promise => { @@ -54,9 +47,10 @@ const setupRepo = async (): Promise => { enabled: true, sortOrder: 0, createdAt: '2026-08-01T00:00:00.000Z', - updatedAt: CACHE_GENERATION.updatedAt, + updatedAt: '2026-08-01T00:00:00.000Z', config: CACHE_CONFIG, state: null, + configVersion: 1, modelsCache: null, flagOverrides: {}, disabledPublicModelIds: [], @@ -367,10 +361,10 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); const nextConfig = { identity: 'new' }; - const nextGeneration = { updatedAt: CACHE_GENERATION.updatedAt, fetchIdentity: fetchIdentityForConfig(nextConfig) }; + const nextGeneration = { configVersion: CACHE_GENERATION.configVersion + 1 }; const current = await repo.upstreams.getById(UPSTREAM_ID); if (!current) throw new Error('upstream row missing'); - await repo.upstreams.replaceForModels({ previous: current, upstream: { ...current, updatedAt: nextGeneration.updatedAt, config: nextConfig }, cachePolicy: 'clear' }); + await repo.upstreams.replaceForModels({ previous: current, upstream: { ...current, config: nextConfig } }); const newFetch = vi.fn(async () => [aModel('new-tenant-model')]); const newResult = await fetchUpstreamModels(stubInstance(newFetch, null, nextGeneration), directFetcher); @@ -459,6 +453,7 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { updatedAt: '2026-08-01T00:00:00.000Z', config: {}, state: null, + configVersion: 1, modelsCache: null, flagOverrides: {}, disabledPublicModelIds: [], @@ -479,7 +474,7 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { const fetchFn = vi.fn(async () => [aModel('current-catalog')]); const scheduled = captureScheduled(); const result = readUpstreamModelsSnapshotAndScheduleRefresh( - stubInstance(fetchFn, hydrated.modelsCache, { updatedAt: hydrated.updatedAt, fetchIdentity: modelsFetchIdentity(hydrated) }), + stubInstance(fetchFn, hydrated.modelsCache, modelsCacheGeneration(hydrated)), { scheduler: scheduled.scheduler, fetcher: directFetcher }, ); diff --git a/packages/gateway/__tests__/data-plane/providers/registry_test.ts b/packages/gateway/__tests__/data-plane/providers/registry_test.ts index a368294c1..43ac061ce 100644 --- a/packages/gateway/__tests__/data-plane/providers/registry_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/registry_test.ts @@ -1,21 +1,12 @@ -import { expect, test } from 'vitest'; +import { test } from 'vitest'; import { MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; -import { listModelProviders, modelsCatalogIdentity } from '../../../src/data-plane/providers/registry.ts'; +import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; import { buildCopilotUpstreamRecord, buildCustomUpstreamRecord, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; -test('Copilot catalog identity follows the account rather than rotated credentials', () => { - const first = buildCopilotUpstreamRecord({ token: 'ghu_first', user: { id: 1, login: 'one', avatar_url: '', name: null } }); - const rotated = buildCopilotUpstreamRecord({ token: 'ghu_rotated', user: { id: 1, login: 'one-renamed', avatar_url: '', name: null } }); - const otherAccount = buildCopilotUpstreamRecord({ token: 'ghu_other', user: { id: 2, login: 'two', avatar_url: '', name: null } }); - - assertEquals(modelsCatalogIdentity(first), modelsCatalogIdentity(rotated)); - expect(modelsCatalogIdentity(first)).not.toBe(modelsCatalogIdentity(otherAccount)); -}); - test('listModelProviders creates enabled provider instances with upstream row ids', async () => { const { githubAccount, repo } = await setupAppTest(); await repo.upstreams.deleteAll(); @@ -42,6 +33,7 @@ test('listModelProviders creates enabled provider instances with upstream row id disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, state: null, diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index 59843307b..935abee71 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -307,6 +307,7 @@ test('enumerateRealModelCandidates rejects a model id disabled on that upstream disabledPublicModelIds: ['disabled-model'], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, state: null, diff --git a/packages/gateway/__tests__/dial/per-request_test.ts b/packages/gateway/__tests__/dial/per-request_test.ts index 2226934f9..151c3fbb5 100644 --- a/packages/gateway/__tests__/dial/per-request_test.ts +++ b/packages/gateway/__tests__/dial/per-request_test.ts @@ -30,6 +30,7 @@ const upstream = (id: string, proxyFallbackList: ProxyFallbackEntry[]) => ({ disabledPublicModelIds: [], proxyFallbackList, modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: COPILOT_CONFIG, diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index 4ef773f05..2c6c5567d 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -3,7 +3,7 @@ import { partitionTelemetryOverviewRecords } from './telemetry-overview-oracle.t import { buildKeyToUserMap } from '../../src/control-plane/shared/key-to-user.ts'; import { normalizeDisabledPublicModelIds } from '../../src/repo/disabled-public-models.ts'; import { normalizeFlagOverrides } from '../../src/repo/flag-overrides.ts'; -import { MODEL_CATALOG_REVISION, modelsFetchIdentity } from '../../src/repo/models-cache-contract.ts'; +import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; import { normalizeProxyFallbackList } from '../../src/repo/proxy-fallback-list.ts'; import { @@ -748,21 +748,33 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(found ? cloneUpstreamRecord(found) : null); } - // Mirrors the SQL INSERT/UPDATE column list, which omits the cache column: - // an existing row keeps whatever the refresh path last wrote there, and a new - // row starts uncached whatever the caller's record carried. + // Mirrors the SQL upsert: config changes advance the generation and clear + // the snapshot; other writes preserve it. New rows always start uncached. save(upstream: UpstreamRecord): Promise { const existing = this.store.get(upstream.id); + if (existing === undefined && upstream.configVersion !== 1) { + throw new Error(`New upstream ${upstream.id} must start at config version 1`); + } + const configChanged = existing !== undefined + && (existing.kind !== upstream.kind + || serializeStoredConfig(existing.config) !== serializeStoredConfig(upstream.config)); const preserved = existing - ? { ...upstream, createdAt: existing.createdAt, modelsCache: existing.modelsCache } + ? { + ...upstream, + createdAt: existing.createdAt, + configVersion: existing.configVersion + (configChanged ? 1 : 0), + modelsCache: configChanged ? null : existing.modelsCache, + } : { ...upstream, modelsCache: null }; this.store.set(preserved.id, cloneUpstreamRecord(preserved)); const refresh = this.modelsRefreshes.get(preserved.id); - if (refresh) this.modelsRefreshes.set(preserved.id, { ...refresh, claimToken: null, claimedAt: null }); + if (configChanged) this.modelsRefreshes.delete(preserved.id); + else if (refresh) this.modelsRefreshes.set(preserved.id, { ...refresh, claimToken: null, claimedAt: null }); return Promise.resolve(); } insertForModels(upstream: UpstreamRecord): Promise { + if (upstream.configVersion !== 1) throw new Error(`New upstream ${upstream.id} must start at config version 1`); if (this.store.has(upstream.id)) return Promise.resolve(false); this.store.set(upstream.id, cloneUpstreamRecord({ ...upstream, modelsCache: null })); return Promise.resolve(true); @@ -770,10 +782,13 @@ class MemoryUpstreamRepo implements UpstreamRepo { replaceForModels(input: { previous: UpstreamRecord; - upstream: UpstreamRecord; - cachePolicy: 'preserve' | 'reset-refresh' | 'clear'; + upstream: Omit; }): Promise { - const { previous, upstream, cachePolicy } = input; + const { previous, upstream } = input; + const configChanged = previous.kind !== upstream.kind + || serializeStoredConfig(previous.config) !== serializeStoredConfig(upstream.config); + const configVersion = previous.configVersion + (configChanged ? 1 : 0); + const transportChanged = serializeStoredConfig(previous.proxyFallbackList) !== serializeStoredConfig(upstream.proxyFallbackList); const existing = this.store.get(upstream.id); if (existing === undefined) return Promise.resolve(false); const replaceState = serializeStoredState(previous.state) !== serializeStoredState(upstream.state); @@ -783,11 +798,12 @@ class MemoryUpstreamRepo implements UpstreamRepo { const next = cloneUpstreamRecord({ ...upstream, createdAt: existing.createdAt, + configVersion, state: replaceState ? upstream.state : existing.state, - modelsCache: cachePolicy === 'clear' ? null : existing.modelsCache, + modelsCache: configChanged ? null : existing.modelsCache, }); this.store.set(upstream.id, next); - if (cachePolicy === 'preserve') { + if (!configChanged && !transportChanged) { const refresh = this.modelsRefreshes.get(upstream.id); if (refresh !== undefined) this.modelsRefreshes.set(upstream.id, { ...refresh, claimToken: null, claimedAt: null }); } else { @@ -824,7 +840,7 @@ class MemoryUpstreamRepo implements UpstreamRepo { const { id, generation, token, cache } = input; if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); const existing = this.store.get(id); - if (!existing || existing.updatedAt !== generation.updatedAt || modelsFetchIdentity(existing) !== generation.fetchIdentity) return Promise.resolve(false); + if (!existing || existing.configVersion !== generation.configVersion) return Promise.resolve(false); existing.modelsCache = { revision: cache.revision, fetchedAt: cache.fetchedAt, models: [...cache.models], lastError: null }; this.modelsRefreshes.delete(id); return Promise.resolve(true); @@ -837,7 +853,7 @@ class MemoryUpstreamRepo implements UpstreamRepo { const refresh = this.modelsRefreshes.get(id); if (refresh?.claimToken !== token || refresh.failCount !== previousFailureCount) return Promise.resolve(false); const existing = this.store.get(id); - if (!existing || existing.updatedAt !== generation.updatedAt || modelsFetchIdentity(existing) !== generation.fetchIdentity) return Promise.resolve(false); + if (!existing || existing.configVersion !== generation.configVersion) return Promise.resolve(false); if (existing.modelsCache) existing.modelsCache.lastError = error; else existing.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }; this.modelsRefreshes.set(id, { failCount: failureCount, retryAt, claimToken: null, claimedAt: null }); @@ -849,8 +865,7 @@ class MemoryUpstreamRepo implements UpstreamRepo { const existing = this.store.get(id); const refresh = this.modelsRefreshes.get(id); if (!existing - || existing.updatedAt !== generation.updatedAt - || modelsFetchIdentity(existing) !== generation.fetchIdentity + || existing.configVersion !== generation.configVersion || refresh?.claimToken !== token) return Promise.resolve(false); this.modelsRefreshes.set(id, { ...refresh, claimToken: null, claimedAt: null }); return Promise.resolve(true); @@ -859,7 +874,7 @@ class MemoryUpstreamRepo implements UpstreamRepo { claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { const { id, generation, token, now, staleClaimedBefore, bypassBackoff, observedActiveToken } = input; const stored = this.store.get(id); - if (!stored || stored.updatedAt !== generation.updatedAt || modelsFetchIdentity(stored) !== generation.fetchIdentity) return Promise.resolve({ kind: 'generation-mismatch' }); + if (!stored || stored.configVersion !== generation.configVersion) return Promise.resolve({ kind: 'generation-mismatch' }); const existing = this.modelsRefreshes.get(id); if (observedActiveToken !== null && existing === undefined) return Promise.resolve({ kind: 'completed' }); if (existing !== undefined) { diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index de54110ca..072b80608 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -18,6 +18,7 @@ const record: UpstreamRecord = { updatedAt: '2026-08-01T00:00:00.000Z', config: { tenant: 'current' }, state: null, + configVersion: 1, modelsCache: null, flagOverrides: {}, disabledPublicModelIds: [], @@ -34,6 +35,20 @@ const factories: [string, () => Promise][] = [ ]; describe.each(factories)('%s models refresh coordination', (_name, createRepo) => { + test('config writes advance the generation while state writes do not', async () => { + const repo = await createRepo(); + await repo.upstreams.save(record); + await repo.upstreams.saveState(record.id, () => ({ accessToken: 'rotated' })); + expect((await repo.upstreams.getById(record.id))?.configVersion).toBe(1); + + const current = await repo.upstreams.getById(record.id); + if (current === null) throw new Error('upstream row missing'); + await repo.upstreams.save({ ...current, config: { tenant: 'next' } }); + const changed = await repo.upstreams.getById(record.id); + expect(changed?.configVersion).toBe(2); + expect(changed?.state).toEqual({ accessToken: 'rotated' }); + }); + test('claims atomically, applies one backoff schedule, and lets force bypass cooldown', async () => { const repo = await createRepo(); await repo.upstreams.save(record); @@ -64,7 +79,7 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'after-success', now: now + 2, staleClaimedBefore: now - 899_998, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); }); - test('recovers abandoned claims and fences tokens, timestamps, and config', async () => { + test('recovers abandoned claims and fences tokens and config versions', async () => { const repo = await createRepo(); await repo.upstreams.save(record); const now = 1_800_000_000_000; @@ -75,13 +90,15 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'racer', now: now + 900_002, staleClaimedBefore: now + 2, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'replacement' }); const next = { ...record, config: { tenant: 'next' } }; - await repo.upstreams.replaceForModels({ previous: record, upstream: next, cachePolicy: 'clear' }); + await repo.upstreams.replaceForModels({ previous: record, upstream: next }); await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'old-config', now: now + 900_003, staleClaimedBefore: now + 3, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'generation-mismatch' }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(next), token: 'current', now: now + 900_003, staleClaimedBefore: now + 3, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + const storedNext = await repo.upstreams.getById(record.id); + if (storedNext === null) throw new Error('upstream row missing'); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(storedNext), token: 'current', now: now + 900_003, staleClaimedBefore: now + 3, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - const newer = { ...next, updatedAt: '2026-08-01T00:01:00.000Z' }; - await repo.upstreams.replaceForModels({ previous: next, upstream: newer, cachePolicy: 'clear' }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(next), token: 'old-time', now: now + 900_004, staleClaimedBefore: now + 4, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'generation-mismatch' }); + const renamed = { ...storedNext, name: 'Renamed', updatedAt: '2026-08-01T00:01:00.000Z' }; + await repo.upstreams.replaceForModels({ previous: storedNext, upstream: renamed }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(storedNext), token: 'after-rename', now: now + 900_004, staleClaimedBefore: now + 4, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); }); test('metadata saves preserve backoff while invalidating an active owner', async () => { @@ -93,7 +110,7 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: 'failed', error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); const next = { ...record, name: 'Renamed', updatedAt: '2026-08-01T00:01:00.000Z' }; - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next, cachePolicy: 'preserve' })).resolves.toBe(true); + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next })).resolves.toBe(true); await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(next), @@ -105,7 +122,7 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = })).resolves.toEqual({ kind: 'backoff' }); }); - test('operator credential changes preserve the snapshot while resetting refresh cooldown', async () => { + test('state changes preserve the snapshot generation and refresh cooldown', async () => { const repo = await createRepo(); await repo.upstreams.save(record); const now = 1_800_000_000_000; @@ -114,7 +131,7 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: 'failed', error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); const next = { ...record, state: { credential: 'rotated' }, updatedAt: '2026-08-01T00:01:00.000Z' }; - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next, cachePolicy: 'reset-refresh' })).resolves.toBe(true); + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next })).resolves.toBe(true); expect((await repo.upstreams.getById(record.id))?.modelsCache?.lastError?.message).toBe('failure'); await expect(repo.upstreams.claimModelsRefresh({ id: record.id, @@ -124,7 +141,7 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = staleClaimedBefore: now - 899_999, bypassBackoff: false, observedActiveToken: null, - })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + })).resolves.toEqual({ kind: 'backoff' }); }); test('provider-managed credential state can rotate without invalidating its own owner', async () => { @@ -166,8 +183,8 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = const winner = { ...record, name: 'Winner', updatedAt: '2026-08-01T00:01:00.000Z' }; const stale = { ...record, name: 'Stale', updatedAt: '2026-08-01T00:02:00.000Z' }; - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: winner, cachePolicy: 'preserve' })).resolves.toBe(true); - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: stale, cachePolicy: 'clear' })).resolves.toBe(false); + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: winner })).resolves.toBe(true); + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: stale })).resolves.toBe(false); expect((await repo.upstreams.getById(record.id))?.name).toBe('Winner'); expect((await repo.upstreams.getById(record.id))?.state).toEqual({ providerManaged: 'newer' }); }); diff --git a/packages/gateway/__tests__/repo/proxies_test.ts b/packages/gateway/__tests__/repo/proxies_test.ts index 37a5f5da9..11883a013 100644 --- a/packages/gateway/__tests__/repo/proxies_test.ts +++ b/packages/gateway/__tests__/repo/proxies_test.ts @@ -31,6 +31,7 @@ const upstreamFixture = (id: string, proxyFallbackList: ProxyFallbackEntry[]): U disabledPublicModelIds: [], proxyFallbackList, modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/gateway/__tests__/repo/sql_test.ts b/packages/gateway/__tests__/repo/sql_test.ts index d7915dffa..c7423e8f7 100644 --- a/packages/gateway/__tests__/repo/sql_test.ts +++ b/packages/gateway/__tests__/repo/sql_test.ts @@ -25,6 +25,7 @@ const baseRecord = (overrides: Partial = {}): UpstreamRecord => disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, ...overrides, @@ -167,7 +168,7 @@ test('SQL upstream repo catalog-aware replacement can clear the cached catalog a name: 'New identity', config: { accounts: [{ email: 'new@example.com', chatgptAccountId: 'new-account', chatgptUserId: 'new-user', planType: 'plus' }] }, }); - await repo.replaceForModels({ previous: baseRecord(), upstream: newIdentity, cachePolicy: 'clear' }); + await repo.replaceForModels({ previous: baseRecord(), upstream: newIdentity }); const stored = await repo.getById('up_test'); assertEquals(stored?.name, 'New identity'); diff --git a/packages/gateway/__tests__/repo/upstreams_test.ts b/packages/gateway/__tests__/repo/upstreams_test.ts index 81099d1f6..7df823beb 100644 --- a/packages/gateway/__tests__/repo/upstreams_test.ts +++ b/packages/gateway/__tests__/repo/upstreams_test.ts @@ -19,6 +19,7 @@ const upstream = (overrides: Partial & Pick { sort_order: 0, created_at: '2026-05-21T10:00:00.000Z', updated_at: '2026-05-21T10:00:00.000Z', + config_version: 1, config_json: '{bad json', state_json: null, models_cache_json: null, @@ -268,6 +270,7 @@ test('SQL upstream repo rejects malformed stored flag overrides JSON', async () sort_order: 0, created_at: '2026-05-21T10:00:00.000Z', updated_at: '2026-05-21T10:00:00.000Z', + config_version: 1, config_json: '{}', state_json: null, models_cache_json: null, @@ -291,6 +294,7 @@ test('SQL upstream repo rejects array-shaped flag_overrides with helpful message sort_order: 0, created_at: '2026-05-21T10:00:00.000Z', updated_at: '2026-05-21T10:00:00.000Z', + config_version: 1, config_json: '{}', state_json: null, models_cache_json: null, @@ -318,6 +322,7 @@ test('SQL upstream repo rejects non-boolean value in flag_overrides with helpful sort_order: 0, created_at: '2026-05-21T10:00:00.000Z', updated_at: '2026-05-21T10:00:00.000Z', + config_version: 1, config_json: '{}', state_json: null, models_cache_json: null, @@ -345,6 +350,7 @@ test('SQL upstream repo rejects malformed stored model_prefix_json', async () => sort_order: 0, created_at: '2026-05-21T10:00:00.000Z', updated_at: '2026-05-21T10:00:00.000Z', + config_version: 1, config_json: '{}', state_json: null, models_cache_json: null, @@ -368,6 +374,7 @@ test('SQL upstream repo rejects shape-invalid model_prefix_json', async () => { sort_order: 0, created_at: '2026-05-21T10:00:00.000Z', updated_at: '2026-05-21T10:00:00.000Z', + config_version: 1, config_json: '{}', state_json: null, models_cache_json: null, @@ -400,6 +407,7 @@ test('SQL upstream repo round-trips a non-null model_prefix', async () => { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: { prefix: 'or/', addressable: ['unprefixed', 'prefixed'], listed: ['prefixed'] }, + configVersion: 1, modelsCache: null, hue: 210, }; @@ -439,6 +447,7 @@ test('SQL upstream repo rejects a stored hue outside the circle', async () => { sort_order: 0, created_at: '2026-07-01T00:00:00.000Z', updated_at: '2026-07-01T00:00:00.000Z', + config_version: 1, config_json: '{}', state_json: null, models_cache_json: null, @@ -954,6 +963,7 @@ type FakeUpstreamRow = { sort_order: number; created_at: string; updated_at: string; + config_version: number; config_json: string; state_json: string | null; models_cache_json: string | null; @@ -1031,7 +1041,7 @@ class FakeUpstreamsSqlDatabase implements SqlDatabase { } upsert(binds: unknown[]): void { - const [id, provider, name, enabled, sortOrder, createdAt, updatedAt, configJson, stateJson, flagOverrides, disabledPublicModelIds, proxyFallbackListJson, modelPrefixJson, hue] = binds as [string, string, string, number, number, string, string, string, string | null, string, string, string, string | null, number]; + const [id, provider, name, enabled, sortOrder, createdAt, updatedAt, configVersion, configJson, stateJson, flagOverrides, disabledPublicModelIds, proxyFallbackListJson, modelPrefixJson, hue] = binds as [string, string, string, number, number, string, string, number, string, string | null, string, string, string, string | null, number]; const existingIndex = this.rows.findIndex(candidate => candidate.id === id); const existing = existingIndex >= 0 ? this.rows[existingIndex] : undefined; const preservedCreatedAt = existing ? existing.created_at : createdAt; @@ -1043,11 +1053,10 @@ class FakeUpstreamsSqlDatabase implements SqlDatabase { sort_order: sortOrder, created_at: preservedCreatedAt, updated_at: updatedAt, + config_version: configVersion, config_json: configJson, state_json: stateJson, - // The upsert statement names no cache column, so an existing row keeps - // whatever the refresh path wrote and a new row starts uncached. - models_cache_json: existing?.models_cache_json ?? null, + models_cache_json: existing?.config_version === configVersion ? existing.models_cache_json : null, flag_overrides: flagOverrides, disabled_public_model_ids: disabledPublicModelIds, proxy_fallback_list_json: proxyFallbackListJson, diff --git a/packages/gateway/__tests__/scheduled/models-refresh_test.ts b/packages/gateway/__tests__/scheduled/models-refresh_test.ts index 828437910..55db95549 100644 --- a/packages/gateway/__tests__/scheduled/models-refresh_test.ts +++ b/packages/gateway/__tests__/scheduled/models-refresh_test.ts @@ -23,6 +23,7 @@ const custom = (id: string, enabled: boolean): UpstreamRecord => ({ ingressHeadersRules: [], }, state: null, + configVersion: 1, modelsCache: null, flagOverrides: {}, disabledPublicModelIds: [], diff --git a/packages/gateway/__tests__/test-utils/app.ts b/packages/gateway/__tests__/test-utils/app.ts index fc4387c34..5846b7167 100644 --- a/packages/gateway/__tests__/test-utils/app.ts +++ b/packages/gateway/__tests__/test-utils/app.ts @@ -80,6 +80,7 @@ export const buildCopilotUpstreamRecord = (githubAccount: CopilotAccountFixture, disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, ...rest, @@ -110,6 +111,7 @@ export const buildCustomUpstreamRecord = (overrides: Partial = { disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, ...rest, diff --git a/packages/gateway/migrations/0079_upstream_config_version.sql b/packages/gateway/migrations/0079_upstream_config_version.sql new file mode 100644 index 000000000..9d37fdd47 --- /dev/null +++ b/packages/gateway/migrations/0079_upstream_config_version.sql @@ -0,0 +1,4 @@ +-- Catalog publication is fenced by provider configuration, independently of +-- runtime state and operator metadata updates. +ALTER TABLE upstreams ADD COLUMN config_version INTEGER NOT NULL DEFAULT 1 + CHECK (typeof(config_version) = 'integer' AND config_version >= 1); diff --git a/packages/gateway/src/control-plane/data-transfer/import-schema.ts b/packages/gateway/src/control-plane/data-transfer/import-schema.ts index 0ebf9f7d5..8a13d2980 100644 --- a/packages/gateway/src/control-plane/data-transfer/import-schema.ts +++ b/packages/gateway/src/control-plane/data-transfer/import-schema.ts @@ -159,6 +159,9 @@ const upstreamWireSchema = parsedBy((value): UpstreamRecord => { sortOrder, createdAt: parseValue(nonEmptyStringSchema('created_at'), wire.created_at), updatedAt: parseValue(nonEmptyStringSchema('updated_at'), wire.updated_at), + // Import establishes a local generation; merge mode will advance it when + // the imported provider config differs from the row already stored. + configVersion: 1, flagOverrides: parseValue(parsedBy(parseFlagOverridesWire), wire.flag_overrides), disabledPublicModelIds: parseValue(parsedBy(parseDisabledPublicModelIdsWire).optional().default([]), wire.disabled_public_model_ids), proxyFallbackList: parseValue(proxyFallbackListSchema, wire.proxy_fallback_list), diff --git a/packages/gateway/src/control-plane/routes.ts b/packages/gateway/src/control-plane/routes.ts index 3623ddda2..2447d7b9f 100644 --- a/packages/gateway/src/control-plane/routes.ts +++ b/packages/gateway/src/control-plane/routes.ts @@ -9,7 +9,7 @@ import { createAlias, deleteAlias, listAliases, updateAlias } from './model-alia import { controlPlaneModels } from './models/routes.ts'; import { performanceOverview } from './performance/routes.ts'; import { createProxy, deleteProxy, listAllBackoffs, listProxies, listProxyBackoffs, resetProxyBackoffs, testProxy, updateProxy } from './proxies/routes.ts'; -import { authLoginBody, changeOwnPasswordBody, claudeCodeOAuthAuthorizeUrlBody, claudeCodeOAuthExchangeBody, claudeCodeOAuthRefreshBody, claudeCodeProbeBody, claudeCodeSetupTokenAuthorizeUrlBody, claudeCodeSetupTokenExchangeBody, codexOAuthAuthorizeUrlBody, codexOAuthExchangeBody, codexOAuthRefreshBody, copilotOAuthDeviceLoginPollBody, copilotOAuthDeviceLoginStartBody, copilotQuotaBody, createAliasBody, createKeyBody, createProxyBody, createUpstreamBody, createUserBody, exportQuery, importBody, listModelsBody, modelsQuery, performanceQuery, resetBackoffBody, rotateKeyBody, webSearchConfigSchema, webSearchUsageQuery, testProxyBody, tokenUsageOverviewQuery, tokenUsageQuery, updateAliasBody, updateKeyBody, updateProxyBody, updateUpstreamBody, updateUserBody } from './schemas.ts'; +import { authLoginBody, changeOwnPasswordBody, claudeCodeOAuthAuthorizeUrlBody, claudeCodeOAuthExchangeBody, claudeCodeOAuthRefreshBody, claudeCodeProbeBody, claudeCodeSetupTokenAuthorizeUrlBody, claudeCodeSetupTokenExchangeBody, codexOAuthAuthorizeUrlBody, codexOAuthExchangeBody, codexOAuthRefreshBody, copilotOAuthDeviceLoginPollBody, copilotOAuthDeviceLoginStartBody, copilotQuotaBody, createAliasBody, createKeyBody, createProxyBody, createUpstreamBody, createUserBody, exportQuery, importBody, modelsQuery, performanceQuery, previewModelsBody, resetBackoffBody, rotateKeyBody, webSearchConfigSchema, webSearchUsageQuery, testProxyBody, tokenUsageOverviewQuery, tokenUsageQuery, updateAliasBody, updateKeyBody, updateProxyBody, updateUpstreamBody, updateUserBody } from './schemas.ts'; import { getWebSearchConfigRoute, putWebSearchConfigRoute, testWebSearchConfigRoute } from './search-config/routes.ts'; import { webSearchUsage } from './search-usage/routes.ts'; import { tokenUsageOverview } from './token-usage/overview.ts'; @@ -17,7 +17,7 @@ import { tokenUsage } from './token-usage/routes.ts'; import { claudeCodeOAuthAuthorizeUrl, claudeCodeOAuthExchange, claudeCodeOAuthRefresh, claudeCodeProbe, claudeCodeSetupTokenAuthorizeUrl, claudeCodeSetupTokenExchange } from './upstreams/claude-code.ts'; import { codexOAuthAuthorizeUrl, codexOAuthExchange, codexOAuthRefresh } from './upstreams/codex.ts'; import { copilotOAuthDeviceLoginPoll, copilotOAuthDeviceLoginStart, copilotQuota } from './upstreams/copilot.ts'; -import { listModels } from './upstreams/models.ts'; +import { fetchSavedModels, previewModels } from './upstreams/models.ts'; import { createUpstream, deleteUpstream, getUpstream, getUpstreamBlueprint, listUpstreamOptions, listUpstreams, updateUpstream } from './upstreams/routes.ts'; import { changeOwnPassword, createUser, deleteUser, listUsers, updateUser } from './users/routes.ts'; import { type AuthedContext, type AuthVars, userFromContext } from '../middleware/auth.ts'; @@ -90,7 +90,8 @@ export const controlPlaneRoutes = new Hono<{ Variables: AuthVars }>() .post('/upstreams/claude-code/setup-token/authorize-url', zValidator('json', claudeCodeSetupTokenAuthorizeUrlBody), claudeCodeSetupTokenAuthorizeUrl) .post('/upstreams/claude-code/setup-token/exchange', zValidator('json', claudeCodeSetupTokenExchangeBody), claudeCodeSetupTokenExchange) .post('/upstreams/claude-code/probe', zValidator('json', claudeCodeProbeBody), claudeCodeProbe) - .post('/upstreams/list-models', zValidator('json', listModelsBody), listModels) + .post('/upstreams/preview-models', zValidator('json', previewModelsBody), previewModels) + .post('/upstreams/:id/list-models', fetchSavedModels) .post('/upstreams', zValidator('json', createUpstreamBody), createUpstream) .get('/upstreams/:id', getUpstream) .patch('/upstreams/:id', zValidator('json', updateUpstreamBody), updateUpstream) diff --git a/packages/gateway/src/control-plane/schemas.ts b/packages/gateway/src/control-plane/schemas.ts index 5d8428592..76a0baa56 100644 --- a/packages/gateway/src/control-plane/schemas.ts +++ b/packages/gateway/src/control-plane/schemas.ts @@ -393,7 +393,7 @@ export const updateUpstreamBody = z.object({ }); // Shared envelope for the record-body action contract used by every -// action endpoint (OAuth exchange/refresh, quota, probe, list-models, +// action endpoint (OAuth exchange/refresh, quota, probe, draft preview, // etc.). The client posts its full draft record; the server reads only // fields relevant to the specific action (credentials in config/state, // proxy_fallback_list for routing) and produces a targeted patch. Kind @@ -408,7 +408,7 @@ export const upstreamRecordEnvelope = z.object({ }).passthrough(); // The bare envelope contract — every action endpoint that takes no extras -// beyond `record` (refresh, probe, quota, list-models) shares this shape. +// beyond `record` (refresh, probe, quota, draft preview) shares this shape. const recordOnlyBody = z.object({ record: upstreamRecordEnvelope }); export const copilotOAuthDeviceLoginStartBody = recordOnlyBody; @@ -487,11 +487,9 @@ export const claudeCodeSetupTokenExchangeBody = z.object({ export const claudeCodeProbeBody = recordOnlyBody; -// Unified live-model listing for both create-time preview and edit-time -// refresh. Custom returns the raw upstream row (dashboard translates -// through the draft's endpoints); every other kind returns the fully -// projected UpstreamModelConfig catalog. -export const listModelsBody = recordOnlyBody; +// A draft preview always remains detached from storage, even when the +// envelope originated from an existing editor record. +export const previewModelsBody = recordOnlyBody; // --- agent setup --- // diff --git a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts index f0f3e80b0..0f4a9345d 100644 --- a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts +++ b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts @@ -1,12 +1,11 @@ import type { Context } from 'hono'; import { warmUpstreamModels } from '../../data-plane/providers/models-refresh.ts'; -import { modelsCatalogIdentity, createProvider } from '../../data-plane/providers/registry.ts'; +import { createProvider } from '../../data-plane/providers/registry.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; import { getRepo } from '../../repo/index.ts'; -import { modelsOperatorRefreshIdentity } from '../../repo/models-cache-contract.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; -import type { UpstreamModelsCache, UpstreamRecord } from '@floway-dev/provider'; +import type { UpstreamRecord } from '@floway-dev/provider'; import { logInfo } from '@floway-dev/provider-claude-code'; export interface UpstreamModelsChange { @@ -23,18 +22,14 @@ const saveUpstreamForModels = async ({ previous, next }: UpstreamModelsChange): if (!inserted) throw new Error(`Upstream ${next.id} changed concurrently`); return; } - let cachePolicy: 'preserve' | 'reset-refresh' | 'clear'; - if (modelsCatalogIdentity(previous) !== modelsCatalogIdentity(next)) cachePolicy = 'clear'; - else if (modelsOperatorRefreshIdentity(previous) !== modelsOperatorRefreshIdentity(next)) cachePolicy = 'reset-refresh'; - else cachePolicy = 'preserve'; - const saved = await upstreams.replaceForModels({ previous, upstream: next, cachePolicy }); + const saved = await upstreams.replaceForModels({ previous, upstream: next }); if (!saved) throw new Error(`Upstream ${next.id} changed concurrently`); }; export const saveAndWarmUpstreamsForModels = async ( changes: readonly UpstreamModelsChange[], c: Context, -): Promise> => { +): Promise> => { if (new Set(changes.map(change => change.next.id)).size !== changes.length) { throw new Error('Duplicate upstream ids in models save batch'); } @@ -53,8 +48,9 @@ export const saveAndWarmUpstreamsForModels = async ( } catch (error) { logInfo('warm_models_cache_failed', { upstream_id: record.id, error: errorMessage(error) }); } - const cache = (await getRepo().upstreams.getById(record.id))?.modelsCache ?? null; - return [record.id, cache] as const; + const refreshed = await getRepo().upstreams.getById(record.id); + if (refreshed === null) throw new Error(`Upstream ${record.id} disappeared after warm`); + return [record.id, refreshed] as const; })); return new Map(entries); }; @@ -62,8 +58,8 @@ export const saveAndWarmUpstreamsForModels = async ( export const saveAndWarmUpstreamForModels = async ( change: UpstreamModelsChange, c: Context, -): Promise => { +): Promise => { const result = (await saveAndWarmUpstreamsForModels([change], c)).get(change.next.id); - if (result === undefined) throw new Error(`Missing models cache result for ${change.next.id}`); + if (result === undefined) throw new Error(`Missing saved upstream result for ${change.next.id}`); return result; }; diff --git a/packages/gateway/src/control-plane/upstreams/models.ts b/packages/gateway/src/control-plane/upstreams/models.ts index ef7602d6f..b93e2601b 100644 --- a/packages/gateway/src/control-plane/upstreams/models.ts +++ b/packages/gateway/src/control-plane/upstreams/models.ts @@ -3,12 +3,12 @@ import { isValidProviderKind, upstreamErrorMessage as errorMessage } from './sha import type { ListedUpstreamModel } from './types.ts'; import { MODEL_LISTING_FAILURE_CODE, MODEL_LISTING_FAILURE_MESSAGE } from '../../data-plane/models/shared.ts'; import { fetchUpstreamModels } from '../../data-plane/providers/models-refresh.ts'; -import { createProvider, modelsRequestIdentity } from '../../data-plane/providers/registry.ts'; +import { createProvider } from '../../data-plane/providers/registry.ts'; +import type { AuthedContext } from '../../middleware/auth.ts'; import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; -import { modelsCacheGeneration } from '../../repo/models-cache-contract.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; -import type { listModelsBody } from '../schemas.ts'; +import type { previewModelsBody } from '../schemas.ts'; import { ProviderModelsUnavailableError, type Fetcher, type ProviderModel, type ProxyFallbackEntry, type UpstreamRecord } from '@floway-dev/provider'; import { assertCustomUpstreamRecord, fetchCustomModels, projectCustomModels } from '@floway-dev/provider-custom'; @@ -35,33 +35,33 @@ const reshapeModelForDashboard = (model: ProviderModel): ListedUpstreamModel => }; }; -// Unified model catalog fetch for draft previews and saved records. A request -// matching the saved fetch inputs atomically publishes its result to the -// persisted snapshot; an unsaved draft is fetched without touching that row. -// Custom keeps the raw upstream response shape for the dashboard; every other -// provider returns its ProviderModel projection. -export const listModels = async (c: CtxWithJson) => { +const malformedConfigResponse = (error: unknown): boolean => + error instanceof Error && /Malformed .* upstream config/.test(error.message); + +// Draft previews are deliberately detached from storage. The request carries +// the exact editor values to probe, and neither a matching id nor matching +// credentials can turn this operation into a cache write. +export const previewModels = async (c: CtxWithJson) => { const { record } = c.req.valid('json'); if (!isValidProviderKind(record.kind)) { return c.json({ error: { message: `Invalid kind: ${record.kind}`, type: 'invalid_request_error' } }, 400); } const kind = record.kind; - const persisted = record.id === '' ? null : await getRepo().upstreams.getById(record.id); - if (record.id !== '' && persisted === null) return c.json({ error: 'Upstream not found' }, 404); - const effectiveProxyFallbackList = (record.proxy_fallback_list ?? persisted?.proxyFallbackList ?? []) as ProxyFallbackEntry[]; + const proxyFallbackList = (record.proxy_fallback_list ?? []) as ProxyFallbackEntry[]; const now = new Date().toISOString(); const synthRecord: UpstreamRecord = { - id: record.id || 'draft', + id: 'draft', kind, name: 'draft', enabled: true, sortOrder: 0, createdAt: now, - updatedAt: persisted?.updatedAt ?? now, + updatedAt: now, + configVersion: 1, flagOverrides: {}, disabledPublicModelIds: [], - proxyFallbackList: effectiveProxyFallbackList, + proxyFallbackList, modelPrefix: null, // A draft only lists models; nothing renders its badge. hue: 0, @@ -71,14 +71,10 @@ export const listModels = async (c: CtxWithJson) => { // never carries a cached catalog. modelsCache: null, }; - const canRefreshPersistedCache = persisted !== null - && modelsRequestIdentity(persisted) === modelsRequestIdentity(synthRecord); - let fetcher: Fetcher; try { fetcher = await resolveControlPlaneFetcher({ - override: effectiveProxyFallbackList, - upstreamId: record.id || undefined, + override: proxyFallbackList, runtimeLocation: getRuntimeLocation(c.req.raw), }); } catch (err) { @@ -88,36 +84,60 @@ export const listModels = async (c: CtxWithJson) => { try { if (kind === 'custom') { const assertedConfig = assertCustomUpstreamRecord(synthRecord).config; - const provider = createProvider(synthRecord, persisted === null ? undefined : modelsCacheGeneration(persisted)); - let result: Awaited> | undefined; - if (!canRefreshPersistedCache) { - result = await fetchCustomModels(assertedConfig, fetcher); - } else { - await fetchUpstreamModels(provider, fetcher, async () => { - result = await fetchCustomModels(assertedConfig, fetcher); - return projectCustomModels(synthRecord, result); - }); - // A concurrent refresh may already own the cache's in-flight slot, in - // which case our raw-shape loader was not invoked. The dashboard still - // needs its raw response, so only that joined-flight case fetches it - // separately. - result ??= await fetchCustomModels(assertedConfig, fetcher); - } + const result = await fetchCustomModels(assertedConfig, fetcher); return c.json({ kind, data: result.data }); } - // Copilot / codex / claude-code / azure / ollama use the provider factory. - const provider = createProvider(synthRecord, persisted === null ? undefined : modelsCacheGeneration(persisted)); - const models = canRefreshPersistedCache - ? await fetchUpstreamModels(provider, fetcher) - : await provider.instance.getProvidedModels(fetcher); + const models = await createProvider(synthRecord).instance.getProvidedModels(fetcher); return c.json({ kind, data: models.map(reshapeModelForDashboard) }); } catch (e) { if (e instanceof ProviderModelsUnavailableError) { return c.json({ error: { message: MODEL_LISTING_FAILURE_MESSAGE, type: 'api_error', code: MODEL_LISTING_FAILURE_CODE } }, 502); } - if (e instanceof Error && /Malformed .* upstream config/.test(e.message)) { + if (malformedConfigResponse(e)) { return c.json({ error: errorMessage(e) }, 400); } throw e; } }; + +// Saved refreshes accept only an id, then read the current config and version +// from storage. A stale editor cannot publish a draft under the saved row. +export const fetchSavedModels = async (c: AuthedContext<'/:id/list-models'>) => { + const id = c.req.param('id'); + const record = await getRepo().upstreams.getById(id); + if (record === null) return c.json({ error: 'Upstream not found' }, 404); + + let fetcher: Fetcher; + try { + fetcher = await resolveControlPlaneFetcher({ + override: record.proxyFallbackList, + upstreamId: id, + runtimeLocation: getRuntimeLocation(c.req.raw), + }); + } catch (err) { + return c.json({ error: errorMessage(err) }, 400); + } + + try { + if (record.kind === 'custom') { + const config = assertCustomUpstreamRecord(record).config; + let result: Awaited> | undefined; + await fetchUpstreamModels(createProvider(record), fetcher, async () => { + result = await fetchCustomModels(config, fetcher); + return projectCustomModels(record, result); + }); + // Joining another runtime's refresh does not expose its raw custom wire + // response, which the editor needs for endpoint inference. + result ??= await fetchCustomModels(config, fetcher); + return c.json({ kind: record.kind, data: result.data }); + } + const models = await fetchUpstreamModels(createProvider(record), fetcher); + return c.json({ kind: record.kind, data: models.map(reshapeModelForDashboard) }); + } catch (e) { + if (e instanceof ProviderModelsUnavailableError) { + return c.json({ error: { message: MODEL_LISTING_FAILURE_MESSAGE, type: 'api_error', code: MODEL_LISTING_FAILURE_CODE } }, 502); + } + if (malformedConfigResponse(e)) return c.json({ error: errorMessage(e) }, 400); + throw e; + } +}; diff --git a/packages/gateway/src/control-plane/upstreams/routes.ts b/packages/gateway/src/control-plane/upstreams/routes.ts index f598f22a5..c1d9f2813 100644 --- a/packages/gateway/src/control-plane/upstreams/routes.ts +++ b/packages/gateway/src/control-plane/upstreams/routes.ts @@ -246,6 +246,7 @@ export const createUpstream = async (c: CtxWithJson) sortOrder: body.sort_order ?? nextSortOrder(existing), createdAt: now, updatedAt: now, + configVersion: 1, flagOverrides: body.flag_overrides ?? {}, disabledPublicModelIds: body.disabled_public_model_ids ?? [], proxyFallbackList, @@ -280,8 +281,8 @@ export const createUpstream = async (c: CtxWithJson) const record = { ...upstream, config: config.value }; // Answer with the catalog status this warm produced, not the one the record // was built with — the dashboard re-seeds its draft from this body. - const modelsCache = await saveAndWarmUpstreamForModels({ previous: null, next: record }, c); - return c.json(await serializeForResponse({ ...record, modelsCache }, knownProxyIds), 201); + const saved = await saveAndWarmUpstreamForModels({ previous: null, next: record }, c); + return c.json(await serializeForResponse(saved, knownProxyIds), 201); }; export const updateUpstream = async (c: CtxWithJson) => { @@ -334,8 +335,8 @@ export const updateUpstream = async (c: CtxWithJson) => { diff --git a/packages/gateway/src/data-plane/models/shared.ts b/packages/gateway/src/data-plane/models/shared.ts index f415d9b50..7c7ad60ac 100644 --- a/packages/gateway/src/data-plane/models/shared.ts +++ b/packages/gateway/src/data-plane/models/shared.ts @@ -5,7 +5,7 @@ export const MODEL_LISTING_FAILURE_MESSAGE = 'Upstream model listing failed'; // The message says nothing about the upstream and is prose, so the upstream -// list-models route pairs it with this code and the dashboard tells that +// catalog action pairs it with this code and the dashboard tells that // failure apart from an arbitrary one without matching English. The model-list // snapshot listing routes do not use this action-specific discriminator. export const MODEL_LISTING_FAILURE_CODE = 'upstream_model_listing_failed'; diff --git a/packages/gateway/src/data-plane/providers/models-refresh.ts b/packages/gateway/src/data-plane/providers/models-refresh.ts index e6dc89354..04584731f 100644 --- a/packages/gateway/src/data-plane/providers/models-refresh.ts +++ b/packages/gateway/src/data-plane/providers/models-refresh.ts @@ -1,6 +1,6 @@ import type { GatewayProvider } from './registry.ts'; import { getRepo } from '../../repo/index.ts'; -import { MODEL_CATALOG_REVISION, modelsFetchIdentity } from '../../repo/models-cache-contract.ts'; +import { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; import { MODELS_REFRESH_CLAIM_LEASE_MS } from '../../repo/models-refresh-contract.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; import type { Fetcher, ProviderModel } from '@floway-dev/provider'; @@ -99,8 +99,7 @@ const runClaimedRefresh = async ( if (outcome.kind === 'completed') { const current = await repo.upstreams.getById(instance.upstreamId); if (current === null - || current.updatedAt !== instance.modelsCacheGeneration.updatedAt - || modelsFetchIdentity(current) !== instance.modelsCacheGeneration.fetchIdentity) return null; + || current.configVersion !== instance.modelsCacheGeneration.configVersion) return null; instance.modelsCache = current.modelsCache; if (intent === 'explicit' && current.modelsCache?.lastError !== null && current.modelsCache?.lastError !== undefined) { observedActiveToken = null; @@ -171,7 +170,7 @@ const runClaimedRefresh = async ( const inFlightKey = (instance: GatewayProvider): string => { const generation = instance.modelsCacheGeneration; - return `${instance.upstreamId}\0${generation.updatedAt}\0${generation.fetchIdentity}`; + return `${instance.upstreamId}\0${generation.configVersion}`; }; export const fetchUpstreamModels = async ( diff --git a/packages/gateway/src/data-plane/providers/registry.ts b/packages/gateway/src/data-plane/providers/registry.ts index 04d786b0a..c6502ca09 100644 --- a/packages/gateway/src/data-plane/providers/registry.ts +++ b/packages/gateway/src/data-plane/providers/registry.ts @@ -1,7 +1,6 @@ import { getRepo } from '../../repo/index.ts'; import { modelsCacheGeneration } from '../../repo/models-cache-contract.ts'; import type { ModelsCacheGeneration } from '../../repo/types.ts'; -import { serializeStoredConfig } from '../../repo/upstream-json.ts'; import type { FlagDefaults, Provider, ProviderModule, UpstreamProviderKind, UpstreamRecord } from '@floway-dev/provider'; import { azureProviderModule } from '@floway-dev/provider-azure'; import { claudeCodeProviderModule } from '@floway-dev/provider-claude-code'; @@ -23,16 +22,6 @@ export type GatewayProvider = Provider & { readonly modelsCacheGeneration: ModelsCacheGeneration; }; -export const modelsCatalogIdentity = (record: UpstreamRecord): string => - serializeStoredConfig({ kind: record.kind, identity: providersByKind[record.kind].modelCatalogIdentity(record) }); - -export const modelsRequestIdentity = (record: UpstreamRecord): string => - serializeStoredConfig({ - kind: record.kind, - identity: providersByKind[record.kind].modelRequestIdentity(record), - proxyFallbackList: record.proxyFallbackList, - }); - export const createProvider = ( record: UpstreamRecord, cacheGeneration: ModelsCacheGeneration = modelsCacheGeneration(record), diff --git a/packages/gateway/src/repo/models-cache-contract.ts b/packages/gateway/src/repo/models-cache-contract.ts index ad0f35c6e..852248c8f 100644 --- a/packages/gateway/src/repo/models-cache-contract.ts +++ b/packages/gateway/src/repo/models-cache-contract.ts @@ -1,5 +1,4 @@ import type { ModelsCacheGeneration } from './types.ts'; -import { serializeStoredConfig } from './upstream-json.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; // Persisted ProviderModel rows contain code-derived metadata as well as the @@ -9,28 +8,8 @@ export const MODEL_CATALOG_REVISION = 5; // Fetch ownership survives provider-managed state writes such as token // rotation, but changes whenever static request inputs or egress policy do. -export const modelsFetchIdentity = ( - record: Pick, -): string => serializeStoredConfig({ - kind: record.kind, - config: record.config, - proxyFallbackList: record.proxyFallbackList, -}); - -// Control-plane credential and transport changes reset refresh cooldown even -// when the provider says the previous catalog remains valid. -export const modelsOperatorRefreshIdentity = ( - record: Pick, -): string => serializeStoredConfig({ - kind: record.kind, - config: record.config, - state: record.state ?? null, - proxyFallbackList: record.proxyFallbackList, -}); - export const modelsCacheGeneration = ( - record: Pick, + record: Pick, ): ModelsCacheGeneration => ({ - updatedAt: record.updatedAt, - fetchIdentity: modelsFetchIdentity(record), + configVersion: record.configVersion, }); diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index a5de8cbba..678bbab97 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -2,7 +2,7 @@ import { normalizeDisabledPublicModelIds } from './disabled-public-models.ts'; import { SqlExpirationSweepsRepo } from './expiration-sweeps-sql.ts'; import { normalizeFlagOverrides } from './flag-overrides.ts'; import { decodeAliasTargets, decodeAnnouncedMetadata, encodeAliasTargets, encodeAnnouncedMetadata } from './model-alias-codecs.ts'; -import { MODEL_CATALOG_REVISION, modelsFetchIdentity } from './models-cache-contract.ts'; +import { MODEL_CATALOG_REVISION } from './models-cache-contract.ts'; import { modelsRefreshRetryAt } from './models-refresh-contract.ts'; import { querySqlPerformanceOverview } from './performance-overview-sql.ts'; import { normalizeProxyFallbackList } from './proxy-fallback-list.ts'; @@ -20,7 +20,6 @@ import type { AgentSetupRenewal, AgentSetupRepository, BackoffRow, - ModelsCacheGeneration, ModelsRefreshClaimInput, ModelsRefreshClaimResult, ModelsRefreshFailureInput, @@ -882,14 +881,14 @@ class SqlUpstreamRepo implements UpstreamRepo { async list(): Promise { const { results } = await this.db - .prepare('SELECT id, provider, name, enabled, sort_order, created_at, updated_at, config_json, state_json, models_cache_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue FROM upstreams ORDER BY sort_order, created_at') + .prepare('SELECT id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, models_cache_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue FROM upstreams ORDER BY sort_order, created_at') .all(); return results.map(toUpstreamRecord); } async getById(id: string): Promise { const row = await this.db - .prepare('SELECT id, provider, name, enabled, sort_order, created_at, updated_at, config_json, state_json, models_cache_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue FROM upstreams WHERE id = ?') + .prepare('SELECT id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, models_cache_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue FROM upstreams WHERE id = ?') .bind(id) .first(); return row ? toUpstreamRecord(row) : null; @@ -900,8 +899,9 @@ class SqlUpstreamRepo implements UpstreamRepo { } async insertForModels(upstream: UpstreamRecord): Promise { + if (upstream.configVersion !== 1) throw new Error(`New upstream ${upstream.id} must start at config version 1`); const result = await this.db - .prepare('INSERT INTO upstreams (id, provider, name, enabled, sort_order, created_at, updated_at, config_json, state_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING') + .prepare('INSERT INTO upstreams (id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING') .bind( upstream.id, upstream.kind, @@ -910,6 +910,7 @@ class SqlUpstreamRepo implements UpstreamRepo { upstream.sortOrder, upstream.createdAt, upstream.updatedAt, + upstream.configVersion, serializeStoredConfig(upstream.config), serializeStoredState(upstream.state), JSON.stringify(normalizeFlagOverrides(upstream.flagOverrides)), @@ -924,15 +925,18 @@ class SqlUpstreamRepo implements UpstreamRepo { async replaceForModels(input: { previous: UpstreamRecord; - upstream: UpstreamRecord; - cachePolicy: 'preserve' | 'reset-refresh' | 'clear'; + upstream: Omit; }): Promise { - const { previous, upstream, cachePolicy } = input; + const { previous, upstream } = input; + const configChanged = previous.kind !== upstream.kind + || serializeStoredConfig(previous.config) !== serializeStoredConfig(upstream.config); + const configVersion = previous.configVersion + (configChanged ? 1 : 0); + const transportChanged = serializeStoredConfig(previous.proxyFallbackList) !== serializeStoredConfig(upstream.proxyFallbackList); const replaceState = serializeStoredState(previous.state) !== serializeStoredState(upstream.state); - const modelsRefreshUpdate = cachePolicy === 'preserve' - ? "CASE WHEN models_refresh_json IS NULL THEN NULL ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) END" - : 'NULL'; - const modelsCacheUpdate = cachePolicy === 'clear' ? ', models_cache_json = NULL' : ''; + const modelsRefreshUpdate = configChanged || transportChanged + ? 'NULL' + : "CASE WHEN models_refresh_json IS NULL THEN NULL ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) END"; + const modelsCacheUpdate = configChanged ? ', models_cache_json = NULL' : ''; const result = await this.db .prepare( `UPDATE upstreams SET @@ -941,6 +945,7 @@ class SqlUpstreamRepo implements UpstreamRepo { enabled = ?, sort_order = ?, updated_at = ?, + config_version = ?, config_json = ?, state_json = CASE WHEN ? THEN ? ELSE state_json END, flag_overrides = ?, @@ -955,6 +960,7 @@ class SqlUpstreamRepo implements UpstreamRepo { AND enabled = ? AND sort_order = ? AND updated_at = ? + AND config_version = ? AND config_json = ? AND (? = 0 OR state_json IS ?) AND flag_overrides = ? @@ -969,6 +975,7 @@ class SqlUpstreamRepo implements UpstreamRepo { upstream.enabled ? 1 : 0, upstream.sortOrder, upstream.updatedAt, + configVersion, serializeStoredConfig(upstream.config), sqliteBoolean(replaceState), serializeStoredState(upstream.state), @@ -983,6 +990,7 @@ class SqlUpstreamRepo implements UpstreamRepo { previous.enabled ? 1 : 0, previous.sortOrder, previous.updatedAt, + previous.configVersion, serializeStoredConfig(previous.config), sqliteBoolean(replaceState), serializeStoredState(previous.state), @@ -997,17 +1005,21 @@ class SqlUpstreamRepo implements UpstreamRepo { } private async saveRecord(upstream: UpstreamRecord): Promise { + if (upstream.configVersion !== 1 && await this.getById(upstream.id) === null) { + throw new Error(`New upstream ${upstream.id} must start at config version 1`); + } // created_at is deliberately not in the ON CONFLICT update list: the row's first INSERT // wins, and re-saves preserve that timestamp regardless of what the caller passes. await this.db .prepare( - `INSERT INTO upstreams (id, provider, name, enabled, sort_order, created_at, updated_at, config_json, state_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `INSERT INTO upstreams (id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET provider = excluded.provider, name = excluded.name, enabled = excluded.enabled, sort_order = excluded.sort_order, updated_at = excluded.updated_at, + config_version = CASE WHEN provider = excluded.provider AND config_json = excluded.config_json THEN config_version ELSE config_version + 1 END, config_json = excluded.config_json, state_json = excluded.state_json, flag_overrides = excluded.flag_overrides, @@ -1015,7 +1027,12 @@ class SqlUpstreamRepo implements UpstreamRepo { proxy_fallback_list_json = excluded.proxy_fallback_list_json, model_prefix_json = excluded.model_prefix_json, hue = excluded.hue, - models_refresh_json = CASE WHEN models_refresh_json IS NULL THEN NULL ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) END`, + models_cache_json = CASE WHEN provider = excluded.provider AND config_json = excluded.config_json THEN models_cache_json ELSE NULL END, + models_refresh_json = CASE + WHEN provider != excluded.provider OR config_json != excluded.config_json THEN NULL + WHEN models_refresh_json IS NULL THEN NULL + ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) + END`, ) .bind( upstream.id, @@ -1025,6 +1042,7 @@ class SqlUpstreamRepo implements UpstreamRepo { upstream.sortOrder, upstream.createdAt, upstream.updatedAt, + upstream.configVersion, serializeStoredConfig(upstream.config), serializeStoredState(upstream.state), JSON.stringify(normalizeFlagOverrides(upstream.flagOverrides)), @@ -1047,11 +1065,9 @@ class SqlUpstreamRepo implements UpstreamRepo { async finalizeModelsRefreshSuccess(input: ModelsRefreshSuccessInput): Promise { const { id, generation, token, cache } = input; - const fence = await this.modelsRefreshWriteFence(id, generation); - if (fence === null) return false; const result = await this.db - .prepare("UPDATE upstreams SET models_cache_json = ?, models_refresh_json = NULL WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") - .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, token) + .prepare("UPDATE upstreams SET models_cache_json = ?, models_refresh_json = NULL WHERE id = ? AND config_version = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") + .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, generation.configVersion, token) .run(); return (result.meta.changes ?? 0) > 0; } @@ -1060,8 +1076,6 @@ class SqlUpstreamRepo implements UpstreamRepo { const { id, generation, token, error, previousFailureCount, failedAt } = input; const failureCount = previousFailureCount + 1; const retryAt = modelsRefreshRetryAt(failedAt, previousFailureCount); - const fence = await this.modelsRefreshWriteFence(id, generation); - if (fence === null) return false; // A cold failure remains immediately stale while preserving the error for // the next request and dashboard read. const coldFailure = encodeUpstreamModelsCache({ revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }); @@ -1070,30 +1084,26 @@ class SqlUpstreamRepo implements UpstreamRepo { `UPDATE upstreams SET models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END, models_refresh_json = json_object('failCount', ?, 'retryAt', ?, 'claimToken', NULL, 'claimedAt', NULL) - WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ? + WHERE id = ? AND config_version = ? AND json_extract(models_refresh_json, '$.claimToken') = ? AND coalesce(json_extract(models_refresh_json, '$.failCount'), 0) = ?`, ) - .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, token, previousFailureCount) + .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, generation.configVersion, token, previousFailureCount) .run(); return (result.meta.changes ?? 0) > 0; } async abandonModelsRefresh(input: ModelsRefreshOwnerInput): Promise { const { id, generation, token } = input; - const fence = await this.modelsRefreshWriteFence(id, generation); - if (fence === null) return false; const result = await this.db - .prepare("UPDATE upstreams SET models_refresh_json = json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") - .bind(id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, token) + .prepare("UPDATE upstreams SET models_refresh_json = json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) WHERE id = ? AND config_version = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") + .bind(id, generation.configVersion, token) .run(); return (result.meta.changes ?? 0) > 0; } async claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { const { id, generation, token, now, staleClaimedBefore, bypassBackoff, observedActiveToken } = input; - const fence = await this.modelsRefreshWriteFence(id, generation); - if (fence === null) return { kind: 'generation-mismatch' }; while (true) { const row = await this.db .prepare( @@ -1104,7 +1114,7 @@ class SqlUpstreamRepo implements UpstreamRepo { 'claimToken', ?, 'claimedAt', ? ) - WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ? AND ( + WHERE id = ? AND config_version = ? AND ( ? IS NULL AND ( models_refresh_json IS NULL OR ( @@ -1121,7 +1131,7 @@ class SqlUpstreamRepo implements UpstreamRepo { ) RETURNING json_extract(models_refresh_json, '$.failCount') AS fail_count`, ) - .bind(token, now, id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList, observedActiveToken, sqliteBoolean(bypassBackoff), now, staleClaimedBefore, observedActiveToken, staleClaimedBefore) + .bind(token, now, id, generation.configVersion, observedActiveToken, sqliteBoolean(bypassBackoff), now, staleClaimedBefore, observedActiveToken, staleClaimedBefore) .first<{ fail_count: number }>(); if (row !== null) return { kind: 'claimed', failureCount: row.fail_count }; @@ -1131,9 +1141,9 @@ class SqlUpstreamRepo implements UpstreamRepo { json_extract(models_refresh_json, '$.retryAt') AS retry_at, json_extract(models_refresh_json, '$.claimToken') AS claim_token, json_extract(models_refresh_json, '$.claimedAt') AS claimed_at - FROM upstreams WHERE id = ? AND updated_at = ? AND provider = ? AND config_json = ? AND proxy_fallback_list_json = ?`, + FROM upstreams WHERE id = ? AND config_version = ?`, ) - .bind(id, generation.updatedAt, fence.provider, fence.config, fence.proxyFallbackList) + .bind(id, generation.configVersion) .first<{ models_refresh_json: string | null; retry_at: number | null; claim_token: string | null; claimed_at: number | null }>(); if (state === null) return { kind: 'generation-mismatch' }; if (state.models_refresh_json === null) { @@ -1146,26 +1156,6 @@ class SqlUpstreamRepo implements UpstreamRepo { } } - private async modelsRefreshWriteFence(id: string, generation: ModelsCacheGeneration): Promise<{ - provider: string; - config: string; - proxyFallbackList: string; - } | null> { - const row = await this.db - .prepare('SELECT updated_at, provider, config_json, proxy_fallback_list_json FROM upstreams WHERE id = ?') - .bind(id) - .first<{ updated_at: string; provider: string; config_json: string; proxy_fallback_list_json: string }>(); - if (row === null || row.updated_at !== generation.updatedAt) return null; - const identity = modelsFetchIdentity({ - kind: parseUpstreamKind(id, row.provider), - config: decodeUpstreamConfig(row.config_json, id), - proxyFallbackList: parseProxyFallbackList(id, row.proxy_fallback_list_json), - }); - return identity === generation.fetchIdentity - ? { provider: row.provider, config: row.config_json, proxyFallbackList: row.proxy_fallback_list_json } - : null; - } - // Read-modify-write under optimistic concurrency, retried against the winner // on a loss. `IS` (not `=`) so a row whose state_json is SQL NULL still // matches. The predicate binds the exact text this method read rather than a @@ -1209,6 +1199,7 @@ interface UpstreamRow { sort_order: number; created_at: string; updated_at: string; + config_version: number; config_json: string; state_json: string | null; models_cache_json: string | null; @@ -1222,6 +1213,9 @@ interface UpstreamRow { const toUpstreamRecord = (row: UpstreamRow): UpstreamRecord => { const config = decodeUpstreamConfig(row.config_json, row.id); const state = row.state_json === null ? null : decodeUpstreamState(row.state_json, row.id); + if (!Number.isSafeInteger(row.config_version) || row.config_version < 1) { + throw new Error(`Invalid upstream config version for ${row.id}`); + } return { id: row.id, @@ -1232,6 +1226,7 @@ const toUpstreamRecord = (row: UpstreamRow): UpstreamRecord => { sortOrder: row.sort_order, createdAt: row.created_at, updatedAt: row.updated_at, + configVersion: row.config_version, config, state, flagOverrides: parseFlagOverrides(row.id, row.flag_overrides), diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index 09f43e64c..f3cb0f213 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -347,8 +347,7 @@ export interface UpstreamRepo { insertForModels(upstream: UpstreamRecord): Promise; replaceForModels(input: { previous: UpstreamRecord; - upstream: UpstreamRecord; - cachePolicy: 'preserve' | 'reset-refresh' | 'clear'; + upstream: Omit; }): Promise; delete(id: string): Promise; deleteAll(): Promise; @@ -412,8 +411,7 @@ export type ModelsRefreshClaimResult = ModelsRefreshClaim | { kind: 'generation-mismatch' }; export interface ModelsCacheGeneration { - updatedAt: string; - fetchIdentity: string; + configVersion: number; } export interface ProxyRecord { diff --git a/packages/provider-azure/__tests__/config_test.ts b/packages/provider-azure/__tests__/config_test.ts index 7e5aed10f..c8cbd7b17 100644 --- a/packages/provider-azure/__tests__/config_test.ts +++ b/packages/provider-azure/__tests__/config_test.ts @@ -27,6 +27,7 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-azure/__tests__/fetch_test.ts b/packages/provider-azure/__tests__/fetch_test.ts index e63805b4e..db73df8fd 100644 --- a/packages/provider-azure/__tests__/fetch_test.ts +++ b/packages/provider-azure/__tests__/fetch_test.ts @@ -38,6 +38,7 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-azure/__tests__/provider_test.ts b/packages/provider-azure/__tests__/provider_test.ts index e2aaf356a..9951e4f14 100644 --- a/packages/provider-azure/__tests__/provider_test.ts +++ b/packages/provider-azure/__tests__/provider_test.ts @@ -39,6 +39,7 @@ const azureRecord = (overrides: Partial = {}): UpstreamRecord => disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, ...rest, @@ -328,6 +329,7 @@ test('createAzureProvider exposes image models and routes generations with api-v disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { @@ -377,6 +379,7 @@ test('createAzureProvider callImagesEdits posts multipart with model replaced by disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/provider-azure/src/index.ts b/packages/provider-azure/src/index.ts index 640c54fda..744129dbd 100644 --- a/packages/provider-azure/src/index.ts +++ b/packages/provider-azure/src/index.ts @@ -1,12 +1,9 @@ -import { assertAzureUpstreamRecord } from './config.ts'; import { AZURE_DEFAULT_FLAGS } from './defaults.ts'; import { createAzureProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const azureProviderModule: ProviderModule = { create: createAzureProvider, - modelCatalogIdentity: record => assertAzureUpstreamRecord(record).config, - modelRequestIdentity: record => assertAzureUpstreamRecord(record).config, defaultFlags: AZURE_DEFAULT_FLAGS, }; export { assertAzureUpstreamRecord, type AzureUpstreamConfig } from './config.ts'; diff --git a/packages/provider-claude-code/__tests__/access-token_test.ts b/packages/provider-claude-code/__tests__/access-token_test.ts index 862382165..85e3e0776 100644 --- a/packages/provider-claude-code/__tests__/access-token_test.ts +++ b/packages/provider-claude-code/__tests__/access-token_test.ts @@ -37,6 +37,7 @@ const makeRecord = (state: ClaudeCodeUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-claude-code/__tests__/config_test.ts b/packages/provider-claude-code/__tests__/config_test.ts index e2ebd1bce..503895753 100644 --- a/packages/provider-claude-code/__tests__/config_test.ts +++ b/packages/provider-claude-code/__tests__/config_test.ts @@ -14,7 +14,7 @@ const good = { accounts: [goodAccount] }; const wrap = (config: unknown): UpstreamRecord => ({ id: 'up', kind: 'claude-code', name: 'n', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: config as UpstreamRecord['config'], state: null, - flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210, + flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, configVersion: 1, modelsCache: null, hue: 210, }); describe('assertClaudeCodeUpstreamRecord (config validation)', () => { @@ -66,7 +66,7 @@ describe('assertClaudeCodeUpstreamRecord (record-level checks)', () => { const record: UpstreamRecord = { id: 'up', kind: 'copilot', name: 'n', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: {}, state: null, - flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210, + flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, configVersion: 1, modelsCache: null, hue: 210, }; expect(() => assertClaudeCodeUpstreamRecord(record)).toThrow(); }); diff --git a/packages/provider-claude-code/__tests__/fetch_test.ts b/packages/provider-claude-code/__tests__/fetch_test.ts index f923c7aae..aa91610e0 100644 --- a/packages/provider-claude-code/__tests__/fetch_test.ts +++ b/packages/provider-claude-code/__tests__/fetch_test.ts @@ -60,6 +60,7 @@ const makeRecord = (state: ClaudeCodeUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-claude-code/__tests__/provider_test.ts b/packages/provider-claude-code/__tests__/provider_test.ts index 0db3f87d0..8f4466955 100644 --- a/packages/provider-claude-code/__tests__/provider_test.ts +++ b/packages/provider-claude-code/__tests__/provider_test.ts @@ -54,6 +54,7 @@ const makeRecord = (state: ClaudeCodeUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-claude-code/src/index.ts b/packages/provider-claude-code/src/index.ts index 66dbbbb78..13fe12d88 100644 --- a/packages/provider-claude-code/src/index.ts +++ b/packages/provider-claude-code/src/index.ts @@ -1,18 +1,9 @@ -import { assertClaudeCodeUpstreamRecord } from './config.ts'; import { CLAUDE_CODE_DEFAULT_FLAGS } from './defaults.ts'; import { createClaudeCodeProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const claudeCodeProviderModule: ProviderModule = { create: createClaudeCodeProvider, - modelCatalogIdentity: record => { - assertClaudeCodeUpstreamRecord(record); - return record.config; - }, - modelRequestIdentity: record => { - assertClaudeCodeUpstreamRecord(record); - return record.config; - }, defaultFlags: CLAUDE_CODE_DEFAULT_FLAGS, }; diff --git a/packages/provider-codex/__tests__/access-token_test.ts b/packages/provider-codex/__tests__/access-token_test.ts index 31e4eface..51acc843a 100644 --- a/packages/provider-codex/__tests__/access-token_test.ts +++ b/packages/provider-codex/__tests__/access-token_test.ts @@ -28,6 +28,7 @@ const makeRecord = (state: CodexUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-codex/__tests__/config_test.ts b/packages/provider-codex/__tests__/config_test.ts index 293437603..580f71610 100644 --- a/packages/provider-codex/__tests__/config_test.ts +++ b/packages/provider-codex/__tests__/config_test.ts @@ -9,7 +9,7 @@ const good = { accounts: [goodAccount] }; const wrap = (config: unknown): UpstreamRecord => ({ id: 'up', kind: 'codex', name: 'n', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: config as UpstreamRecord['config'], state: null, - flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210, + flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, configVersion: 1, modelsCache: null, hue: 210, }); describe('assertCodexUpstreamRecord (config validation)', () => { @@ -42,7 +42,7 @@ describe('assertCodexUpstreamRecord (record-level checks)', () => { const record: UpstreamRecord = { id: 'up', kind: 'copilot', name: 'n', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: {}, state: null, - flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210, + flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, configVersion: 1, modelsCache: null, hue: 210, }; expect(() => assertCodexUpstreamRecord(record)).toThrow(); }); diff --git a/packages/provider-codex/__tests__/fetch_test.ts b/packages/provider-codex/__tests__/fetch_test.ts index f3b222930..e55a7fd18 100644 --- a/packages/provider-codex/__tests__/fetch_test.ts +++ b/packages/provider-codex/__tests__/fetch_test.ts @@ -39,6 +39,7 @@ const makeRecord = (state: CodexUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-codex/__tests__/interceptors/responses/action-pivot_test.ts b/packages/provider-codex/__tests__/interceptors/responses/action-pivot_test.ts index 6b9dda50e..1a93caea2 100644 --- a/packages/provider-codex/__tests__/interceptors/responses/action-pivot_test.ts +++ b/packages/provider-codex/__tests__/interceptors/responses/action-pivot_test.ts @@ -49,6 +49,7 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-codex/__tests__/provider_test.ts b/packages/provider-codex/__tests__/provider_test.ts index 84c2fd9a1..4a09a1680 100644 --- a/packages/provider-codex/__tests__/provider_test.ts +++ b/packages/provider-codex/__tests__/provider_test.ts @@ -24,6 +24,7 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-codex/__tests__/quota_test.ts b/packages/provider-codex/__tests__/quota_test.ts index d4ee123ef..5fbc7c91c 100644 --- a/packages/provider-codex/__tests__/quota_test.ts +++ b/packages/provider-codex/__tests__/quota_test.ts @@ -30,6 +30,7 @@ const makeRecord = (state: CodexUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-codex/src/index.ts b/packages/provider-codex/src/index.ts index e03e4d078..d9e52b55d 100644 --- a/packages/provider-codex/src/index.ts +++ b/packages/provider-codex/src/index.ts @@ -1,18 +1,9 @@ -import { assertCodexUpstreamRecord } from './config.ts'; import { CODEX_DEFAULT_FLAGS } from './defaults.ts'; import { createCodexProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const codexProviderModule: ProviderModule = { create: createCodexProvider, - modelCatalogIdentity: record => { - assertCodexUpstreamRecord(record); - return record.config; - }, - modelRequestIdentity: record => { - assertCodexUpstreamRecord(record); - return record.config; - }, defaultFlags: CODEX_DEFAULT_FLAGS, }; diff --git a/packages/provider-copilot/__tests__/auth_test.ts b/packages/provider-copilot/__tests__/auth_test.ts index 00f9a7e98..7ed7e622b 100644 --- a/packages/provider-copilot/__tests__/auth_test.ts +++ b/packages/provider-copilot/__tests__/auth_test.ts @@ -31,6 +31,7 @@ const installRepoAndClearCache = async () => { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { githubHost: 'github.com', githubToken: 'ghu_test', user: { id: 1, login: 't', name: null, avatar_url: '' } }, @@ -426,6 +427,7 @@ test('copilotAuthedFetch persists a minted token even when the row changed durin disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { githubHost: 'github.com', githubToken: 'ghu_test', user: { id: 1, login: 't', name: null, avatar_url: '' } }, diff --git a/packages/provider-copilot/__tests__/fetch-models_test.ts b/packages/provider-copilot/__tests__/fetch-models_test.ts index d0d6d95b9..03c803010 100644 --- a/packages/provider-copilot/__tests__/fetch-models_test.ts +++ b/packages/provider-copilot/__tests__/fetch-models_test.ts @@ -21,6 +21,7 @@ const installRepoAndConfig = async () => { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { githubHost: 'github.com', githubToken, user: { id: 1, login: 't', name: null, avatar_url: '' } }, diff --git a/packages/provider-copilot/__tests__/interceptors/responses/action-pivot_test.ts b/packages/provider-copilot/__tests__/interceptors/responses/action-pivot_test.ts index f27083f67..5b1c26c9f 100644 --- a/packages/provider-copilot/__tests__/interceptors/responses/action-pivot_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/responses/action-pivot_test.ts @@ -50,6 +50,7 @@ test('Copilot provider terminal dispatches on post-chain ctx.action (interceptor disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/provider-copilot/__tests__/provider_test.ts b/packages/provider-copilot/__tests__/provider_test.ts index 0411ce1b9..05758013c 100644 --- a/packages/provider-copilot/__tests__/provider_test.ts +++ b/packages/provider-copilot/__tests__/provider_test.ts @@ -39,6 +39,7 @@ const buildCopilotUpstream = (overrides: Partial = {}): Upstream disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, ...rest, diff --git a/packages/provider-copilot/src/index.ts b/packages/provider-copilot/src/index.ts index aee7fdd59..c97adaedb 100644 --- a/packages/provider-copilot/src/index.ts +++ b/packages/provider-copilot/src/index.ts @@ -1,18 +1,9 @@ -import { assertCopilotUpstreamRecord } from './config.ts'; import { COPILOT_DEFAULT_FLAGS } from './defaults.ts'; import { createCopilotProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const copilotProviderModule: ProviderModule = { create: createCopilotProvider, - modelCatalogIdentity: record => { - const upstream = assertCopilotUpstreamRecord(record); - return { githubHost: upstream.config.githubHost, userId: upstream.config.user.id }; - }, - modelRequestIdentity: record => { - const upstream = assertCopilotUpstreamRecord(record); - return upstream.config; - }, defaultFlags: COPILOT_DEFAULT_FLAGS, }; diff --git a/packages/provider-custom/__tests__/config_test.ts b/packages/provider-custom/__tests__/config_test.ts index 2f69cb7a2..fdb20b9fa 100644 --- a/packages/provider-custom/__tests__/config_test.ts +++ b/packages/provider-custom/__tests__/config_test.ts @@ -24,6 +24,7 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-custom/__tests__/fetch-models_test.ts b/packages/provider-custom/__tests__/fetch-models_test.ts index 4796ea9cf..6e7c55751 100644 --- a/packages/provider-custom/__tests__/fetch-models_test.ts +++ b/packages/provider-custom/__tests__/fetch-models_test.ts @@ -16,6 +16,7 @@ const upstreamRecord = () => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/provider-custom/__tests__/fetch_test.ts b/packages/provider-custom/__tests__/fetch_test.ts index 0896b04e7..26c0ab18a 100644 --- a/packages/provider-custom/__tests__/fetch_test.ts +++ b/packages/provider-custom/__tests__/fetch_test.ts @@ -37,6 +37,7 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-custom/__tests__/infer-endpoints_test.ts b/packages/provider-custom/__tests__/infer-endpoints_test.ts index 6bf930e5b..9199727f5 100644 --- a/packages/provider-custom/__tests__/infer-endpoints_test.ts +++ b/packages/provider-custom/__tests__/infer-endpoints_test.ts @@ -117,6 +117,7 @@ test('Custom provider projects gpt-image-* models with kind=image and both image disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-custom/__tests__/provider_test.ts b/packages/provider-custom/__tests__/provider_test.ts index ffe10a4a5..8af363489 100644 --- a/packages/provider-custom/__tests__/provider_test.ts +++ b/packages/provider-custom/__tests__/provider_test.ts @@ -26,6 +26,7 @@ const buildCustomUpstream = (options: BuildOptions = {}): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/provider-custom/src/index.ts b/packages/provider-custom/src/index.ts index ec41e2929..6c3fc8264 100644 --- a/packages/provider-custom/src/index.ts +++ b/packages/provider-custom/src/index.ts @@ -1,12 +1,9 @@ -import { assertCustomUpstreamRecord } from './config.ts'; import { CUSTOM_DEFAULT_FLAGS } from './defaults.ts'; import { createCustomProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const customProviderModule: ProviderModule = { create: createCustomProvider, - modelCatalogIdentity: record => assertCustomUpstreamRecord(record).config, - modelRequestIdentity: record => assertCustomUpstreamRecord(record).config, defaultFlags: CUSTOM_DEFAULT_FLAGS, }; diff --git a/packages/provider-ollama/__tests__/config_test.ts b/packages/provider-ollama/__tests__/config_test.ts index 6a99a3674..361a77e76 100644 --- a/packages/provider-ollama/__tests__/config_test.ts +++ b/packages/provider-ollama/__tests__/config_test.ts @@ -21,6 +21,7 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-ollama/__tests__/fetch-models_test.ts b/packages/provider-ollama/__tests__/fetch-models_test.ts index 3dc01d6ea..7d6a3179b 100644 --- a/packages/provider-ollama/__tests__/fetch-models_test.ts +++ b/packages/provider-ollama/__tests__/fetch-models_test.ts @@ -19,6 +19,7 @@ const config: OllamaUpstreamConfig = assertOllamaUpstreamRecord({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }).config; diff --git a/packages/provider-ollama/__tests__/fetch_test.ts b/packages/provider-ollama/__tests__/fetch_test.ts index d81628d16..809bced6a 100644 --- a/packages/provider-ollama/__tests__/fetch_test.ts +++ b/packages/provider-ollama/__tests__/fetch_test.ts @@ -32,6 +32,7 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-ollama/__tests__/provider_test.ts b/packages/provider-ollama/__tests__/provider_test.ts index d22e222b0..ab222778c 100644 --- a/packages/provider-ollama/__tests__/provider_test.ts +++ b/packages/provider-ollama/__tests__/provider_test.ts @@ -19,6 +19,7 @@ const buildRecord = (overrides: Partial = {}): UpstreamRecord => disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, + configVersion: 1, modelsCache: null, hue: 210, ...overrides, diff --git a/packages/provider-ollama/src/index.ts b/packages/provider-ollama/src/index.ts index 6ef118e80..49dfb890b 100644 --- a/packages/provider-ollama/src/index.ts +++ b/packages/provider-ollama/src/index.ts @@ -1,12 +1,9 @@ -import { assertOllamaUpstreamRecord } from './config.ts'; import { OLLAMA_DEFAULT_FLAGS } from './defaults.ts'; import { createOllamaProvider } from './provider.ts'; import type { ProviderModule } from '@floway-dev/provider'; export const ollamaProviderModule: ProviderModule = { create: createOllamaProvider, - modelCatalogIdentity: record => assertOllamaUpstreamRecord(record).config, - modelRequestIdentity: record => assertOllamaUpstreamRecord(record).config, defaultFlags: OLLAMA_DEFAULT_FLAGS, }; diff --git a/packages/provider/src/model-config.ts b/packages/provider/src/model-config.ts index 806fdd334..429fb4bf1 100644 --- a/packages/provider/src/model-config.ts +++ b/packages/provider/src/model-config.ts @@ -13,8 +13,8 @@ export type UpstreamChatModelConfig = ChatModelInfo; // operator authored it, PATCH persists it, and `modelsField` below is its // validator. // • Auto — the live projection of a provider's own emission, rendered by -// `POST /api/upstreams/list-models` from the `ProviderModel` the provider -// returned. Read-only; it never persists. +// the saved-refresh or draft-preview catalog operation. Read-only; it +// never persists as operator configuration. export interface UpstreamModelConfig { // Mirrors of fields that flow through to PublicModel (snake_case for parity). kind: ModelKind; diff --git a/packages/provider/src/model.ts b/packages/provider/src/model.ts index 7a927982d..ad7206210 100644 --- a/packages/provider/src/model.ts +++ b/packages/provider/src/model.ts @@ -68,6 +68,9 @@ export interface UpstreamRecord { sortOrder: number; createdAt: string; updatedAt: string; + // Monotonic generation of provider kind/configuration. Runtime state and + // operator metadata do not change it. + configVersion: number; config: unknown; // Gateway-written state that can change without an operator editing config; // null when a provider has no runtime state. diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index e88ef4137..99aefaa94 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -166,12 +166,6 @@ export interface ProviderModule { // fetch) happens on demand inside the per-request methods on the // returned ProviderInstance. create: (record: UpstreamRecord) => Provider; - // Stable identity of the upstream account/catalog namespace. Each provider - // decides which of its configuration changes can preserve a snapshot. - modelCatalogIdentity: (record: UpstreamRecord) => unknown; - // Normalized request inputs captured by the provider instance. Provider- - // managed state is reread from storage when the catalog request runs. - modelRequestIdentity: (record: UpstreamRecord) => unknown; // Exhaustive default map over every catalog flag id for a fresh // upstream of this kind; see each provider package's `defaults.ts`. defaultFlags: FlagDefaults; From e73dd6cf9f248cac6ea44a480a0390e5a8d0cdf3 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 6 Aug 2026 23:39:32 +0800 Subject: [PATCH 43/46] refactor(gateway): simplify model catalog lifecycle Keep catalog generations inside the gateway repository, unify provider-owned discovery projection, and separate saved refresh from draft preview through the dashboard. Warm only changed catalog inputs, preserve unrelated active owners, return cache status with saved refreshes, and make warm coordination side-effect-only. Fix abandoned claims so waiters reacquire ownership instead of treating release as successful completion. --- .../__tests__/node-sqlite-repo_test.ts | 5 +- .../components/upstream-editor/data_test.ts | 16 ++- .../upstream-editor/model-contract_test.ts | 40 ------- apps/web/src/api/types.ts | 2 - .../src/components/upstream-editor/data.ts | 103 +++++++----------- .../src/components/upstream-editor/page.tsx | 16 ++- .../data-transfer/routes_test.ts | 22 ++-- .../control-plane/models/routes_test.ts | 1 - .../upstreams/copilot-device-login_test.ts | 5 +- .../control-plane/upstreams/routes_test.ts | 83 +++----------- .../control-plane/upstreams/serialize_test.ts | 3 - .../__tests__/data-plane/audio/http_test.ts | 1 - .../affinity/copilot-roundtrip_test.ts | 1 - .../image-generation-integration_test.ts | 1 - .../chat/shared/target-picker_test.ts | 1 - .../data-plane/codex/routes_images_test.ts | 1 - .../__tests__/data-plane/images/http_test.ts | 1 - .../data-plane/providers/catalog_test.ts | 1 - .../data-plane/providers/models-cache_test.ts | 13 ++- .../data-plane/providers/registry_test.ts | 6 +- .../data-plane/providers/resolution_test.ts | 1 - .../__tests__/dial/per-request_test.ts | 1 - packages/gateway/__tests__/repo/memory.ts | 70 ++++++------ .../__tests__/repo/models-cache-fixture.ts | 10 ++ .../__tests__/repo/models-refresh_test.ts | 52 ++++++--- .../gateway/__tests__/repo/proxies_test.ts | 1 - packages/gateway/__tests__/repo/sql_test.ts | 4 +- .../gateway/__tests__/repo/upstreams_test.ts | 14 ++- .../scheduled/models-refresh_test.ts | 1 - packages/gateway/__tests__/test-utils/app.ts | 2 - .../data-transfer/import-schema.ts | 3 - .../src/control-plane/data-transfer/routes.ts | 4 +- packages/gateway/src/control-plane/schemas.ts | 6 +- .../shared/save-upstream-for-models.ts | 41 ++++--- .../control-plane/upstreams/claude-code.ts | 6 +- .../src/control-plane/upstreams/codex.ts | 4 +- .../src/control-plane/upstreams/copilot.ts | 4 +- .../upstreams/models-cache-status.ts | 9 ++ .../src/control-plane/upstreams/models.ts | 37 +++---- .../src/control-plane/upstreams/routes.ts | 17 ++- .../src/control-plane/upstreams/types.ts | 21 +--- .../data-plane/providers/models-refresh.ts | 62 +++++------ .../src/data-plane/providers/registry.ts | 16 +-- .../data-plane/shared/listing/addressable.ts | 5 +- .../gateway/src/repo/models-cache-contract.ts | 5 +- packages/gateway/src/repo/sql.ts | 75 +++++++------ packages/gateway/src/repo/types.ts | 16 ++- .../provider-azure/__tests__/config_test.ts | 1 - .../provider-azure/__tests__/fetch_test.ts | 1 - .../provider-azure/__tests__/provider_test.ts | 3 - .../__tests__/access-token_test.ts | 1 - .../__tests__/config_test.ts | 4 +- .../__tests__/fetch_test.ts | 1 - .../__tests__/provider_test.ts | 1 - .../__tests__/access-token_test.ts | 1 - .../provider-codex/__tests__/config_test.ts | 4 +- .../provider-codex/__tests__/fetch_test.ts | 1 - .../responses/action-pivot_test.ts | 1 - .../provider-codex/__tests__/provider_test.ts | 1 - .../provider-codex/__tests__/quota_test.ts | 1 - .../provider-copilot/__tests__/auth_test.ts | 2 - .../__tests__/fetch-models_test.ts | 1 - .../responses/action-pivot_test.ts | 1 - .../__tests__/provider_test.ts | 1 - .../provider-custom/__tests__/config_test.ts | 1 - .../__tests__/fetch-models_test.ts | 1 - .../provider-custom/__tests__/fetch_test.ts | 1 - .../__tests__/infer-endpoints_test.ts | 41 ++++++- .../__tests__/provider_test.ts | 1 - packages/provider-custom/src/index.ts | 2 +- packages/provider-custom/src/provider.ts | 25 ++++- .../provider-ollama/__tests__/config_test.ts | 1 - .../__tests__/fetch-models_test.ts | 1 - .../provider-ollama/__tests__/fetch_test.ts | 1 - .../__tests__/provider_test.ts | 1 - packages/provider/src/model.ts | 6 +- 76 files changed, 409 insertions(+), 508 deletions(-) create mode 100644 packages/gateway/src/control-plane/upstreams/models-cache-status.ts diff --git a/apps/platform-node/__tests__/node-sqlite-repo_test.ts b/apps/platform-node/__tests__/node-sqlite-repo_test.ts index f092e0c46..4120adfaf 100644 --- a/apps/platform-node/__tests__/node-sqlite-repo_test.ts +++ b/apps/platform-node/__tests__/node-sqlite-repo_test.ts @@ -87,7 +87,6 @@ test('repository JSON codecs round-trip upstream, alias, and Responses state thr updatedAt: '2026-08-05T00:00:00.000Z', config: { opaque: { value: true } }, state: { cursor: ['a', 1] }, - configVersion: 1, modelsCache: null, flagOverrides: {}, disabledPublicModelIds: [], @@ -96,7 +95,9 @@ test('repository JSON codecs round-trip upstream, alias, and Responses state thr hue: 210, }; await repo.upstreams.save(upstreamRecord); - const cacheGeneration = modelsCacheGeneration(upstreamRecord); + const storedUpstream = await repo.upstreams.getById(upstreamRecord.id); + if (storedUpstream === null) throw new Error('expected stored upstream fixture'); + const cacheGeneration = modelsCacheGeneration(storedUpstream); const cacheToken = 'node-cache-fixture'; const cacheClaim = await repo.upstreams.claimModelsRefresh({ id: 'up_node', generation: cacheGeneration, token: cacheToken, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, bypassBackoff: true, observedActiveToken: null }); if (cacheClaim.kind !== 'claimed') throw new Error('expected model-cache fixture claim'); diff --git a/apps/web/__tests__/components/upstream-editor/data_test.ts b/apps/web/__tests__/components/upstream-editor/data_test.ts index 65770035e..31ab41e57 100644 --- a/apps/web/__tests__/components/upstream-editor/data_test.ts +++ b/apps/web/__tests__/components/upstream-editor/data_test.ts @@ -1,7 +1,7 @@ import { expect, test } from 'vitest'; import type { UpstreamRecord } from '../../../src/api/types'; -import { createBody, modelCatalogOperation, previewRecord, updateBody, valuesFromRecord } from '../../../src/components/upstream-editor/data'; +import { createBody, hasDraftModelInputs, previewRecord, updateBody, valuesFromRecord } from '../../../src/components/upstream-editor/data'; import { upstreamRecord } from '../../api/upstream-fixture'; type CustomRecord = Extract; @@ -45,12 +45,10 @@ test('Custom editor values add one blank ingress row and never serialize it', () expect((previewRecord(record, values).config as CustomRecord['config']).ingressHeadersRules).toEqual(expected); }); -test('model catalog operations write cache only for the unchanged saved config', () => { - expect(modelCatalogOperation(record, {})).toBe('saved'); - expect(modelCatalogOperation(record, { config: true })).toBe('preview'); - expect(modelCatalogOperation(record, { state: true })).toBe('preview'); - expect(modelCatalogOperation(record, { proxyFallbackList: true })).toBe('preview'); - expect(modelCatalogOperation({ ...record, id: '' }, {})).toBe('preview'); - // Metadata does not alter the provider request inputs. - expect(modelCatalogOperation(record, { name: true })).toBe('saved'); +test('draft model inputs exclude metadata-only edits', () => { + expect(hasDraftModelInputs({})).toBe(false); + expect(hasDraftModelInputs({ config: true })).toBe(true); + expect(hasDraftModelInputs({ state: true })).toBe(true); + expect(hasDraftModelInputs({ proxyFallbackList: true })).toBe(true); + expect(hasDraftModelInputs({ name: true })).toBe(false); }); diff --git a/apps/web/__tests__/components/upstream-editor/model-contract_test.ts b/apps/web/__tests__/components/upstream-editor/model-contract_test.ts index fa3ed8ff6..166af9121 100644 --- a/apps/web/__tests__/components/upstream-editor/model-contract_test.ts +++ b/apps/web/__tests__/components/upstream-editor/model-contract_test.ts @@ -1,47 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { discoveredModelsFromResponse } from '../../../src/components/upstream-editor/data'; import { modelsAreValid } from '../../../src/components/upstream-editor/model-detail'; -describe('custom discovered model projection', () => { - it('maps fixed kinds to their own endpoint families', () => { - const models = discoveredModelsFromResponse({ - kind: 'custom', - data: [ - { id: 'speech', kind: 'transcription' }, - { id: 'ranker', kind: 'rerank' }, - ], - }, { chatCompletions: {} }); - - expect(models[0]?.endpoints).toEqual({ audioTranscriptions: {} }); - expect(models[1]?.endpoints).toEqual({ rerank: {} }); - }); - - it('gives a row that declares no kind the configured map the gateway gives it', () => { - const models = discoveredModelsFromResponse({ - kind: 'custom', - data: [{ id: 'bge-m3' }, { id: 'talker', kind: 'chat' }], - }, { embeddings: {} }); - - expect(models[0]?.endpoints).toEqual({ embeddings: {} }); - expect(models[1]?.endpoints).toEqual({ embeddings: {} }); - }); - - it('projects every discovered row into a shape the gateway accepts', () => { - const models = discoveredModelsFromResponse({ - kind: 'custom', - data: [ - { id: 'talker', kind: 'chat' }, - { id: 'painter', kind: 'image' }, - { id: 'speech', kind: 'transcription' }, - { id: 'ranker', kind: 'rerank' }, - ], - }, { chatCompletions: {} }); - - expect(modelsAreValid(models)).toBe(true); - }); -}); - describe('manual model validation', () => { it('rejects the same incomplete identities and endpoint contracts as the gateway', () => { expect(modelsAreValid([{ upstreamModelId: '', kind: 'chat', endpoints: { chatCompletions: {} } }])).toBe(false); diff --git a/apps/web/src/api/types.ts b/apps/web/src/api/types.ts index 1706fde00..2cea563ee 100644 --- a/apps/web/src/api/types.ts +++ b/apps/web/src/api/types.ts @@ -10,8 +10,6 @@ export type { CodexAccountCredentialState, CodexQuotaSnapshot, CodexQuotaSnapshotMap, - CustomRawModel, - ListUpstreamModelsResponse, UpstreamRecord, } from '@floway-dev/gateway/control-plane/upstreams/types'; diff --git a/apps/web/src/components/upstream-editor/data.ts b/apps/web/src/components/upstream-editor/data.ts index 889e6e909..3dab02fe3 100644 --- a/apps/web/src/components/upstream-editor/data.ts +++ b/apps/web/src/components/upstream-editor/data.ts @@ -1,16 +1,14 @@ import type { InferRequestType } from 'hono/client'; -import { configuredEndpoints, PATH_OVERRIDE_PATHS, shapeForKind } from './endpoints'; +import { PATH_OVERRIDE_PATHS } from './endpoints'; import { api, callApi } from '../../api/client'; import type { BackoffRow, - ListUpstreamModelsResponse, ProxyRecord, UpstreamRecord, UpstreamRecordEnvelope, } from '../../api/types'; import type { MODEL_LISTING_FAILURE_CODE as GatewayModelListingFailureCode } from '@floway-dev/gateway/data-plane/models/shared'; -import type { ModelEndpoints } from '@floway-dev/protocols/common'; import type { UpstreamModelConfig } from '@floway-dev/provider'; import type { UpstreamProviderKind } from '@floway-dev/provider/model'; import { MODEL_PREFIX_MAX_LENGTH, MODEL_PREFIX_REGEX } from '@floway-dev/provider/model-prefix'; @@ -48,15 +46,9 @@ export type UpstreamEditorLoaderData = UpstreamEditorLoaderDataBase & ( // record. export const isPersisted = (record: UpstreamRecord): boolean => record.id !== ''; -export const modelCatalogOperation = ( - record: UpstreamRecord, +export const hasDraftModelInputs = ( dirtyFields: Partial>, -): 'saved' | 'preview' => isPersisted(record) - && !dirtyFields.config - && !dirtyFields.state - && !dirtyFields.proxyFallbackList - ? 'saved' - : 'preview'; +): boolean => [dirtyFields.config, dirtyFields.state, dirtyFields.proxyFallbackList].some(Boolean); // `hasAuto` says the upstream also lists the model, which is what makes // switching the row back to `auto` possible. @@ -136,7 +128,7 @@ export interface ModelCatalogFetch { /** Null when nothing was listed, which leaves whatever the caller already shows. */ discovered: UpstreamModelConfig[] | null; modelsError: ModelListingFailure | null; - refreshed: UpstreamRecord | null; + modelsCache: UpstreamRecord['modelsCache'] | null; } // The gateway squashes a genuine upstream failure to a message that names @@ -158,49 +150,47 @@ export interface ModelListingFailure { upstreamListingFailed: boolean; } -interface ModelCatalogFetchOptions extends RequestInit { - operation?: 'saved' | 'preview'; -} +const listingFailure = (error: { message: string; raw?: unknown }): ModelCatalogFetch => ({ + discovered: null, + modelsError: { + message: error.message, + upstreamListingFailed: failureCode(error.raw) === MODEL_LISTING_FAILURE_CODE, + }, + modelsCache: null, +}); -export const fetchModelCatalog = async ( +export const previewDraftModelCatalog = async ( record: UpstreamRecord, values: UpstreamEditorValues, - options: ModelCatalogFetchOptions = {}, + init?: RequestInit, ): Promise => { - if (!canFetchModelCatalog(record, values.config)) return { discovered: null, modelsError: null, refreshed: null }; - - const { operation = modelCatalogOperation(record, {}), ...init } = options; - const result = operation === 'saved' - ? await callApi(() => api.api.upstreams[':id']['list-models'].$post({ param: { id: record.id } }, { init })) - : await callApi(() => api.api.upstreams['preview-models'].$post({ - json: { record: previewRecord(record, values) }, - }, { init })); - if (result.error) { - return { - discovered: null, - modelsError: { - message: result.error.message, - upstreamListingFailed: failureCode(result.error.raw) === MODEL_LISTING_FAILURE_CODE, - }, - refreshed: null, - }; - } - - const endpoints = record.kind === 'custom' - ? (values.config as Extract['config']).endpoints - : {}; - const discovered = discoveredModelsFromResponse(result.data, endpoints); - if (operation === 'preview') return { discovered, modelsError: null, refreshed: null }; + if (!canFetchModelCatalog(record, values.config)) return { discovered: null, modelsError: null, modelsCache: null }; + const result = await callApi(() => api.api.upstreams['preview-models'].$post({ + json: { record: previewRecord(record, values) }, + }, { init })); + if (result.error) return listingFailure(result.error); + return { discovered: result.data.data, modelsError: null, modelsCache: null }; +}; - const refreshed = await callApi(() => api.api.upstreams[':id'].$get({ param: { id: record.id } }, { init })); - return refreshed.error - ? { discovered, modelsError: { message: refreshed.error.message, upstreamListingFailed: false }, refreshed: null } - : { discovered, modelsError: null, refreshed: refreshed.data }; +export const fetchSavedModelCatalog = async ( + record: UpstreamRecord, + init?: RequestInit, +): Promise => { + if (!canFetchModelCatalog(record, record.config)) return { discovered: null, modelsError: null, modelsCache: null }; + const result = await callApi(() => api.api.upstreams[':id']['list-models'].$post({ param: { id: record.id } }, { init })); + if (result.error) return listingFailure(result.error); + return { discovered: result.data.data, modelsError: null, modelsCache: result.data.modelsCache }; }; export const loadInitialModelCatalog = async (record: UpstreamRecord) => { - const { discovered, modelsError, refreshed } = await fetchModelCatalog(record, valuesFromRecord(record)); - return { discovered: discovered ?? [], modelsError, record: refreshed ?? record }; + const result = isPersisted(record) + ? await fetchSavedModelCatalog(record) + : await previewDraftModelCatalog(record, valuesFromRecord(record)); + return { + discovered: result.discovered ?? [], + modelsError: result.modelsError, + record: result.modelsCache === null ? record : { ...record, modelsCache: result.modelsCache } as UpstreamRecord, + }; }; // A field react-hook-form has registered owns its key from then on: mounting it @@ -331,25 +321,6 @@ export const updateBody = (record: UpstreamRecord, values: UpstreamEditorValues) } as UpdateUpstreamBody; }; -export const discoveredModelsFromResponse = ( - response: ListUpstreamModelsResponse, - endpoints: ModelEndpoints, -): UpstreamModelConfig[] => { - if (response.kind !== 'custom') return response.data; - return response.data.map(model => { - const kind = model.kind ?? 'chat'; - return { - upstreamModelId: model.id, - publicModelId: model.id, - kind, - ...(kind === 'chat' ? { endpoints: configuredEndpoints(endpoints) } : shapeForKind(kind, { endpoints })), - ...(model.display_name ?? model.name ? { display_name: model.display_name ?? model.name } : {}), - ...(model.limits ? { limits: model.limits } : {}), - ...(model.pricing ? { pricing: model.pricing } : {}), - }; - }); -}; - export const modelPrefixIsValid = (prefix: string) => MODEL_PREFIX_REGEX.test(prefix) && prefix.length <= MODEL_PREFIX_MAX_LENGTH; diff --git a/apps/web/src/components/upstream-editor/page.tsx b/apps/web/src/components/upstream-editor/page.tsx index 0474a1f31..7527b600f 100644 --- a/apps/web/src/components/upstream-editor/page.tsx +++ b/apps/web/src/components/upstream-editor/page.tsx @@ -9,9 +9,11 @@ import { UpstreamConfigSidebar } from './config-sidebar'; import { refineCustomIngressHeaderRules } from './custom-ingress-header-rules-validation'; import { createBody, - fetchModelCatalog, - modelCatalogOperation, + fetchSavedModelCatalog, + hasDraftModelInputs, + isPersisted, modelPrefixIsValid, + previewDraftModelCatalog, updateBody, valuesFromRecord, type ModelListingFailure, @@ -134,17 +136,19 @@ export function UpstreamEditorPage({ data }: { data: UpstreamEditorLoaderData }) return () => window.removeEventListener('beforeunload', handler); }, [hasUnsavedChanges]); + const draftModelInputs = hasDraftModelInputs(formState.dirtyFields); // The workspace's refresh button and the custom provider's fetch switch both // reach this, so runs can overlap; `useRefresh` aborts the superseded one. const { refresh: refreshModels, refreshing: modelsLoading } = useRefresh(useCallback(async (signal: AbortSignal) => { setModelsError(null); - const operation = modelCatalogOperation(record, formState.dirtyFields); - const catalog = await fetchModelCatalog(record, getValues(), { operation, signal }); + const catalog = isPersisted(record) && !draftModelInputs + ? await fetchSavedModelCatalog(record, { signal }) + : await previewDraftModelCatalog(record, getValues(), { signal }); if (signal.aborted) return; setModelsError(catalog.modelsError); if (catalog.discovered) setDiscovered(catalog.discovered); - if (catalog.refreshed) updateRecord({ ...recordRef.current, modelsCache: catalog.refreshed.modelsCache } as UpstreamRecord); - }, [formState.dirtyFields, getValues, record, updateRecord])); + if (catalog.modelsCache) updateRecord({ ...recordRef.current, modelsCache: catalog.modelsCache } as UpstreamRecord); + }, [draftModelInputs, getValues, record, updateRecord])); const applyProviderPatch = (patch: { config?: unknown; state?: unknown }, persisted = false) => { if (patch.config !== undefined) setValue('config', patch.config as UpstreamEditorValues['config'], { shouldDirty: !persisted }); diff --git a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts index ee55fa2e4..3241397ba 100644 --- a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts @@ -1,8 +1,8 @@ import { Hono } from 'hono'; import { expect, test, vi } from 'vitest'; -// The import handler warms the persisted models snapshot for every saved upstream by -// calling each provider's getProvidedModels, which for Copilot / Custom would +// The import handler warms new upstreams and changed catalog inputs by calling +// each provider's getProvidedModels, which for Copilot / Custom would // make real upstream HTTP requests the test sandbox cannot serve and hang // until the vitest timeout. Stub the cache layer to a no-op so the import // path's own behavior (upserts, identity validation, etc.) is what the tests @@ -18,11 +18,11 @@ import { DEFAULT_WEB_SEARCH_CONFIG } from '../../../src/data-plane/tools/web-sea import { initDumpBroker, initDumpStore } from '../../../src/dump/registry.ts'; import { zValidator } from '../../../src/middleware/zod-validator.ts'; import { initRepo } from '../../../src/repo/index.ts'; -import type { ApiKey, PerformanceTelemetryRecord, WebSearchUsageRecord, StoredResponsesItem, UsageRecord, User } from '../../../src/repo/types.ts'; +import type { ApiKey, PerformanceTelemetryRecord, WebSearchUsageRecord, StoredResponsesItem, StoredUpstreamRecord, UsageRecord, User } from '../../../src/repo/types.ts'; import { tokenUsageMetrics } from '../../../src/repo/usage-metrics.ts'; import { installDumpStubs } from '../../dump/test-fixtures.ts'; import { InMemoryRepo } from '../../repo/memory.ts'; -import { ALL_PROVIDER_KINDS, type UpstreamRecord } from '@floway-dev/provider'; +import { ALL_PROVIDER_KINDS } from '@floway-dev/provider'; import { assertEquals } from '@floway-dev/test-utils'; const hasOwn = (value: object, key: string) => Object.prototype.hasOwnProperty.call(value, key); @@ -74,7 +74,7 @@ const USER_BOB: User = { deletedAt: null, }; -const CUSTOM_UPSTREAM: UpstreamRecord = { +const CUSTOM_UPSTREAM: StoredUpstreamRecord = { id: 'up_custom_a', kind: 'custom', name: 'Custom A', @@ -103,7 +103,7 @@ const CUSTOM_UPSTREAM: UpstreamRecord = { state: null, }; -const COPILOT_UPSTREAM: UpstreamRecord = { +const COPILOT_UPSTREAM: StoredUpstreamRecord = { id: 'up_copilot_a', kind: 'copilot', name: 'GitHub Copilot (alice)', @@ -131,7 +131,7 @@ const COPILOT_UPSTREAM: UpstreamRecord = { state: null, }; -const AZURE_UPSTREAM: UpstreamRecord = { +const AZURE_UPSTREAM: StoredUpstreamRecord = { id: 'up_azure_a', kind: 'azure', name: 'Azure A', @@ -166,7 +166,7 @@ const AZURE_UPSTREAM: UpstreamRecord = { state: null, }; -const OLLAMA_UPSTREAM: UpstreamRecord = { +const OLLAMA_UPSTREAM: StoredUpstreamRecord = { id: 'up_ollama_a', kind: 'ollama', name: 'Ollama A', @@ -195,7 +195,7 @@ const OLLAMA_UPSTREAM: UpstreamRecord = { state: null, }; -const CODEX_UPSTREAM: UpstreamRecord = { +const CODEX_UPSTREAM: StoredUpstreamRecord = { id: 'up_codex_a', kind: 'codex', name: 'ChatGPT Codex (alice)', @@ -544,7 +544,7 @@ test('import merge upserts by repository key without clearing unrelated rows', a await repo.usage.set({ ...USAGE_1, requests: 10 }); await repo.webSearchUsage.set({ ...WEB_SEARCH_USAGE_1, requests: 10 }); - const updatedCustom = { ...CUSTOM_UPSTREAM, name: 'Custom Updated', updatedAt: '2026-03-01T00:00:00.000Z' } satisfies UpstreamRecord; + const updatedCustom = { ...CUSTOM_UPSTREAM, name: 'Custom Updated', updatedAt: '2026-03-01T00:00:00.000Z' } satisfies StoredUpstreamRecord; const result = await doImport(app, 'merge', latestImportData({ apiKeys: [{ ...KEY_A, name: 'Alice Updated' }, KEY_B], upstreams: [upstreamRecordToFullJson(updatedCustom), upstreamRecordToFullJson(COPILOT_UPSTREAM)], @@ -1313,7 +1313,7 @@ test('export includes proxies with full credential URIs and round-trips through const { app, repo } = setup(); await repo.proxies.save({ id: 'p_socks', name: 'SOCKS', url: SOCKS_PROXY_URL, dialTimeoutSeconds: 45 }); await repo.proxies.save({ id: 'p_http', name: 'HTTP', url: HTTP_PROXY_URL, dialTimeoutSeconds: null }); - const upstreamWithFallback: UpstreamRecord = { ...CUSTOM_UPSTREAM, proxyFallbackList: [{ id: 'p_socks' }, { id: 'direct_connect' }, { id: 'p_http' }, { id: 'direct_fetch' }] }; + const upstreamWithFallback: StoredUpstreamRecord = { ...CUSTOM_UPSTREAM, proxyFallbackList: [{ id: 'p_socks' }, { id: 'direct_connect' }, { id: 'p_http' }, { id: 'direct_fetch' }] }; await repo.upstreams.save(upstreamWithFallback); const exported = await doExport(app); diff --git a/packages/gateway/__tests__/control-plane/models/routes_test.ts b/packages/gateway/__tests__/control-plane/models/routes_test.ts index 2b71d9242..da31c469a 100644 --- a/packages/gateway/__tests__/control-plane/models/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/models/routes_test.ts @@ -16,7 +16,6 @@ const azureUpstream = (): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts index 7dec9d0cb..e2c7d6d76 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts @@ -15,8 +15,7 @@ vi.mock('../../../src/data-plane/providers/models-refresh.ts', () => ({ clearModelsRefreshesForTesting: () => {}, })); -import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; -import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; +import { seedModelsCache, storedModelsCacheGeneration } from '../../repo/models-cache-fixture.ts'; import { buildCopilotUpstreamRecord, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, assertStringIncludes, jsonResponse, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; @@ -371,7 +370,7 @@ test('/api/upstreams/copilot/oauth/device-login/poll clears the previous identit const existing = buildCopilotUpstreamRecord(githubAccount, { id: 'up_switch_identity' }); await repo.upstreams.deleteAll(); await repo.upstreams.save(existing); - await seedModelsCache(repo.upstreams, existing.id, modelsCacheGeneration(existing), { + await seedModelsCache(repo.upstreams, existing.id, await storedModelsCacheGeneration(repo.upstreams, existing.id), { revision: 1, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'old-tenant-model' })], diff --git a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts index 0ddbdebc9..cc1bcf1b4 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts @@ -4,7 +4,8 @@ import { blueprintUpstreamRecord, upstreamRecordToFullJson } from '../../../src/ import { MODEL_LISTING_FAILURE_CODE } from '../../../src/data-plane/models/shared.ts'; import { MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; -import { seedModelsCache, seedModelsCacheError } from '../../repo/models-cache-fixture.ts'; +import type { StoredUpstreamRecord } from '../../../src/repo/types.ts'; +import { seedModelsCache, seedModelsCacheError, storedModelsCacheGeneration } from '../../repo/models-cache-fixture.ts'; import { MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import type { UpstreamProviderKind, UpstreamRecord } from '@floway-dev/provider'; import { assertEquals, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -343,7 +344,6 @@ test('PATCH /api/upstreams keeps Azure as a single endpoint config', async () => disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { @@ -391,7 +391,6 @@ test('PATCH /api/upstreams round-trips a flat per-model flagOverrides map', asyn disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { @@ -437,7 +436,6 @@ test('GET /api/upstreams attaches models-cache freshness to every row', async () disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { baseUrl: 'https://a.example.com', authStyle: 'bearer', apiKey: 'x', endpoints: { chatCompletions: {} }, ingressHeadersRules: [] }, @@ -450,17 +448,18 @@ test('GET /api/upstreams attaches models-cache freshness to every row', async () await repo.upstreams.save(warmRecord); await repo.upstreams.save(failedRecord); - await seedModelsCache(repo.upstreams, 'up_warm', modelsCacheGeneration(warmRecord), { + await seedModelsCache(repo.upstreams, 'up_warm', await storedModelsCacheGeneration(repo.upstreams, 'up_warm'), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [{ id: 'm1', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], }); - await seedModelsCache(repo.upstreams, 'up_failed', modelsCacheGeneration(failedRecord), { + const failedGeneration = await storedModelsCacheGeneration(repo.upstreams, 'up_failed'); + await seedModelsCache(repo.upstreams, 'up_failed', failedGeneration, { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [{ id: 'm1', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], }); - await seedModelsCacheError(repo.upstreams, 'up_failed', modelsCacheGeneration(failedRecord), { message: 'boom', at: 1_700_000_500_000 }); + await seedModelsCacheError(repo.upstreams, 'up_failed', failedGeneration, { message: 'boom', at: 1_700_000_500_000 }); const list = await requestApp('/api/upstreams', { headers: { 'x-floway-session': adminSession } }); assertEquals(list.status, 200); @@ -490,7 +489,6 @@ test('GET /api/upstream-options returns the minimal picker shape to admin and no disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { baseUrl: 'https://custom.example.com', authStyle: 'bearer', apiKey: 'sk-secret', endpoints: { chatCompletions: {} } }, @@ -544,7 +542,7 @@ test('POST /api/upstreams/preview-models fetches a draft custom upstream model l })); assertEquals(resp.status, 200); const body = (await resp.json()) as { data: Array> }; - assertEquals(body.data.map(m => m.id), ['gpt-a', 'gpt-b']); + assertEquals(body.data.map(m => m.upstreamModelId), ['gpt-a', 'gpt-b']); assertEquals(body.data[1].display_name, 'GPT B'); }, ); @@ -674,7 +672,6 @@ test('POST /api/upstreams/:id/list-models reads the saved config and publishes a disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { ...customConfig, apiKey: 'sk-refresh' }, @@ -698,11 +695,10 @@ test('POST /api/upstreams/:id/list-models reads the saved config and publishes a headers: { 'x-floway-session': adminSession }, }); assertEquals(resp.status, 200); - const body = (await resp.json()) as { data: Array<{ id?: string }> }; - // Custom returns the raw upstream row shape (id-keyed), not the - // dashboard-projected UpstreamModelConfig — the SPA translates - // through the draft's endpoints. - assertEquals(body.data.map(m => m.id), ['fresh-model']); + const body = (await resp.json()) as { data: Array<{ upstreamModelId?: string }>; modelsCache: { fetchedAt: number | null; modelCount: number | null } }; + assertEquals(body.data.map(m => m.upstreamModelId), ['fresh-model']); + assertEquals(body.modelsCache.modelCount, 1); + assertEquals(typeof body.modelsCache.fetchedAt, 'number'); assertEquals(upstreamCalls, 1); const cached = (await repo.upstreams.getById(savedRecord.id))?.modelsCache; assertEquals(cached?.models.map((model: { id: string }) => model.id), ['fresh-model']); @@ -757,62 +753,14 @@ test('POST /api/upstreams warms the models cache before responding', async () => assertEquals(created.modelsCache.lastError, null); }); -test('PATCH /api/upstreams warms the models cache before responding', async () => { - const { repo, adminSession } = await setupAppTest(); - await repo.upstreams.deleteAll(); - - const create = await requestApp('/api/upstreams', authed(adminSession, createBody())); - const created = (await create.json()) as { id: string }; - // Overwrite whatever the create-time warm landed on the row with a marker - // catalog, so the assertion below can only pass if the PATCH-time warm wrote - // over it. - await seedModelsCache(repo.upstreams, created.id, await getCacheGeneration(repo, created.id), { - revision: MODEL_CATALOG_REVISION, - fetchedAt: 1, - models: [{ id: 'warmed-on-create', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], - }); - // …and annotate it with an error the successful PATCH-time warm must clear, - // so the response body cannot pass by echoing the pre-warm row. - await seedModelsCacheError(repo.upstreams, created.id, await getCacheGeneration(repo, created.id), { message: 'stale failure', at: 1 }); - - const patched = await withMockedFetch( - async request => { - const url = new URL(request.url); - if (url.hostname === 'custom.example.com' && url.pathname === '/v1/models') { - return jsonResponse({ object: 'list', data: [{ id: 'warmed-on-update' }] }); - } - throw new Error(`Unhandled fetch ${request.url}`); - }, - async () => { - const patch = await requestApp(`/api/upstreams/${created.id}`, { - method: 'PATCH', - headers: { 'content-type': 'application/json', 'x-floway-session': adminSession }, - body: JSON.stringify({ name: 'Renamed' }), - }); - assertEquals(patch.status, 200); - return (await patch.json()) as { modelsCache: { fetchedAt: number | null; lastError: unknown } }; - }, - ); - - const cached = (await repo.upstreams.getById(created.id))?.modelsCache; - assertEquals(cached?.models.map(model => model.id), ['warmed-on-update']); - assertEquals(patched.modelsCache.fetchedAt, cached?.fetchedAt ?? null); - assertEquals(patched.modelsCache.lastError, null); -}); - -test('PATCH /api/upstreams metadata warm preserves refresh backoff', async () => { +test('PATCH /api/upstreams metadata edit preserves the catalog without model I/O', async () => { const { repo, adminSession } = await setupAppTest(); await repo.upstreams.deleteAll(); const created = await withMockedFetch( () => jsonResponse({ object: 'list', data: [{ id: 'cached-model' }] }), async () => await (await requestApp('/api/upstreams', authed(adminSession, createBody()))).json() as { id: string }, ); - const generation = await getCacheGeneration(repo, created.id); const configVersion = (await repo.upstreams.getById(created.id))?.configVersion; - const now = Date.now(); - const claim = await repo.upstreams.claimModelsRefresh({ id: created.id, generation, token: 'failed-refresh', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); - if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); - await repo.upstreams.finalizeModelsRefreshFailure({ id: created.id, generation, token: 'failed-refresh', error: { message: 'failed refresh', at: now }, previousFailureCount: 0, failedAt: now }); let modelRequests = 0; await withMockedFetch( @@ -852,7 +800,7 @@ test('POST /api/upstreams/preview-models without an id still serves draft previe })); assertEquals(resp.status, 200); const body = (await resp.json()) as { data: Array> }; - assertEquals(body.data.map(m => m.id), ['draft-only']); + assertEquals(body.data.map(m => m.upstreamModelId), ['draft-only']); }, ); }); @@ -915,13 +863,13 @@ const createCodexUpstreamViaExchange = async (adminSession: string, overrides: R return (await create.json()) as { id: string }; }; -const getRecord = async (repo: { upstreams: { getById: (id: string) => Promise } }, id: string): Promise => { +const getRecord = async (repo: { upstreams: { getById: (id: string) => Promise } }, id: string): Promise => { const record = await repo.upstreams.getById(id); if (!record) throw new Error(`Expected upstream ${id} to exist`); return record; }; -const getCacheGeneration = async (repo: { upstreams: { getById: (id: string) => Promise } }, id: string) => { +const getCacheGeneration = async (repo: { upstreams: { getById: (id: string) => Promise } }, id: string) => { const record = await getRecord(repo, id); return modelsCacheGeneration(record); }; @@ -2212,7 +2160,6 @@ test('POST /api/upstreams/preview-models never writes the matching saved row', a disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/control-plane/upstreams/serialize_test.ts b/packages/gateway/__tests__/control-plane/upstreams/serialize_test.ts index 6cfc91e67..6821c51e9 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/serialize_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/serialize_test.ts @@ -27,7 +27,6 @@ const custom: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { @@ -201,7 +200,6 @@ const claudeCodeBase = (overrides: { config?: unknown; state?: unknown }): Upstr disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: overrides.config ?? claudeCodeConfig, @@ -220,7 +218,6 @@ const codexBase = (overrides: { config?: unknown; state?: unknown }): UpstreamRe disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: overrides.config ?? { accounts: [{ email: 'a@example.com', chatgptAccountId: 'account', chatgptUserId: 'user', planType: 'plus' }] }, diff --git a/packages/gateway/__tests__/data-plane/audio/http_test.ts b/packages/gateway/__tests__/data-plane/audio/http_test.ts index 037f80248..c51d0a287 100644 --- a/packages/gateway/__tests__/data-plane/audio/http_test.ts +++ b/packages/gateway/__tests__/data-plane/audio/http_test.ts @@ -25,7 +25,6 @@ const registerAudioModel = async ( disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/data-plane/chat/responses/affinity/copilot-roundtrip_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/affinity/copilot-roundtrip_test.ts index a2f280971..7d039731f 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/affinity/copilot-roundtrip_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/affinity/copilot-roundtrip_test.ts @@ -30,7 +30,6 @@ const upstream: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts index 7fc071000..25bf15efd 100644 --- a/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts @@ -208,7 +208,6 @@ beforeEach(async () => { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts b/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts index 2b1380ec8..f69c4a467 100644 --- a/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts +++ b/packages/gateway/__tests__/data-plane/chat/shared/target-picker_test.ts @@ -30,7 +30,6 @@ const azureUpstream = (id: string, sortOrder: number, modelIds: string[], endpoi disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts b/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts index b778bcd27..2795372fe 100644 --- a/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts +++ b/packages/gateway/__tests__/data-plane/codex/routes_images_test.ts @@ -19,7 +19,6 @@ const saveAzureImages = async (repo: InMemoryRepo): Promise => { disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/data-plane/images/http_test.ts b/packages/gateway/__tests__/data-plane/images/http_test.ts index 18fb48992..67f10565d 100644 --- a/packages/gateway/__tests__/data-plane/images/http_test.ts +++ b/packages/gateway/__tests__/data-plane/images/http_test.ts @@ -198,7 +198,6 @@ test('/v1/images/edits forwards a multipart request through an Azure model and r disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts index 1a83f6242..b6dbaade0 100644 --- a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts @@ -209,7 +209,6 @@ test('disabledPublicModelIds hides models from the catalog and routing, per upst disabledPublicModelIds: over.disabledPublicModelIds, proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 63c75f211..f235f0a44 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -50,7 +50,6 @@ const setupRepo = async (): Promise => { updatedAt: '2026-08-01T00:00:00.000Z', config: CACHE_CONFIG, state: null, - configVersion: 1, modelsCache: null, flagOverrides: {}, disabledPublicModelIds: [], @@ -231,7 +230,7 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { const fetchFn = vi.fn(async () => [aModel('recovered')]); const cache = await storedCache(repo); const warming = stubInstance(fetchFn, cache); - await expect(warmUpstreamModels(warming, directFetcher)).resolves.toEqual([]); + await expect(warmUpstreamModels(warming, directFetcher)).resolves.toBeUndefined(); expect(fetchFn).not.toHaveBeenCalled(); await expect(fetchUpstreamModels(warming, directFetcher)) @@ -259,7 +258,8 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [aModel('remote-model')] }, }); - expect((await warming).map(model => model.id)).toEqual(['remote-model']); + await warming; + expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['remote-model']); expect(localFetch).not.toHaveBeenCalled(); }); @@ -282,7 +282,8 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [aModel('remote-model')] }, }); expect((await explicit).map(model => model.id)).toEqual(['remote-model']); - expect((await warming).map(model => model.id)).toEqual(['remote-model']); + await warming; + expect(instance.modelsCache?.models.map(model => model.id)).toEqual(['remote-model']); }); test('explicit fetch retries after a remote owner records failure', async () => { @@ -316,7 +317,8 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { const explicit = fetchUpstreamModels(instance, directFetcher); resolveWarm!([aModel('warm-owner-model')]); expect((await explicit).map(model => model.id)).toEqual(['warm-owner-model']); - expect((await warming).map(model => model.id)).toEqual(['warm-owner-model']); + await warming; + expect(instance.modelsCache?.models.map(model => model.id)).toEqual(['warm-owner-model']); expect(fetchFn).toHaveBeenCalledTimes(1); }); @@ -453,7 +455,6 @@ describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { updatedAt: '2026-08-01T00:00:00.000Z', config: {}, state: null, - configVersion: 1, modelsCache: null, flagOverrides: {}, disabledPublicModelIds: [], diff --git a/packages/gateway/__tests__/data-plane/providers/registry_test.ts b/packages/gateway/__tests__/data-plane/providers/registry_test.ts index 43ac061ce..cc86ac589 100644 --- a/packages/gateway/__tests__/data-plane/providers/registry_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/registry_test.ts @@ -2,8 +2,7 @@ import { test } from 'vitest'; import { MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; -import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; -import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; +import { seedModelsCache, storedModelsCacheGeneration } from '../../repo/models-cache-fixture.ts'; import { buildCopilotUpstreamRecord, buildCustomUpstreamRecord, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; @@ -33,7 +32,6 @@ test('listModelProviders creates enabled provider instances with upstream row id disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, state: null, @@ -98,7 +96,7 @@ test('listModelProviders carries each row cached catalog onto its instance', asy const cachedRecord = buildCustomUpstreamRecord({ id: 'up_cached', name: 'Cached', sortOrder: 10 }); await repo.upstreams.save(cachedRecord); await repo.upstreams.save(buildCustomUpstreamRecord({ id: 'up_cold', name: 'Cold', sortOrder: 20 })); - await seedModelsCache(repo.upstreams, 'up_cached', modelsCacheGeneration(cachedRecord), { + await seedModelsCache(repo.upstreams, 'up_cached', await storedModelsCacheGeneration(repo.upstreams, 'up_cached'), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index 935abee71..59843307b 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -307,7 +307,6 @@ test('enumerateRealModelCandidates rejects a model id disabled on that upstream disabledPublicModelIds: ['disabled-model'], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, state: null, diff --git a/packages/gateway/__tests__/dial/per-request_test.ts b/packages/gateway/__tests__/dial/per-request_test.ts index 151c3fbb5..2226934f9 100644 --- a/packages/gateway/__tests__/dial/per-request_test.ts +++ b/packages/gateway/__tests__/dial/per-request_test.ts @@ -30,7 +30,6 @@ const upstream = (id: string, proxyFallbackList: ProxyFallbackEntry[]) => ({ disabledPublicModelIds: [], proxyFallbackList, modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: COPILOT_CONFIG, diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index 2c6c5567d..422a7ed3e 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -58,6 +58,7 @@ import type { SessionsRepo, StoredResponsesItem, StoredResponsesSnapshot, + StoredUpstreamRecord, UpstreamRepo, UsageRecord, UsageOverviewAxis, @@ -736,14 +737,14 @@ class MemoryWebSearchConfigRepo implements WebSearchConfigRepo { } class MemoryUpstreamRepo implements UpstreamRepo { - private store = new Map(); + private store = new Map(); private modelsRefreshes = new Map(); - list(): Promise { + list(): Promise { return Promise.resolve([...this.store.values()].map(cloneUpstreamRecord).sort((a, b) => a.sortOrder - b.sortOrder || a.createdAt.localeCompare(b.createdAt))); } - getById(id: string): Promise { + getById(id: string): Promise { const found = this.store.get(id); return Promise.resolve(found ? cloneUpstreamRecord(found) : null); } @@ -752,64 +753,60 @@ class MemoryUpstreamRepo implements UpstreamRepo { // the snapshot; other writes preserve it. New rows always start uncached. save(upstream: UpstreamRecord): Promise { const existing = this.store.get(upstream.id); - if (existing === undefined && upstream.configVersion !== 1) { - throw new Error(`New upstream ${upstream.id} must start at config version 1`); - } - const configChanged = existing !== undefined + const modelConfigChanged = existing !== undefined && (existing.kind !== upstream.kind - || serializeStoredConfig(existing.config) !== serializeStoredConfig(upstream.config)); + || serializeStoredConfig(existing.config) !== serializeStoredConfig(upstream.config) + || serializeStoredConfig(existing.flagOverrides) !== serializeStoredConfig(upstream.flagOverrides)); + const transportChanged = existing !== undefined + && serializeStoredConfig(existing.proxyFallbackList) !== serializeStoredConfig(upstream.proxyFallbackList); + const refreshInputsChanged = modelConfigChanged || transportChanged; const preserved = existing ? { ...upstream, createdAt: existing.createdAt, - configVersion: existing.configVersion + (configChanged ? 1 : 0), - modelsCache: configChanged ? null : existing.modelsCache, + configVersion: existing.configVersion + (refreshInputsChanged ? 1 : 0), + modelsCache: modelConfigChanged ? null : existing.modelsCache, } - : { ...upstream, modelsCache: null }; + : { ...upstream, configVersion: 1, modelsCache: null }; this.store.set(preserved.id, cloneUpstreamRecord(preserved)); - const refresh = this.modelsRefreshes.get(preserved.id); - if (configChanged) this.modelsRefreshes.delete(preserved.id); - else if (refresh) this.modelsRefreshes.set(preserved.id, { ...refresh, claimToken: null, claimedAt: null }); + if (refreshInputsChanged) this.modelsRefreshes.delete(preserved.id); return Promise.resolve(); } - insertForModels(upstream: UpstreamRecord): Promise { - if (upstream.configVersion !== 1) throw new Error(`New upstream ${upstream.id} must start at config version 1`); - if (this.store.has(upstream.id)) return Promise.resolve(false); - this.store.set(upstream.id, cloneUpstreamRecord({ ...upstream, modelsCache: null })); - return Promise.resolve(true); + insertForModels(upstream: UpstreamRecord): Promise { + if (this.store.has(upstream.id)) return Promise.resolve(null); + const stored = cloneUpstreamRecord({ ...upstream, configVersion: 1, modelsCache: null }); + this.store.set(upstream.id, stored); + return Promise.resolve(cloneUpstreamRecord(stored)); } replaceForModels(input: { - previous: UpstreamRecord; - upstream: Omit; - }): Promise { + previous: StoredUpstreamRecord; + upstream: UpstreamRecord; + }): Promise { const { previous, upstream } = input; - const configChanged = previous.kind !== upstream.kind - || serializeStoredConfig(previous.config) !== serializeStoredConfig(upstream.config); - const configVersion = previous.configVersion + (configChanged ? 1 : 0); + const modelConfigChanged = previous.kind !== upstream.kind + || serializeStoredConfig(previous.config) !== serializeStoredConfig(upstream.config) + || serializeStoredConfig(previous.flagOverrides) !== serializeStoredConfig(upstream.flagOverrides); const transportChanged = serializeStoredConfig(previous.proxyFallbackList) !== serializeStoredConfig(upstream.proxyFallbackList); + const refreshInputsChanged = modelConfigChanged || transportChanged; + const configVersion = previous.configVersion + (refreshInputsChanged ? 1 : 0); const existing = this.store.get(upstream.id); - if (existing === undefined) return Promise.resolve(false); + if (existing === undefined) return Promise.resolve(null); const replaceState = serializeStoredState(previous.state) !== serializeStoredState(upstream.state); const comparableExisting = { ...existing, modelsCache: null, state: replaceState ? existing.state : null }; const comparablePrevious = { ...previous, modelsCache: null, state: replaceState ? previous.state : null }; - if (serializeStoredConfig(comparableExisting) !== serializeStoredConfig(comparablePrevious)) return Promise.resolve(false); + if (serializeStoredConfig(comparableExisting) !== serializeStoredConfig(comparablePrevious)) return Promise.resolve(null); const next = cloneUpstreamRecord({ ...upstream, createdAt: existing.createdAt, configVersion, state: replaceState ? upstream.state : existing.state, - modelsCache: configChanged ? null : existing.modelsCache, + modelsCache: modelConfigChanged ? null : existing.modelsCache, }); this.store.set(upstream.id, next); - if (!configChanged && !transportChanged) { - const refresh = this.modelsRefreshes.get(upstream.id); - if (refresh !== undefined) this.modelsRefreshes.set(upstream.id, { ...refresh, claimToken: null, claimedAt: null }); - } else { - this.modelsRefreshes.delete(upstream.id); - } - return Promise.resolve(true); + if (refreshInputsChanged) this.modelsRefreshes.delete(upstream.id); + return Promise.resolve(cloneUpstreamRecord(next)); } delete(id: string): Promise { @@ -879,7 +876,6 @@ class MemoryUpstreamRepo implements UpstreamRepo { if (observedActiveToken !== null && existing === undefined) return Promise.resolve({ kind: 'completed' }); if (existing !== undefined) { if (existing.claimToken !== null && existing.claimedAt! > staleClaimedBefore) return Promise.resolve({ kind: 'active', token: existing.claimToken }); - if (observedActiveToken !== null && existing.claimToken === null) return Promise.resolve({ kind: 'completed' }); if (!bypassBackoff && existing.retryAt > now) return Promise.resolve({ kind: 'backoff' }); } this.modelsRefreshes.set(id, { @@ -893,7 +889,7 @@ class MemoryUpstreamRepo implements UpstreamRepo { } -const cloneUpstreamRecord = (upstream: UpstreamRecord): UpstreamRecord => ({ +const cloneUpstreamRecord = (upstream: StoredUpstreamRecord): StoredUpstreamRecord => ({ ...upstream, config: structuredClone(upstream.config), state: upstream.state === null || upstream.state === undefined ? null : structuredClone(upstream.state), diff --git a/packages/gateway/__tests__/repo/models-cache-fixture.ts b/packages/gateway/__tests__/repo/models-cache-fixture.ts index 771bd865b..53e067ba4 100644 --- a/packages/gateway/__tests__/repo/models-cache-fixture.ts +++ b/packages/gateway/__tests__/repo/models-cache-fixture.ts @@ -1,6 +1,16 @@ +import { modelsCacheGeneration } from '../../src/repo/models-cache-contract.ts'; import type { ModelsCacheGeneration, UpstreamRepo } from '../../src/repo/types.ts'; import type { UpstreamModelsCache } from '@floway-dev/provider'; +export const storedModelsCacheGeneration = async ( + repo: UpstreamRepo, + id: string, +): Promise => { + const record = await repo.getById(id); + if (record === null) throw new Error(`Upstream ${id} not found`); + return modelsCacheGeneration(record); +}; + export const seedModelsCache = async ( repo: UpstreamRepo, id: string, diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index 072b80608..79d4517f3 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -5,10 +5,9 @@ import { createSqliteTestDb } from './test-sqlite.ts'; import { MODEL_CATALOG_REVISION, modelsCacheGeneration } from '../../src/repo/models-cache-contract.ts'; import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; -import type { ModelsCacheGeneration, Repo } from '../../src/repo/types.ts'; -import type { UpstreamRecord } from '@floway-dev/provider'; +import type { ModelsCacheGeneration, Repo, StoredUpstreamRecord } from '../../src/repo/types.ts'; -const record: UpstreamRecord = { +const record: StoredUpstreamRecord = { id: 'up_refresh', kind: 'custom', name: 'Refresh', @@ -35,7 +34,7 @@ const factories: [string, () => Promise][] = [ ]; describe.each(factories)('%s models refresh coordination', (_name, createRepo) => { - test('config writes advance the generation while state writes do not', async () => { + test('catalog refresh inputs advance the generation while state writes do not', async () => { const repo = await createRepo(); await repo.upstreams.save(record); await repo.upstreams.saveState(record.id, () => ({ accessToken: 'rotated' })); @@ -47,6 +46,13 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = const changed = await repo.upstreams.getById(record.id); expect(changed?.configVersion).toBe(2); expect(changed?.state).toEqual({ accessToken: 'rotated' }); + if (changed === null) throw new Error('upstream row missing after config update'); + await repo.upstreams.save({ ...changed, flagOverrides: { 'vendor-deepseek': true } }); + const flagged = await repo.upstreams.getById(record.id); + expect(flagged?.configVersion).toBe(3); + if (flagged === null) throw new Error('upstream row missing after flag update'); + await repo.upstreams.save({ ...flagged, proxyFallbackList: [{ id: 'direct_fetch' }] }); + expect((await repo.upstreams.getById(record.id))?.configVersion).toBe(4); }); test('claims atomically, applies one backoff schedule, and lets force bypass cooldown', async () => { @@ -79,6 +85,15 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'after-success', now: now + 2, staleClaimedBefore: now - 899_998, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); }); + test('a waiter acquires an abandoned claim rather than treating it as completed', async () => { + const repo = await createRepo(); + await repo.upstreams.save(record); + const now = 1_800_000_000_000; + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'owner', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.abandonModelsRefresh({ id: record.id, generation, token: 'owner' })).resolves.toBe(true); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'waiter', now: now + 1, staleClaimedBefore: now - 899_999, bypassBackoff: true, observedActiveToken: 'owner' })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + }); + test('recovers abandoned claims and fences tokens and config versions', async () => { const repo = await createRepo(); await repo.upstreams.save(record); @@ -98,28 +113,33 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = const renamed = { ...storedNext, name: 'Renamed', updatedAt: '2026-08-01T00:01:00.000Z' }; await repo.upstreams.replaceForModels({ previous: storedNext, upstream: renamed }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(storedNext), token: 'after-rename', now: now + 900_004, staleClaimedBefore: now + 4, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(storedNext), token: 'after-rename', now: now + 900_004, staleClaimedBefore: now + 4, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'current' }); }); - test('metadata saves preserve backoff while invalidating an active owner', async () => { + test('metadata saves preserve an active refresh owner', async () => { const repo = await createRepo(); await repo.upstreams.save(record); const now = 1_800_000_000_000; - const claim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'failed', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); + const claim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'active', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); - await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: 'failed', error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); const next = { ...record, name: 'Renamed', updatedAt: '2026-08-01T00:01:00.000Z' }; - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next })).resolves.toBe(true); + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next })).resolves.not.toBeNull(); await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(next), - token: 'next-generation', + token: 'racer', now: now + 1, staleClaimedBefore: now - 899_999, bypassBackoff: false, observedActiveToken: null, - })).resolves.toEqual({ kind: 'backoff' }); + })).resolves.toEqual({ kind: 'active', token: 'active' }); + await expect(repo.upstreams.finalizeModelsRefreshSuccess({ + id: record.id, + generation, + token: 'active', + cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [] }, + })).resolves.toBe(true); }); test('state changes preserve the snapshot generation and refresh cooldown', async () => { @@ -131,7 +151,7 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: 'failed', error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); const next = { ...record, state: { credential: 'rotated' }, updatedAt: '2026-08-01T00:01:00.000Z' }; - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next })).resolves.toBe(true); + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next })).resolves.not.toBeNull(); expect((await repo.upstreams.getById(record.id))?.modelsCache?.lastError?.message).toBe('failure'); await expect(repo.upstreams.claimModelsRefresh({ id: record.id, @@ -183,16 +203,16 @@ describe.each(factories)('%s models refresh coordination', (_name, createRepo) = const winner = { ...record, name: 'Winner', updatedAt: '2026-08-01T00:01:00.000Z' }; const stale = { ...record, name: 'Stale', updatedAt: '2026-08-01T00:02:00.000Z' }; - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: winner })).resolves.toBe(true); - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: stale })).resolves.toBe(false); + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: winner })).resolves.not.toBeNull(); + await expect(repo.upstreams.replaceForModels({ previous: record, upstream: stale })).resolves.toBeNull(); expect((await repo.upstreams.getById(record.id))?.name).toBe('Winner'); expect((await repo.upstreams.getById(record.id))?.state).toEqual({ providerManaged: 'newer' }); }); test('catalog-aware insertion never overwrites a concurrent winner', async () => { const repo = await createRepo(); - await expect(repo.upstreams.insertForModels(record)).resolves.toBe(true); - await expect(repo.upstreams.insertForModels({ ...record, name: 'Loser' })).resolves.toBe(false); + await expect(repo.upstreams.insertForModels(record)).resolves.not.toBeNull(); + await expect(repo.upstreams.insertForModels({ ...record, name: 'Loser' })).resolves.toBeNull(); expect((await repo.upstreams.getById(record.id))?.name).toBe(record.name); }); diff --git a/packages/gateway/__tests__/repo/proxies_test.ts b/packages/gateway/__tests__/repo/proxies_test.ts index 11883a013..37a5f5da9 100644 --- a/packages/gateway/__tests__/repo/proxies_test.ts +++ b/packages/gateway/__tests__/repo/proxies_test.ts @@ -31,7 +31,6 @@ const upstreamFixture = (id: string, proxyFallbackList: ProxyFallbackEntry[]): U disabledPublicModelIds: [], proxyFallbackList, modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/gateway/__tests__/repo/sql_test.ts b/packages/gateway/__tests__/repo/sql_test.ts index c7423e8f7..6905c2f3c 100644 --- a/packages/gateway/__tests__/repo/sql_test.ts +++ b/packages/gateway/__tests__/repo/sql_test.ts @@ -5,13 +5,13 @@ import { createSqliteTestDb } from './test-sqlite.ts'; import { MODEL_CATALOG_REVISION } from '../../src/data-plane/providers/models-cache.ts'; import { modelsCacheGeneration } from '../../src/repo/models-cache-contract.ts'; import { SqlRepo, UPSTREAM_STATE_WRITE_ATTEMPTS } from '../../src/repo/sql.ts'; +import type { StoredUpstreamRecord } from '../../src/repo/types.ts'; import type { SqlDatabase, SqlPreparedStatement } from '@floway-dev/platform'; -import type { UpstreamRecord } from '@floway-dev/provider'; import { assertEquals, assertRejects, stubProviderModel } from '@floway-dev/test-utils'; const goodAccount = { chatgptAccountId: 'aid', refresh_token: 'rt_v1', state: 'active' as const, state_updated_at: '2026-01-01T00:00:00Z' }; const GENERATION = '2026-06-05T00:00:00.000Z'; -const baseRecord = (overrides: Partial = {}): UpstreamRecord => ({ +const baseRecord = (overrides: Partial = {}): StoredUpstreamRecord => ({ id: 'up_test', kind: 'codex', name: 'Codex Test', diff --git a/packages/gateway/__tests__/repo/upstreams_test.ts b/packages/gateway/__tests__/repo/upstreams_test.ts index 7df823beb..539091d85 100644 --- a/packages/gateway/__tests__/repo/upstreams_test.ts +++ b/packages/gateway/__tests__/repo/upstreams_test.ts @@ -19,7 +19,6 @@ const upstream = (overrides: Partial & Pick { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: { prefix: 'or/', addressable: ['unprefixed', 'prefixed'], listed: ['prefixed'] }, - configVersion: 1, modelsCache: null, hue: 210, }; @@ -1041,7 +1039,7 @@ class FakeUpstreamsSqlDatabase implements SqlDatabase { } upsert(binds: unknown[]): void { - const [id, provider, name, enabled, sortOrder, createdAt, updatedAt, configVersion, configJson, stateJson, flagOverrides, disabledPublicModelIds, proxyFallbackListJson, modelPrefixJson, hue] = binds as [string, string, string, number, number, string, string, number, string, string | null, string, string, string, string | null, number]; + const [id, provider, name, enabled, sortOrder, createdAt, updatedAt, configJson, stateJson, flagOverrides, disabledPublicModelIds, proxyFallbackListJson, modelPrefixJson, hue] = binds as [string, string, string, number, number, string, string, string, string | null, string, string, string, string | null, number]; const existingIndex = this.rows.findIndex(candidate => candidate.id === id); const existing = existingIndex >= 0 ? this.rows[existingIndex] : undefined; const preservedCreatedAt = existing ? existing.created_at : createdAt; @@ -1053,10 +1051,14 @@ class FakeUpstreamsSqlDatabase implements SqlDatabase { sort_order: sortOrder, created_at: preservedCreatedAt, updated_at: updatedAt, - config_version: configVersion, + config_version: existing === undefined || (existing.provider === provider && existing.config_json === configJson && existing.flag_overrides === flagOverrides && existing.proxy_fallback_list_json === proxyFallbackListJson) + ? existing?.config_version ?? 1 + : existing.config_version + 1, config_json: configJson, state_json: stateJson, - models_cache_json: existing?.config_version === configVersion ? existing.models_cache_json : null, + models_cache_json: existing === undefined || (existing.provider === provider && existing.config_json === configJson && existing.flag_overrides === flagOverrides) + ? existing?.models_cache_json ?? null + : null, flag_overrides: flagOverrides, disabled_public_model_ids: disabledPublicModelIds, proxy_fallback_list_json: proxyFallbackListJson, diff --git a/packages/gateway/__tests__/scheduled/models-refresh_test.ts b/packages/gateway/__tests__/scheduled/models-refresh_test.ts index 55db95549..828437910 100644 --- a/packages/gateway/__tests__/scheduled/models-refresh_test.ts +++ b/packages/gateway/__tests__/scheduled/models-refresh_test.ts @@ -23,7 +23,6 @@ const custom = (id: string, enabled: boolean): UpstreamRecord => ({ ingressHeadersRules: [], }, state: null, - configVersion: 1, modelsCache: null, flagOverrides: {}, disabledPublicModelIds: [], diff --git a/packages/gateway/__tests__/test-utils/app.ts b/packages/gateway/__tests__/test-utils/app.ts index 5846b7167..fc4387c34 100644 --- a/packages/gateway/__tests__/test-utils/app.ts +++ b/packages/gateway/__tests__/test-utils/app.ts @@ -80,7 +80,6 @@ export const buildCopilotUpstreamRecord = (githubAccount: CopilotAccountFixture, disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, ...rest, @@ -111,7 +110,6 @@ export const buildCustomUpstreamRecord = (overrides: Partial = { disabledPublicModelIds: [], proxyFallbackList: MOCKED_FETCH_EGRESS, modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, ...rest, diff --git a/packages/gateway/src/control-plane/data-transfer/import-schema.ts b/packages/gateway/src/control-plane/data-transfer/import-schema.ts index 8a13d2980..0ebf9f7d5 100644 --- a/packages/gateway/src/control-plane/data-transfer/import-schema.ts +++ b/packages/gateway/src/control-plane/data-transfer/import-schema.ts @@ -159,9 +159,6 @@ const upstreamWireSchema = parsedBy((value): UpstreamRecord => { sortOrder, createdAt: parseValue(nonEmptyStringSchema('created_at'), wire.created_at), updatedAt: parseValue(nonEmptyStringSchema('updated_at'), wire.updated_at), - // Import establishes a local generation; merge mode will advance it when - // the imported provider config differs from the row already stored. - configVersion: 1, flagOverrides: parseValue(parsedBy(parseFlagOverridesWire), wire.flag_overrides), disabledPublicModelIds: parseValue(parsedBy(parseDisabledPublicModelIdsWire).optional().default([]), wire.disabled_public_model_ids), proxyFallbackList: parseValue(proxyFallbackListSchema, wire.proxy_fallback_list), diff --git a/packages/gateway/src/control-plane/data-transfer/routes.ts b/packages/gateway/src/control-plane/data-transfer/routes.ts index 8f5e4c084..6aaf57d7a 100644 --- a/packages/gateway/src/control-plane/data-transfer/routes.ts +++ b/packages/gateway/src/control-plane/data-transfer/routes.ts @@ -17,7 +17,7 @@ import { getRepo } from '../../repo/index.ts'; import { DIRECT_FALLBACK_IDS } from '../../repo/proxy-fallback-list.ts'; import type { ApiKey, PerformanceTelemetryRecord, UsageRecord, User, WebSearchUsageRecord } from '../../repo/types.ts'; import { type exportQuery, type importBody } from '../schemas.ts'; -import { saveAndWarmUpstreamsForModels } from '../shared/save-upstream-for-models.ts'; +import { saveUpstreamsAndWarmChangedModels } from '../shared/save-upstream-for-models.ts'; import { type FullSerializedUpstreamRecord, upstreamRecordToFullJson } from '../upstreams/serialize.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; @@ -189,7 +189,7 @@ export const importData = async (c: CtxWithJson) => { } for (const record of usage) await repo.usage.set(record); for (const record of searchUsage) await repo.webSearchUsage.set(record); - await saveAndWarmUpstreamsForModels(await Promise.all(upstreams.map(async next => ({ + await saveUpstreamsAndWarmChangedModels(await Promise.all(upstreams.map(async next => ({ previous: await repo.upstreams.getById(next.id), next, }))), c); diff --git a/packages/gateway/src/control-plane/schemas.ts b/packages/gateway/src/control-plane/schemas.ts index 76a0baa56..d36e6a3a6 100644 --- a/packages/gateway/src/control-plane/schemas.ts +++ b/packages/gateway/src/control-plane/schemas.ts @@ -11,9 +11,9 @@ // // Deep upstream-config validation (e.g. Azure URL hostname rules, custom // pathOverrides and modelsFetch.endpoint URL parsing, per-model endpoint path -// checks) intentionally stays in the handler functions — they own the -// canonical error messages and downstream cache invalidation. The schemas -// here describe the shape the dashboard sends. +// checks) stays with provider validators and handlers, which own the canonical +// error messages. Repository model-aware writes own catalog generations and +// invalidation. The schemas here describe the shape the dashboard sends. import { z } from 'zod'; diff --git a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts index 0f4a9345d..bac0c2d71 100644 --- a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts +++ b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts @@ -4,45 +4,50 @@ import { warmUpstreamModels } from '../../data-plane/providers/models-refresh.ts import { createProvider } from '../../data-plane/providers/registry.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; import { getRepo } from '../../repo/index.ts'; +import type { StoredUpstreamRecord } from '../../repo/types.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { UpstreamRecord } from '@floway-dev/provider'; import { logInfo } from '@floway-dev/provider-claude-code'; export interface UpstreamModelsChange { - previous: UpstreamRecord | null; + previous: StoredUpstreamRecord | null; next: UpstreamRecord; } const errorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error); -const saveUpstreamForModels = async ({ previous, next }: UpstreamModelsChange): Promise => { +interface SavedUpstream { + record: StoredUpstreamRecord; + modelsChanged: boolean; +} + +const saveUpstreamForModels = async ({ previous, next }: UpstreamModelsChange): Promise => { const upstreams = getRepo().upstreams; if (previous === null) { const inserted = await upstreams.insertForModels(next); if (!inserted) throw new Error(`Upstream ${next.id} changed concurrently`); - return; + return { record: inserted, modelsChanged: true }; } const saved = await upstreams.replaceForModels({ previous, upstream: next }); if (!saved) throw new Error(`Upstream ${next.id} changed concurrently`); + return { record: saved, modelsChanged: saved.configVersion !== previous.configVersion }; }; -export const saveAndWarmUpstreamsForModels = async ( +export const saveUpstreamsAndWarmChangedModels = async ( changes: readonly UpstreamModelsChange[], c: Context, -): Promise> => { +): Promise> => { if (new Set(changes.map(change => change.next.id)).size !== changes.length) { throw new Error('Duplicate upstream ids in models save batch'); } - for (const change of changes) await saveUpstreamForModels(change); if (changes.length === 0) return new Map(); + const saved: SavedUpstream[] = []; + for (const change of changes) saved.push(await saveUpstreamForModels(change)); + const recordsToWarm = saved.filter(result => result.modelsChanged).map(result => result.record); + if (recordsToWarm.length === 0) return new Map(saved.map(result => [result.record.id, result.record])); - const records = await Promise.all(changes.map(async change => { - const record = await getRepo().upstreams.getById(change.next.id); - if (record === null) throw new Error(`Upstream ${change.next.id} disappeared after save`); - return record; - })); - const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw), records); - const entries = await Promise.all(records.map(async record => { + const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw), recordsToWarm); + const warmedEntries = await Promise.all(recordsToWarm.map(async record => { try { await warmUpstreamModels(createProvider(record), fetcherForUpstream(record.id)); } catch (error) { @@ -52,14 +57,16 @@ export const saveAndWarmUpstreamsForModels = async ( if (refreshed === null) throw new Error(`Upstream ${record.id} disappeared after warm`); return [record.id, refreshed] as const; })); - return new Map(entries); + const byId = new Map(saved.map(result => [result.record.id, result.record])); + for (const [id, record] of warmedEntries) byId.set(id, record); + return byId; }; -export const saveAndWarmUpstreamForModels = async ( +export const saveUpstreamAndWarmChangedModels = async ( change: UpstreamModelsChange, c: Context, -): Promise => { - const result = (await saveAndWarmUpstreamsForModels([change], c)).get(change.next.id); +): Promise => { + const result = (await saveUpstreamsAndWarmChangedModels([change], c)).get(change.next.id); if (result === undefined) throw new Error(`Missing saved upstream result for ${change.next.id}`); return result; }; diff --git a/packages/gateway/src/control-plane/upstreams/claude-code.ts b/packages/gateway/src/control-plane/upstreams/claude-code.ts index 6195bd63a..ee5d9d3e7 100644 --- a/packages/gateway/src/control-plane/upstreams/claude-code.ts +++ b/packages/gateway/src/control-plane/upstreams/claude-code.ts @@ -5,7 +5,7 @@ import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { claudeCodeOAuthAuthorizeUrlBody, claudeCodeOAuthExchangeBody, claudeCodeOAuthRefreshBody, claudeCodeProbeBody, claudeCodeSetupTokenAuthorizeUrlBody, claudeCodeSetupTokenExchangeBody } from '../schemas.ts'; -import { saveAndWarmUpstreamForModels } from '../shared/save-upstream-for-models.ts'; +import { saveUpstreamAndWarmChangedModels } from '../shared/save-upstream-for-models.ts'; import type { Fetcher, UpstreamRecord } from '@floway-dev/provider'; import { type ClaudeCodeAccountCredential, @@ -78,7 +78,7 @@ export const claudeCodeOAuthExchange = async (c: CtxWithJson ({ + fetchedAt: record.modelsCache?.fetchedAt ?? null, + lastError: record.modelsCache?.lastError ?? null, + modelCount: storedCatalogSize(record), +}); diff --git a/packages/gateway/src/control-plane/upstreams/models.ts b/packages/gateway/src/control-plane/upstreams/models.ts index b93e2601b..ef765e499 100644 --- a/packages/gateway/src/control-plane/upstreams/models.ts +++ b/packages/gateway/src/control-plane/upstreams/models.ts @@ -1,16 +1,16 @@ +import { modelsCacheStatus } from './models-cache-status.ts'; import { resolveControlPlaneFetcher } from './proxy-resolution.ts'; import { isValidProviderKind, upstreamErrorMessage as errorMessage } from './shared.ts'; -import type { ListedUpstreamModel } from './types.ts'; import { MODEL_LISTING_FAILURE_CODE, MODEL_LISTING_FAILURE_MESSAGE } from '../../data-plane/models/shared.ts'; import { fetchUpstreamModels } from '../../data-plane/providers/models-refresh.ts'; -import { createProvider } from '../../data-plane/providers/registry.ts'; +import { createPreviewProvider, createProvider } from '../../data-plane/providers/registry.ts'; import type { AuthedContext } from '../../middleware/auth.ts'; import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { previewModelsBody } from '../schemas.ts'; -import { ProviderModelsUnavailableError, type Fetcher, type ProviderModel, type ProxyFallbackEntry, type UpstreamRecord } from '@floway-dev/provider'; -import { assertCustomUpstreamRecord, fetchCustomModels, projectCustomModels } from '@floway-dev/provider-custom'; +import { ProviderModelsUnavailableError, type Fetcher, type ProviderModel, type ProxyFallbackEntry, type UpstreamModelConfig, type UpstreamRecord } from '@floway-dev/provider'; +import { assertCustomUpstreamRecord, fetchCustomModels, projectCustomModels, projectCustomDiscoveredModels } from '@floway-dev/provider-custom'; // `upstreamModelId` is the wire-side identifier the provider will send when // a caller invokes the public `model.id` — Claude Code exposes @@ -19,7 +19,7 @@ import { assertCustomUpstreamRecord, fetchCustomModels, projectCustomModels } fr // not a universal upstream-id field: only the providers that shape it as // `{ upstreamModelId }` surface a distinct wire id here, and the rest // (Copilot carries its raw variant list there) report the public id. -const reshapeModelForDashboard = (model: ProviderModel): ListedUpstreamModel => { +const reshapeModelForDashboard = (model: ProviderModel): UpstreamModelConfig => { const providerData = typeof model.providerData === 'object' && model.providerData !== null ? model.providerData as { upstreamModelId?: unknown } : null; const wireId = typeof providerData?.upstreamModelId === 'string' && providerData.upstreamModelId.length > 0 ? providerData.upstreamModelId : model.id; return { @@ -58,7 +58,6 @@ export const previewModels = async (c: CtxWithJson) => sortOrder: 0, createdAt: now, updatedAt: now, - configVersion: 1, flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList, @@ -85,10 +84,10 @@ export const previewModels = async (c: CtxWithJson) => if (kind === 'custom') { const assertedConfig = assertCustomUpstreamRecord(synthRecord).config; const result = await fetchCustomModels(assertedConfig, fetcher); - return c.json({ kind, data: result.data }); + return c.json({ data: projectCustomDiscoveredModels(synthRecord, result) }); } - const models = await createProvider(synthRecord).instance.getProvidedModels(fetcher); - return c.json({ kind, data: models.map(reshapeModelForDashboard) }); + const models = await createPreviewProvider(synthRecord).instance.getProvidedModels(fetcher); + return c.json({ data: models.map(reshapeModelForDashboard) }); } catch (e) { if (e instanceof ProviderModelsUnavailableError) { return c.json({ error: { message: MODEL_LISTING_FAILURE_MESSAGE, type: 'api_error', code: MODEL_LISTING_FAILURE_CODE } }, 502); @@ -119,20 +118,18 @@ export const fetchSavedModels = async (c: AuthedContext<'/:id/list-models'>) => } try { + let data: UpstreamModelConfig[]; if (record.kind === 'custom') { const config = assertCustomUpstreamRecord(record).config; - let result: Awaited> | undefined; - await fetchUpstreamModels(createProvider(record), fetcher, async () => { - result = await fetchCustomModels(config, fetcher); - return projectCustomModels(record, result); - }); - // Joining another runtime's refresh does not expose its raw custom wire - // response, which the editor needs for endpoint inference. - result ??= await fetchCustomModels(config, fetcher); - return c.json({ kind: record.kind, data: result.data }); + const result = await fetchCustomModels(config, fetcher); + await fetchUpstreamModels(createProvider(record), fetcher, async () => projectCustomModels(record, result)); + data = projectCustomDiscoveredModels(record, result); + } else { + data = (await fetchUpstreamModels(createProvider(record), fetcher)).map(reshapeModelForDashboard); } - const models = await fetchUpstreamModels(createProvider(record), fetcher); - return c.json({ kind: record.kind, data: models.map(reshapeModelForDashboard) }); + const refreshed = await getRepo().upstreams.getById(id); + if (refreshed === null) throw new Error(`Upstream ${id} disappeared after models refresh`); + return c.json({ data, modelsCache: modelsCacheStatus(refreshed) }); } catch (e) { if (e instanceof ProviderModelsUnavailableError) { return c.json({ error: { message: MODEL_LISTING_FAILURE_MESSAGE, type: 'api_error', code: MODEL_LISTING_FAILURE_CODE } }, 502); diff --git a/packages/gateway/src/control-plane/upstreams/routes.ts b/packages/gateway/src/control-plane/upstreams/routes.ts index c1d9f2813..86748feb0 100644 --- a/packages/gateway/src/control-plane/upstreams/routes.ts +++ b/packages/gateway/src/control-plane/upstreams/routes.ts @@ -1,5 +1,6 @@ import type { Context } from 'hono'; +import { modelsCacheStatus } from './models-cache-status.ts'; import { blueprintUpstreamRecord, upstreamRecordToFullJson, upstreamRecordToJson } from './serialize.ts'; import { isValidProviderKind, upstreamErrorMessage as errorMessage } from './shared.ts'; import type { FullSerializedUpstreamRecord, ModelsCacheStatus, RedactedSerializedUpstreamRecord } from './types.ts'; @@ -8,10 +9,11 @@ import { type AuthedContext } from '../../middleware/auth.ts'; import { type CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { isDirectFallbackId, normalizeProxyFallbackList } from '../../repo/proxy-fallback-list.ts'; +import type { StoredUpstreamRecord } from '../../repo/types.ts'; import { shortId } from '../../shared/short-id.ts'; import type { createUpstreamBody, updateUpstreamBody } from '../schemas.ts'; import { isRecord } from '../shared/field-validators.ts'; -import { saveAndWarmUpstreamForModels } from '../shared/save-upstream-for-models.ts'; +import { saveUpstreamAndWarmChangedModels } from '../shared/save-upstream-for-models.ts'; import { nextSortOrder } from '../shared/sort-order.ts'; import { normalizeModelPrefix, @@ -65,7 +67,7 @@ const pruneDeletedProxyEntries = ( // optional baseSerialize override lets callers swap in upstreamRecordToFullJson // to round-trip unredacted secrets instead of the redacted default. const serializeForResponse = async ( - record: UpstreamRecord, + record: StoredUpstreamRecord, knownProxyIds: ReadonlySet, baseSerialize: (r: UpstreamRecord) => SerializedUpstreamRecord = upstreamRecordToJson, ): Promise => { @@ -74,11 +76,7 @@ const serializeForResponse = async ( return { ...serialized, proxy_fallback_list: pruneDeletedProxyEntries(serialized.proxy_fallback_list, knownProxyIds), - modelsCache: { - fetchedAt: record.modelsCache?.fetchedAt ?? null, - lastError: record.modelsCache?.lastError ?? null, - modelCount: storedCatalogSize(record), - }, + modelsCache: modelsCacheStatus(record), ...codexQuota, }; }; @@ -246,7 +244,6 @@ export const createUpstream = async (c: CtxWithJson) sortOrder: body.sort_order ?? nextSortOrder(existing), createdAt: now, updatedAt: now, - configVersion: 1, flagOverrides: body.flag_overrides ?? {}, disabledPublicModelIds: body.disabled_public_model_ids ?? [], proxyFallbackList, @@ -281,7 +278,7 @@ export const createUpstream = async (c: CtxWithJson) const record = { ...upstream, config: config.value }; // Answer with the catalog status this warm produced, not the one the record // was built with — the dashboard re-seeds its draft from this body. - const saved = await saveAndWarmUpstreamForModels({ previous: null, next: record }, c); + const saved = await saveUpstreamAndWarmChangedModels({ previous: null, next: record }, c); return c.json(await serializeForResponse(saved, knownProxyIds), 201); }; @@ -335,7 +332,7 @@ export const updateUpstream = async (c: CtxWithJson; data: ListedUpstreamModel[] }; diff --git a/packages/gateway/src/data-plane/providers/models-refresh.ts b/packages/gateway/src/data-plane/providers/models-refresh.ts index 04584731f..e77384b9e 100644 --- a/packages/gateway/src/data-plane/providers/models-refresh.ts +++ b/packages/gateway/src/data-plane/providers/models-refresh.ts @@ -9,15 +9,14 @@ const ACTIVE_REFRESH_POLL_MS = 100; const ACTIVE_REFRESH_POLL_CAP_MS = 1_000; const ACTIVE_REFRESH_WAIT_MS = 60_000; -// L1: per-isolate in-flight memoization. Callers join only when both their -// actual fetch inputs and persisted-cache ownership match; different drafts -// and superseded rows remain isolated. Not a TTL cache — the entry is removed -// when the promise settles. The conditional delete defends against a stale -// removal racing a later replacement. -type RefreshIntent = 'explicit' | 'warm' | 'background'; +// L1: per-isolate in-flight memoization. Saved upstreams join only within one +// persisted config generation; draft previews bypass this coordinator. Not a +// TTL cache — the entry is removed when the promise settles. The conditional +// delete defends against a stale removal racing a later replacement. +type RefreshMode = 'explicit' | 'warm' | 'background'; interface InFlightRefresh { - kind: 'background-refresh' | 'explicit-refresh' | 'owner-wait'; + mode: RefreshMode; promise: Promise; } @@ -25,10 +24,10 @@ const inFlight = new Map(); const startInFlight = ( key: string, - kind: InFlightRefresh['kind'], + mode: RefreshMode, fn: () => Promise, ): Promise => { - const entry: InFlightRefresh = { kind, promise: fn() }; + const entry: InFlightRefresh = { mode, promise: fn() }; inFlight.set(key, entry); entry.promise.finally(() => { if (inFlight.get(key) === entry) inFlight.delete(key); @@ -38,11 +37,11 @@ const startInFlight = ( const memoInFlight = ( key: string, - kind: InFlightRefresh['kind'], + mode: RefreshMode, fn: () => Promise, ): Promise => { const existing = inFlight.get(key); - return existing?.promise ?? startInFlight(key, kind, fn); + return existing?.promise ?? startInFlight(key, mode, fn); }; const errorMessage = (err: unknown): string => err instanceof Error ? err.message : String(err); @@ -67,16 +66,10 @@ const finalizeRefresh = async ( throw new AggregateError(errors, 'Failed to finalize models refresh'); }; -const runFetch = async ( - instance: GatewayProvider, - fetcher: Fetcher, - loadProvidedModels?: () => Promise, -): Promise => [...await (loadProvidedModels?.() ?? instance.instance.getProvidedModels(fetcher))]; - const runClaimedRefresh = async ( instance: GatewayProvider, fetcher: Fetcher, - intent: RefreshIntent, + mode: RefreshMode, loadProvidedModels?: () => Promise, ): Promise => { const repo = getRepo(); @@ -92,7 +85,7 @@ const runClaimedRefresh = async ( token, now, staleClaimedBefore: now - MODELS_REFRESH_CLAIM_LEASE_MS, - bypassBackoff: intent === 'explicit', + bypassBackoff: mode === 'explicit', observedActiveToken, }); if (outcome.kind === 'backoff' || outcome.kind === 'generation-mismatch') return null; @@ -100,15 +93,12 @@ const runClaimedRefresh = async ( const current = await repo.upstreams.getById(instance.upstreamId); if (current === null || current.configVersion !== instance.modelsCacheGeneration.configVersion) return null; + if (current.modelsCache === null) throw new Error(`Completed models refresh for ${instance.upstreamId} has no cache`); instance.modelsCache = current.modelsCache; - if (intent === 'explicit' && current.modelsCache?.lastError !== null && current.modelsCache?.lastError !== undefined) { - observedActiveToken = null; - continue; - } - return current.modelsCache?.models ?? []; + return current.modelsCache.models; } if (outcome.kind === 'active') { - if (intent === 'background') return null; + if (mode === 'background') return null; if (now >= waitDeadline) throw new Error(`Timed out waiting for models refresh owner for ${instance.upstreamId}`); observedActiveToken = outcome.token; await new Promise(resolve => setTimeout(resolve, pollMs)); @@ -118,7 +108,7 @@ const runClaimedRefresh = async ( let models: ProviderModel[]; try { - models = await runFetch(instance, fetcher, loadProvidedModels); + models = [...await (loadProvidedModels?.() ?? instance.instance.getProvidedModels(fetcher))]; } catch (error) { const failedAt = Date.now(); const lastError = { message: errorMessage(error), at: failedAt }; @@ -143,7 +133,7 @@ const runClaimedRefresh = async ( else instance.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError }; throw error; } - if (intent === 'background') throw error; + if (mode === 'background') throw error; observedActiveToken = token; continue; } @@ -163,7 +153,7 @@ const runClaimedRefresh = async ( instance.modelsCache = entry; return models; } - if (intent === 'background') return models; + if (mode === 'background') return models; observedActiveToken = token; } }; @@ -181,12 +171,12 @@ export const fetchUpstreamModels = async ( const key = inFlightKey(instance); while (true) { const existing = inFlight.get(key); - if (existing?.kind === 'explicit-refresh') { + if (existing?.mode === 'explicit') { const joined = await existing.promise; if (joined === null) throw new Error(`Models refresh generation changed for ${instance.upstreamId}`); return joined; } - if (existing?.kind === 'background-refresh') { + if (existing?.mode === 'background') { try { const joined = await existing.promise; if (joined !== null) return joined; @@ -197,7 +187,7 @@ export const fetchUpstreamModels = async ( if (inFlight.get(key) === existing) inFlight.delete(key); continue; } - const models = await startInFlight(key, 'explicit-refresh', () => runClaimedRefresh(instance, fetcher, 'explicit', loadProvidedModels)); + const models = await startInFlight(key, 'explicit', () => runClaimedRefresh(instance, fetcher, 'explicit', loadProvidedModels)); if (models === null) throw new Error(`Failed to acquire models refresh for ${instance.upstreamId}`); return models; } @@ -206,18 +196,16 @@ export const fetchUpstreamModels = async ( export const warmUpstreamModels = async ( instance: GatewayProvider, fetcher: Fetcher, -): Promise => { +): Promise => { const key = inFlightKey(instance); const existing = inFlight.get(key); if (existing) { const joined = await existing.promise; - if (joined !== null) return joined; - if (existing.kind === 'owner-wait') return instance.modelsCache?.models ?? []; + if (joined !== null || existing.mode === 'warm') return; if (inFlight.get(key) === existing) inFlight.delete(key); } - const models = await memoInFlight(key, 'owner-wait', () => runClaimedRefresh(instance, fetcher, 'warm')); - return models ?? instance.modelsCache?.models ?? []; + await memoInFlight(key, 'warm', () => runClaimedRefresh(instance, fetcher, 'warm')); }; export const scheduleUpstreamModelsRefresh = ( @@ -226,7 +214,7 @@ export const scheduleUpstreamModelsRefresh = ( fetcher: Fetcher, ): void => { const key = inFlightKey(instance); - scheduler(memoInFlight(key, 'background-refresh', () => runClaimedRefresh(instance, fetcher, 'background')).then(() => {})); + scheduler(memoInFlight(key, 'background', () => runClaimedRefresh(instance, fetcher, 'background')).then(() => {})); }; // Test-only: drop the L1 map so a test's setup is independent of any diff --git a/packages/gateway/src/data-plane/providers/registry.ts b/packages/gateway/src/data-plane/providers/registry.ts index c6502ca09..f626281d0 100644 --- a/packages/gateway/src/data-plane/providers/registry.ts +++ b/packages/gateway/src/data-plane/providers/registry.ts @@ -1,6 +1,6 @@ import { getRepo } from '../../repo/index.ts'; import { modelsCacheGeneration } from '../../repo/models-cache-contract.ts'; -import type { ModelsCacheGeneration } from '../../repo/types.ts'; +import type { ModelsCacheGeneration, StoredUpstreamRecord } from '../../repo/types.ts'; import type { FlagDefaults, Provider, ProviderModule, UpstreamProviderKind, UpstreamRecord } from '@floway-dev/provider'; import { azureProviderModule } from '@floway-dev/provider-azure'; import { claudeCodeProviderModule } from '@floway-dev/provider-claude-code'; @@ -23,16 +23,18 @@ export type GatewayProvider = Provider & { }; export const createProvider = ( - record: UpstreamRecord, - cacheGeneration: ModelsCacheGeneration = modelsCacheGeneration(record), + record: StoredUpstreamRecord, ): GatewayProvider => { const provider = providersByKind[record.kind].create(record); return { ...provider, - modelsCacheGeneration: cacheGeneration, + modelsCacheGeneration: modelsCacheGeneration(record), }; }; +export const createPreviewProvider = (record: UpstreamRecord): Provider => + providersByKind[record.kind].create(record); + export const flagDefaultsForKind = (kind: UpstreamProviderKind): FlagDefaults => providersByKind[kind].defaultFlags; @@ -46,10 +48,10 @@ export const flagDefaultsForKind = (kind: UpstreamProviderKind): FlagDefaults => // this request instead of paying a second `upstreams.list()` round-trip. export const listModelProviders = async ( upstreamFilter: readonly string[] | null, - preFetchedUpstreams?: readonly UpstreamRecord[], + preFetchedUpstreams?: readonly StoredUpstreamRecord[], ): Promise => { const upstreams = preFetchedUpstreams ?? await getRepo().upstreams.list(); - const enabledById = new Map(); + const enabledById = new Map(); for (const upstream of upstreams) { if (upstream.enabled) enabledById.set(upstream.id, upstream); } @@ -62,7 +64,7 @@ export const listModelProviders = async ( // selection emptied this way surfaces downstream as "no upstream provider // configured". const selection = upstreamFilter - ? upstreamFilter.map(id => enabledById.get(id)).filter((u): u is UpstreamRecord => u !== undefined) + ? upstreamFilter.map(id => enabledById.get(id)).filter((u): u is StoredUpstreamRecord => u !== undefined) : [...enabledById.values()]; return selection.map(record => createProvider(record)); diff --git a/packages/gateway/src/data-plane/shared/listing/addressable.ts b/packages/gateway/src/data-plane/shared/listing/addressable.ts index 60748c889..082bab686 100644 --- a/packages/gateway/src/data-plane/shared/listing/addressable.ts +++ b/packages/gateway/src/data-plane/shared/listing/addressable.ts @@ -11,11 +11,12 @@ // DTO) read `limits` / `chat` / `endpoints` directly off the entry without // a second registry round trip. +import type { StoredUpstreamRecord } from '../../../repo/types.ts'; import { compareModelIds, getModelsFromProviders } from '../../providers/catalog.ts'; import { readUpstreamModelsSnapshotAndScheduleRefresh } from '../../providers/models-cache.ts'; import { listModelProviders } from '../../providers/registry.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; -import { isAbortError, type Fetcher, type InternalModel, type Provider, type UpstreamRecord } from '@floway-dev/provider'; +import { isAbortError, type Fetcher, type InternalModel, type Provider } from '@floway-dev/provider'; export interface AddressableIdEntry { // The inbound model id the data plane will accept verbatim. @@ -54,7 +55,7 @@ export const enumerateAddressableModelIds = async ( upstreamFilter: readonly string[] | null, fetcherForUpstream: (upstreamId: string) => Fetcher, scheduler: BackgroundScheduler, - preFetchedUpstreams?: readonly UpstreamRecord[], + preFetchedUpstreams?: readonly StoredUpstreamRecord[], ): Promise => { // Resolve providers once and thread them into the catalog assembly so // the upstreams.list() round-trip and provider-instantiation cost is diff --git a/packages/gateway/src/repo/models-cache-contract.ts b/packages/gateway/src/repo/models-cache-contract.ts index 852248c8f..989e0498e 100644 --- a/packages/gateway/src/repo/models-cache-contract.ts +++ b/packages/gateway/src/repo/models-cache-contract.ts @@ -1,5 +1,4 @@ -import type { ModelsCacheGeneration } from './types.ts'; -import type { UpstreamRecord } from '@floway-dev/provider'; +import type { ModelsCacheGeneration, StoredUpstreamRecord } from './types.ts'; // Persisted ProviderModel rows contain code-derived metadata as well as the // upstream response. Increment this whenever that derived catalog contract or @@ -9,7 +8,7 @@ export const MODEL_CATALOG_REVISION = 5; // Fetch ownership survives provider-managed state writes such as token // rotation, but changes whenever static request inputs or egress policy do. export const modelsCacheGeneration = ( - record: Pick, + record: Pick, ): ModelsCacheGeneration => ({ configVersion: record.configVersion, }); diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index 678bbab97..ad0f14ab4 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -42,6 +42,7 @@ import type { ResponsesItemsRepo, ResponsesSnapshotsRepo, SpilledFilesRepo, + StoredUpstreamRecord, WebSearchConfigRepo, WebSearchUsageRecord, WebSearchUsageRepo, @@ -879,14 +880,14 @@ export const UPSTREAM_STATE_WRITE_ATTEMPTS = 4; class SqlUpstreamRepo implements UpstreamRepo { constructor(private db: SqlDatabase) {} - async list(): Promise { + async list(): Promise { const { results } = await this.db .prepare('SELECT id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, models_cache_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue FROM upstreams ORDER BY sort_order, created_at') .all(); return results.map(toUpstreamRecord); } - async getById(id: string): Promise { + async getById(id: string): Promise { const row = await this.db .prepare('SELECT id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, models_cache_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue FROM upstreams WHERE id = ?') .bind(id) @@ -898,10 +899,10 @@ class SqlUpstreamRepo implements UpstreamRepo { return this.saveRecord(upstream); } - async insertForModels(upstream: UpstreamRecord): Promise { - if (upstream.configVersion !== 1) throw new Error(`New upstream ${upstream.id} must start at config version 1`); - const result = await this.db - .prepare('INSERT INTO upstreams (id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING') + async insertForModels(upstream: UpstreamRecord): Promise { + const row = await this.db + .prepare(`INSERT INTO upstreams (id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue) VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO NOTHING + RETURNING id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, models_cache_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue`) .bind( upstream.id, upstream.kind, @@ -910,7 +911,6 @@ class SqlUpstreamRepo implements UpstreamRepo { upstream.sortOrder, upstream.createdAt, upstream.updatedAt, - upstream.configVersion, serializeStoredConfig(upstream.config), serializeStoredState(upstream.state), JSON.stringify(normalizeFlagOverrides(upstream.flagOverrides)), @@ -919,25 +919,25 @@ class SqlUpstreamRepo implements UpstreamRepo { upstream.modelPrefix === null ? null : JSON.stringify(upstream.modelPrefix), upstream.hue, ) - .run(); - return (result.meta.changes ?? 0) > 0; + .first(); + return row === null ? null : toUpstreamRecord(row); } async replaceForModels(input: { - previous: UpstreamRecord; - upstream: Omit; - }): Promise { + previous: StoredUpstreamRecord; + upstream: UpstreamRecord; + }): Promise { const { previous, upstream } = input; - const configChanged = previous.kind !== upstream.kind - || serializeStoredConfig(previous.config) !== serializeStoredConfig(upstream.config); - const configVersion = previous.configVersion + (configChanged ? 1 : 0); + const modelConfigChanged = previous.kind !== upstream.kind + || serializeStoredConfig(previous.config) !== serializeStoredConfig(upstream.config) + || serializeStoredConfig(previous.flagOverrides) !== serializeStoredConfig(upstream.flagOverrides); const transportChanged = serializeStoredConfig(previous.proxyFallbackList) !== serializeStoredConfig(upstream.proxyFallbackList); + const refreshInputsChanged = modelConfigChanged || transportChanged; + const configVersion = previous.configVersion + (refreshInputsChanged ? 1 : 0); const replaceState = serializeStoredState(previous.state) !== serializeStoredState(upstream.state); - const modelsRefreshUpdate = configChanged || transportChanged - ? 'NULL' - : "CASE WHEN models_refresh_json IS NULL THEN NULL ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) END"; - const modelsCacheUpdate = configChanged ? ', models_cache_json = NULL' : ''; - const result = await this.db + const modelsRefreshUpdate = refreshInputsChanged ? ', models_refresh_json = NULL' : ''; + const modelsCacheUpdate = modelConfigChanged ? ', models_cache_json = NULL' : ''; + const row = await this.db .prepare( `UPDATE upstreams SET provider = ?, @@ -952,8 +952,7 @@ class SqlUpstreamRepo implements UpstreamRepo { disabled_public_model_ids = ?, proxy_fallback_list_json = ?, model_prefix_json = ?, - hue = ?, - models_refresh_json = ${modelsRefreshUpdate}${modelsCacheUpdate} + hue = ?${modelsRefreshUpdate}${modelsCacheUpdate} WHERE id = ? AND provider = ? AND name = ? @@ -967,7 +966,8 @@ class SqlUpstreamRepo implements UpstreamRepo { AND disabled_public_model_ids = ? AND proxy_fallback_list_json = ? AND model_prefix_json IS ? - AND hue = ?`, + AND hue = ? + RETURNING id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, models_cache_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue`, ) .bind( upstream.kind, @@ -1000,26 +1000,23 @@ class SqlUpstreamRepo implements UpstreamRepo { previous.modelPrefix === null ? null : JSON.stringify(previous.modelPrefix), previous.hue, ) - .run(); - return (result.meta.changes ?? 0) > 0; + .first(); + return row === null ? null : toUpstreamRecord(row); } private async saveRecord(upstream: UpstreamRecord): Promise { - if (upstream.configVersion !== 1 && await this.getById(upstream.id) === null) { - throw new Error(`New upstream ${upstream.id} must start at config version 1`); - } // created_at is deliberately not in the ON CONFLICT update list: the row's first INSERT // wins, and re-saves preserve that timestamp regardless of what the caller passes. await this.db .prepare( - `INSERT INTO upstreams (id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `INSERT INTO upstreams (id, provider, name, enabled, sort_order, created_at, updated_at, config_version, config_json, state_json, flag_overrides, disabled_public_model_ids, proxy_fallback_list_json, model_prefix_json, hue) VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET provider = excluded.provider, name = excluded.name, enabled = excluded.enabled, sort_order = excluded.sort_order, updated_at = excluded.updated_at, - config_version = CASE WHEN provider = excluded.provider AND config_json = excluded.config_json THEN config_version ELSE config_version + 1 END, + config_version = CASE WHEN provider = excluded.provider AND config_json = excluded.config_json AND flag_overrides = excluded.flag_overrides AND proxy_fallback_list_json = excluded.proxy_fallback_list_json THEN config_version ELSE config_version + 1 END, config_json = excluded.config_json, state_json = excluded.state_json, flag_overrides = excluded.flag_overrides, @@ -1027,11 +1024,10 @@ class SqlUpstreamRepo implements UpstreamRepo { proxy_fallback_list_json = excluded.proxy_fallback_list_json, model_prefix_json = excluded.model_prefix_json, hue = excluded.hue, - models_cache_json = CASE WHEN provider = excluded.provider AND config_json = excluded.config_json THEN models_cache_json ELSE NULL END, + models_cache_json = CASE WHEN provider = excluded.provider AND config_json = excluded.config_json AND flag_overrides = excluded.flag_overrides THEN models_cache_json ELSE NULL END, models_refresh_json = CASE - WHEN provider != excluded.provider OR config_json != excluded.config_json THEN NULL - WHEN models_refresh_json IS NULL THEN NULL - ELSE json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) + WHEN provider != excluded.provider OR config_json != excluded.config_json OR flag_overrides != excluded.flag_overrides OR proxy_fallback_list_json != excluded.proxy_fallback_list_json THEN NULL + ELSE models_refresh_json END`, ) .bind( @@ -1042,7 +1038,6 @@ class SqlUpstreamRepo implements UpstreamRepo { upstream.sortOrder, upstream.createdAt, upstream.updatedAt, - upstream.configVersion, serializeStoredConfig(upstream.config), serializeStoredState(upstream.state), JSON.stringify(normalizeFlagOverrides(upstream.flagOverrides)), @@ -1103,7 +1098,8 @@ class SqlUpstreamRepo implements UpstreamRepo { } async claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { - const { id, generation, token, now, staleClaimedBefore, bypassBackoff, observedActiveToken } = input; + const { id, generation, token, now, staleClaimedBefore, bypassBackoff } = input; + let observedActiveToken = input.observedActiveToken; while (true) { const row = await this.db .prepare( @@ -1151,7 +1147,10 @@ class SqlUpstreamRepo implements UpstreamRepo { continue; } if (state.claim_token !== null && state.claimed_at !== null && state.claimed_at > staleClaimedBefore) return { kind: 'active', token: state.claim_token }; - if (observedActiveToken !== null && state.claim_token === null) return { kind: 'completed' }; + if (observedActiveToken !== null && state.claim_token === null) { + observedActiveToken = null; + continue; + } if (!bypassBackoff && state.retry_at !== null && state.retry_at > now) return { kind: 'backoff' }; } } @@ -1210,7 +1209,7 @@ interface UpstreamRow { hue: number; } -const toUpstreamRecord = (row: UpstreamRow): UpstreamRecord => { +const toUpstreamRecord = (row: UpstreamRow): StoredUpstreamRecord => { const config = decodeUpstreamConfig(row.config_json, row.id); const state = row.state_json === null ? null : decodeUpstreamState(row.state_json, row.id); if (!Number.isSafeInteger(row.config_version) || row.config_version < 1) { diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index f3cb0f213..3291383b2 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -3,6 +3,10 @@ import type { AgentSetupRepository } from '@floway-dev/agent-setup'; import type { AliasSelection, AliasTarget, AnnouncedMetadata, BillingMetric, DecimalString, ModelKind, PricingSelector } from '@floway-dev/protocols/common'; import type { PerformanceTelemetryContext, UpstreamModelsCache, UpstreamRecord } from '@floway-dev/provider'; +// Persistence-owned catalog generation. Provider config, flag overrides, and +// catalog transport advance it; runtime state and non-model metadata do not. +export type StoredUpstreamRecord = UpstreamRecord & { configVersion: number }; + export interface ApiKey { id: string; userId: number; @@ -341,14 +345,14 @@ export interface WebSearchConfigRepo { } export interface UpstreamRepo { - list(): Promise; - getById(id: string): Promise; + list(): Promise; + getById(id: string): Promise; save(upstream: UpstreamRecord): Promise; - insertForModels(upstream: UpstreamRecord): Promise; + insertForModels(upstream: UpstreamRecord): Promise; replaceForModels(input: { - previous: UpstreamRecord; - upstream: Omit; - }): Promise; + previous: StoredUpstreamRecord; + upstream: UpstreamRecord; + }): Promise; delete(id: string): Promise; deleteAll(): Promise; // Upstream state write with optimistic concurrency, used both by the diff --git a/packages/provider-azure/__tests__/config_test.ts b/packages/provider-azure/__tests__/config_test.ts index c8cbd7b17..7e5aed10f 100644 --- a/packages/provider-azure/__tests__/config_test.ts +++ b/packages/provider-azure/__tests__/config_test.ts @@ -27,7 +27,6 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-azure/__tests__/fetch_test.ts b/packages/provider-azure/__tests__/fetch_test.ts index db73df8fd..e63805b4e 100644 --- a/packages/provider-azure/__tests__/fetch_test.ts +++ b/packages/provider-azure/__tests__/fetch_test.ts @@ -38,7 +38,6 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-azure/__tests__/provider_test.ts b/packages/provider-azure/__tests__/provider_test.ts index 9951e4f14..e2aaf356a 100644 --- a/packages/provider-azure/__tests__/provider_test.ts +++ b/packages/provider-azure/__tests__/provider_test.ts @@ -39,7 +39,6 @@ const azureRecord = (overrides: Partial = {}): UpstreamRecord => disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, ...rest, @@ -329,7 +328,6 @@ test('createAzureProvider exposes image models and routes generations with api-v disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { @@ -379,7 +377,6 @@ test('createAzureProvider callImagesEdits posts multipart with model replaced by disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/provider-claude-code/__tests__/access-token_test.ts b/packages/provider-claude-code/__tests__/access-token_test.ts index 85e3e0776..862382165 100644 --- a/packages/provider-claude-code/__tests__/access-token_test.ts +++ b/packages/provider-claude-code/__tests__/access-token_test.ts @@ -37,7 +37,6 @@ const makeRecord = (state: ClaudeCodeUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-claude-code/__tests__/config_test.ts b/packages/provider-claude-code/__tests__/config_test.ts index 503895753..e2ebd1bce 100644 --- a/packages/provider-claude-code/__tests__/config_test.ts +++ b/packages/provider-claude-code/__tests__/config_test.ts @@ -14,7 +14,7 @@ const good = { accounts: [goodAccount] }; const wrap = (config: unknown): UpstreamRecord => ({ id: 'up', kind: 'claude-code', name: 'n', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: config as UpstreamRecord['config'], state: null, - flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, configVersion: 1, modelsCache: null, hue: 210, + flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210, }); describe('assertClaudeCodeUpstreamRecord (config validation)', () => { @@ -66,7 +66,7 @@ describe('assertClaudeCodeUpstreamRecord (record-level checks)', () => { const record: UpstreamRecord = { id: 'up', kind: 'copilot', name: 'n', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: {}, state: null, - flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, configVersion: 1, modelsCache: null, hue: 210, + flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210, }; expect(() => assertClaudeCodeUpstreamRecord(record)).toThrow(); }); diff --git a/packages/provider-claude-code/__tests__/fetch_test.ts b/packages/provider-claude-code/__tests__/fetch_test.ts index aa91610e0..f923c7aae 100644 --- a/packages/provider-claude-code/__tests__/fetch_test.ts +++ b/packages/provider-claude-code/__tests__/fetch_test.ts @@ -60,7 +60,6 @@ const makeRecord = (state: ClaudeCodeUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-claude-code/__tests__/provider_test.ts b/packages/provider-claude-code/__tests__/provider_test.ts index 8f4466955..0db3f87d0 100644 --- a/packages/provider-claude-code/__tests__/provider_test.ts +++ b/packages/provider-claude-code/__tests__/provider_test.ts @@ -54,7 +54,6 @@ const makeRecord = (state: ClaudeCodeUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-codex/__tests__/access-token_test.ts b/packages/provider-codex/__tests__/access-token_test.ts index 51acc843a..31e4eface 100644 --- a/packages/provider-codex/__tests__/access-token_test.ts +++ b/packages/provider-codex/__tests__/access-token_test.ts @@ -28,7 +28,6 @@ const makeRecord = (state: CodexUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-codex/__tests__/config_test.ts b/packages/provider-codex/__tests__/config_test.ts index 580f71610..293437603 100644 --- a/packages/provider-codex/__tests__/config_test.ts +++ b/packages/provider-codex/__tests__/config_test.ts @@ -9,7 +9,7 @@ const good = { accounts: [goodAccount] }; const wrap = (config: unknown): UpstreamRecord => ({ id: 'up', kind: 'codex', name: 'n', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: config as UpstreamRecord['config'], state: null, - flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, configVersion: 1, modelsCache: null, hue: 210, + flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210, }); describe('assertCodexUpstreamRecord (config validation)', () => { @@ -42,7 +42,7 @@ describe('assertCodexUpstreamRecord (record-level checks)', () => { const record: UpstreamRecord = { id: 'up', kind: 'copilot', name: 'n', enabled: true, sortOrder: 0, createdAt: '', updatedAt: '', config: {}, state: null, - flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, configVersion: 1, modelsCache: null, hue: 210, + flagOverrides: {}, disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, modelsCache: null, hue: 210, }; expect(() => assertCodexUpstreamRecord(record)).toThrow(); }); diff --git a/packages/provider-codex/__tests__/fetch_test.ts b/packages/provider-codex/__tests__/fetch_test.ts index e55a7fd18..f3b222930 100644 --- a/packages/provider-codex/__tests__/fetch_test.ts +++ b/packages/provider-codex/__tests__/fetch_test.ts @@ -39,7 +39,6 @@ const makeRecord = (state: CodexUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-codex/__tests__/interceptors/responses/action-pivot_test.ts b/packages/provider-codex/__tests__/interceptors/responses/action-pivot_test.ts index 1a93caea2..6b9dda50e 100644 --- a/packages/provider-codex/__tests__/interceptors/responses/action-pivot_test.ts +++ b/packages/provider-codex/__tests__/interceptors/responses/action-pivot_test.ts @@ -49,7 +49,6 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-codex/__tests__/provider_test.ts b/packages/provider-codex/__tests__/provider_test.ts index 4a09a1680..84c2fd9a1 100644 --- a/packages/provider-codex/__tests__/provider_test.ts +++ b/packages/provider-codex/__tests__/provider_test.ts @@ -24,7 +24,6 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-codex/__tests__/quota_test.ts b/packages/provider-codex/__tests__/quota_test.ts index 5fbc7c91c..d4ee123ef 100644 --- a/packages/provider-codex/__tests__/quota_test.ts +++ b/packages/provider-codex/__tests__/quota_test.ts @@ -30,7 +30,6 @@ const makeRecord = (state: CodexUpstreamState): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }); diff --git a/packages/provider-copilot/__tests__/auth_test.ts b/packages/provider-copilot/__tests__/auth_test.ts index 7ed7e622b..00f9a7e98 100644 --- a/packages/provider-copilot/__tests__/auth_test.ts +++ b/packages/provider-copilot/__tests__/auth_test.ts @@ -31,7 +31,6 @@ const installRepoAndClearCache = async () => { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { githubHost: 'github.com', githubToken: 'ghu_test', user: { id: 1, login: 't', name: null, avatar_url: '' } }, @@ -427,7 +426,6 @@ test('copilotAuthedFetch persists a minted token even when the row changed durin disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { githubHost: 'github.com', githubToken: 'ghu_test', user: { id: 1, login: 't', name: null, avatar_url: '' } }, diff --git a/packages/provider-copilot/__tests__/fetch-models_test.ts b/packages/provider-copilot/__tests__/fetch-models_test.ts index 03c803010..d0d6d95b9 100644 --- a/packages/provider-copilot/__tests__/fetch-models_test.ts +++ b/packages/provider-copilot/__tests__/fetch-models_test.ts @@ -21,7 +21,6 @@ const installRepoAndConfig = async () => { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { githubHost: 'github.com', githubToken, user: { id: 1, login: 't', name: null, avatar_url: '' } }, diff --git a/packages/provider-copilot/__tests__/interceptors/responses/action-pivot_test.ts b/packages/provider-copilot/__tests__/interceptors/responses/action-pivot_test.ts index 5b1c26c9f..f27083f67 100644 --- a/packages/provider-copilot/__tests__/interceptors/responses/action-pivot_test.ts +++ b/packages/provider-copilot/__tests__/interceptors/responses/action-pivot_test.ts @@ -50,7 +50,6 @@ test('Copilot provider terminal dispatches on post-chain ctx.action (interceptor disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/provider-copilot/__tests__/provider_test.ts b/packages/provider-copilot/__tests__/provider_test.ts index 05758013c..0411ce1b9 100644 --- a/packages/provider-copilot/__tests__/provider_test.ts +++ b/packages/provider-copilot/__tests__/provider_test.ts @@ -39,7 +39,6 @@ const buildCopilotUpstream = (overrides: Partial = {}): Upstream disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, ...rest, diff --git a/packages/provider-custom/__tests__/config_test.ts b/packages/provider-custom/__tests__/config_test.ts index fdb20b9fa..2f69cb7a2 100644 --- a/packages/provider-custom/__tests__/config_test.ts +++ b/packages/provider-custom/__tests__/config_test.ts @@ -24,7 +24,6 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-custom/__tests__/fetch-models_test.ts b/packages/provider-custom/__tests__/fetch-models_test.ts index 6e7c55751..4796ea9cf 100644 --- a/packages/provider-custom/__tests__/fetch-models_test.ts +++ b/packages/provider-custom/__tests__/fetch-models_test.ts @@ -16,7 +16,6 @@ const upstreamRecord = () => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/provider-custom/__tests__/fetch_test.ts b/packages/provider-custom/__tests__/fetch_test.ts index 26c0ab18a..0896b04e7 100644 --- a/packages/provider-custom/__tests__/fetch_test.ts +++ b/packages/provider-custom/__tests__/fetch_test.ts @@ -37,7 +37,6 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-custom/__tests__/infer-endpoints_test.ts b/packages/provider-custom/__tests__/infer-endpoints_test.ts index 9199727f5..da172c061 100644 --- a/packages/provider-custom/__tests__/infer-endpoints_test.ts +++ b/packages/provider-custom/__tests__/infer-endpoints_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; import { inferEndpointsFromModelId } from '../src/infer-endpoints.ts'; -import { createCustomProvider } from '../src/provider.ts'; +import { createCustomProvider, projectCustomDiscoveredModels } from '../src/provider.ts'; import { directFetcher, type UpstreamRecord } from '@floway-dev/provider'; import { assertEquals, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; @@ -117,7 +117,6 @@ test('Custom provider projects gpt-image-* models with kind=image and both image disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }; @@ -137,3 +136,41 @@ test('Custom provider projects gpt-image-* models with kind=image and both image }, ); }); + +test('Custom dashboard projection shares endpoint inference and preserves unroutable rerank rows', () => { + const record: UpstreamRecord = { + id: 'up_custom_preview', + kind: 'custom', + name: 'Custom Preview', + enabled: true, + sortOrder: 0, + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + config: { + baseUrl: 'https://custom.example.com', + authStyle: 'bearer', + apiKey: 'sk-custom', + endpoints: { chatCompletions: {} }, + ingressHeadersRules: [], + }, + state: null, + flagOverrides: {}, + disabledPublicModelIds: [], + proxyFallbackList: [], + modelPrefix: null, + modelsCache: null, + hue: 210, + }; + const models = projectCustomDiscoveredModels(record, { + data: [ + { id: 'gpt-image-2' }, + { id: 'speech', kind: 'transcription' }, + { id: 'ranker', kind: 'rerank' }, + ], + }); + assertEquals(models.map(model => ({ id: model.upstreamModelId, kind: model.kind, endpoints: model.endpoints })), [ + { id: 'gpt-image-2', kind: 'image', endpoints: IMAGES }, + { id: 'speech', kind: 'transcription', endpoints: AUDIO }, + { id: 'ranker', kind: 'rerank', endpoints: { rerank: {} } }, + ]); +}); diff --git a/packages/provider-custom/__tests__/provider_test.ts b/packages/provider-custom/__tests__/provider_test.ts index 8af363489..ffe10a4a5 100644 --- a/packages/provider-custom/__tests__/provider_test.ts +++ b/packages/provider-custom/__tests__/provider_test.ts @@ -26,7 +26,6 @@ const buildCustomUpstream = (options: BuildOptions = {}): UpstreamRecord => ({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, config: { diff --git a/packages/provider-custom/src/index.ts b/packages/provider-custom/src/index.ts index 6c3fc8264..19f4a9c61 100644 --- a/packages/provider-custom/src/index.ts +++ b/packages/provider-custom/src/index.ts @@ -9,4 +9,4 @@ export const customProviderModule: ProviderModule = { export { assertCustomUpstreamRecord, type CustomIngressHeaderRule, type CustomModelsFetch, type CustomUpstreamConfig } from './config.ts'; export { fetchCustomModels, type CustomModelsResponse, type CustomRawModel } from './fetch-models.ts'; -export { projectCustomModels } from './provider.ts'; +export { projectCustomModels, projectCustomDiscoveredModels } from './provider.ts'; diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index 333fc57fc..f7f2c9672 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -8,7 +8,7 @@ import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/com import { parseMessagesStream } from '@floway-dev/protocols/messages'; import { DEFAULT_RERANK_PATHS, serializeRerankRequest } from '@floway-dev/protocols/rerank'; import { parseResponsesStream, type ResponsesCompactionResult, toCompactPayloadShape } from '@floway-dev/protocols/responses'; -import { headersForMessagesCall, serializeOpenAIAudioTranscriptionRequest, serializeOpenAIImagesEditsRequest, publicModelId, resolveEffectiveFlags, streamingProviderCall, type FlagId, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; +import { headersForMessagesCall, serializeOpenAIAudioTranscriptionRequest, serializeOpenAIImagesEditsRequest, publicModelId, resolveEffectiveFlags, streamingProviderCall, type FlagId, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamModelConfig, type UpstreamRecord } from '@floway-dev/provider'; const rawModelIdOf = (model: ProviderModel): string => model.providerData as string; @@ -47,6 +47,29 @@ const autoModelEndpoints = (model: CustomRawModel, configured: ModelEndpoints): return inferEndpointsFromModelId(model.id) ?? configured; }; +export const projectCustomDiscoveredModels = ( + record: UpstreamRecord, + response: CustomModelsResponse, +): UpstreamModelConfig[] => { + const { config } = assertCustomUpstreamRecord(record); + return response.data.map(model => { + const endpoints = model.kind === 'rerank' ? { rerank: {} } : autoModelEndpoints(model, config.endpoints); + const kind = model.kind === 'rerank' ? 'rerank' : kindForEndpoints(endpoints); + const projected: UpstreamModelConfig = { + upstreamModelId: model.id, + publicModelId: model.id, + kind, + endpoints, + }; + const displayName = model.display_name ?? model.name; + if (displayName !== undefined) projected.display_name = displayName; + if (model.limits !== undefined) projected.limits = { ...model.limits }; + if (model.pricing !== undefined) projected.pricing = model.pricing; + if (model.chat !== undefined) projected.chat = model.chat; + return projected; + }); +}; + const finalizeCustomModels = ( response: CustomModelsResponse, configuredEndpoints: ModelEndpoints, diff --git a/packages/provider-ollama/__tests__/config_test.ts b/packages/provider-ollama/__tests__/config_test.ts index 361a77e76..6a99a3674 100644 --- a/packages/provider-ollama/__tests__/config_test.ts +++ b/packages/provider-ollama/__tests__/config_test.ts @@ -21,7 +21,6 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-ollama/__tests__/fetch-models_test.ts b/packages/provider-ollama/__tests__/fetch-models_test.ts index 7d6a3179b..3dc01d6ea 100644 --- a/packages/provider-ollama/__tests__/fetch-models_test.ts +++ b/packages/provider-ollama/__tests__/fetch-models_test.ts @@ -19,7 +19,6 @@ const config: OllamaUpstreamConfig = assertOllamaUpstreamRecord({ disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }).config; diff --git a/packages/provider-ollama/__tests__/fetch_test.ts b/packages/provider-ollama/__tests__/fetch_test.ts index 809bced6a..d81628d16 100644 --- a/packages/provider-ollama/__tests__/fetch_test.ts +++ b/packages/provider-ollama/__tests__/fetch_test.ts @@ -32,7 +32,6 @@ const baseRecord: UpstreamRecord = { disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, }; diff --git a/packages/provider-ollama/__tests__/provider_test.ts b/packages/provider-ollama/__tests__/provider_test.ts index ab222778c..d22e222b0 100644 --- a/packages/provider-ollama/__tests__/provider_test.ts +++ b/packages/provider-ollama/__tests__/provider_test.ts @@ -19,7 +19,6 @@ const buildRecord = (overrides: Partial = {}): UpstreamRecord => disabledPublicModelIds: [], proxyFallbackList: [], modelPrefix: null, - configVersion: 1, modelsCache: null, hue: 210, ...overrides, diff --git a/packages/provider/src/model.ts b/packages/provider/src/model.ts index ad7206210..7fff1cb38 100644 --- a/packages/provider/src/model.ts +++ b/packages/provider/src/model.ts @@ -68,17 +68,13 @@ export interface UpstreamRecord { sortOrder: number; createdAt: string; updatedAt: string; - // Monotonic generation of provider kind/configuration. Runtime state and - // operator metadata do not change it. - configVersion: number; config: unknown; // Gateway-written state that can change without an operator editing config; // null when a provider has no runtime state. state: unknown; // The upstream's cached catalog, read on the same round trip as the row // rather than through a second query. Null until the first successful fetch. - // Written only by the catalog refresh path — an operator save leaves it - // alone. + // Catalog refresh writes it; changing provider configuration clears it. modelsCache: UpstreamModelsCache | null; flagOverrides: FlagOverrides; // Model ids the operator switched off for this upstream, matched against the From 99ace166a3f5421ebe747e460402aaa2f8d46046 Mon Sep 17 00:00:00 2001 From: Menci Date: Fri, 7 Aug 2026 03:53:35 +0800 Subject: [PATCH 44/46] refactor(platform): unify durable execution cells Replace BroadcastDO with a protocol-driven ExecutionDO and migrate broadcast fan-out onto the platform execution-cell namespace. Route scheduled model refreshes through per-epoch cells; Cloudflare delegates database-owning work to a loopback WorkerEntrypoint while Node coalesces in process. Preserve the historical Durable Object class through a rename migration and enable ctx.exports. --- .../durable-object-execution-cell_test.ts | 30 +++++++ ... => execution-cell-channel-broker_test.ts} | 85 ++++++++++--------- ...adcast-do_test.ts => execution-do_test.ts} | 76 +++++++++++------ apps/platform-cloudflare/entry.ts | 17 +++- apps/platform-cloudflare/src/bootstrap.ts | 16 ++-- apps/platform-cloudflare/src/broadcast-do.ts | 42 --------- .../src/cloudflare-workers.d.ts | 24 ++++-- .../src/durable-object-execution-cell.ts | 14 +++ ...er.ts => execution-cell-channel-broker.ts} | 56 +++++------- apps/platform-cloudflare/src/execution-do.ts | 59 +++++++++++++ apps/platform-node/entry.ts | 5 +- .../scheduled/models-refresh_test.ts | 5 +- packages/gateway/__tests__/vitest.setup.ts | 5 +- packages/gateway/src/execution/handler.ts | 26 ++++++ .../gateway/src/execution/models-refresh.ts | 50 +++++++++++ packages/gateway/src/index.ts | 2 + packages/gateway/src/runtime/execution.ts | 12 +++ .../gateway/src/scheduled/models-refresh.ts | 8 +- packages/platform/src/execution-cell.ts | 45 ++++++++++ packages/platform/src/index.ts | 1 + wrangler.example.jsonc | 22 +++-- 21 files changed, 419 insertions(+), 181 deletions(-) create mode 100644 apps/platform-cloudflare/__tests__/durable-object-execution-cell_test.ts rename apps/platform-cloudflare/__tests__/{durable-object-channel-broker_test.ts => execution-cell-channel-broker_test.ts} (75%) rename apps/platform-cloudflare/__tests__/{broadcast-do_test.ts => execution-do_test.ts} (63%) delete mode 100644 apps/platform-cloudflare/src/broadcast-do.ts create mode 100644 apps/platform-cloudflare/src/durable-object-execution-cell.ts rename apps/platform-cloudflare/src/{durable-object-channel-broker.ts => execution-cell-channel-broker.ts} (66%) create mode 100644 apps/platform-cloudflare/src/execution-do.ts create mode 100644 packages/gateway/src/execution/handler.ts create mode 100644 packages/gateway/src/execution/models-refresh.ts create mode 100644 packages/gateway/src/runtime/execution.ts create mode 100644 packages/platform/src/execution-cell.ts diff --git a/apps/platform-cloudflare/__tests__/durable-object-execution-cell_test.ts b/apps/platform-cloudflare/__tests__/durable-object-execution-cell_test.ts new file mode 100644 index 000000000..eb4c4663b --- /dev/null +++ b/apps/platform-cloudflare/__tests__/durable-object-execution-cell_test.ts @@ -0,0 +1,30 @@ +import { test } from 'vitest'; + +import { DurableObjectExecutionCellNamespace, type ExecutionDurableObjectNamespace } from '../src/durable-object-execution-cell.ts'; +import { assertEquals } from '@floway-dev/test-utils'; + +test('DurableObject execution cells route a stable name to the matching stub', async () => { + const names: string[] = []; + const requests: Request[] = []; + const namespace: ExecutionDurableObjectNamespace = { + idFromName(name) { + names.push(name); + return name; + }, + get(_id) { + return { + async fetch(request) { + requests.push(request); + return new Response('done'); + }, + }; + }, + }; + + const cells = new DurableObjectExecutionCellNamespace(namespace); + const response = await cells.fetch('models:upstream-a:3', new Request('https://execution.do/models/refresh')); + + assertEquals(names, ['models:upstream-a:3']); + assertEquals(requests[0].url, 'https://execution.do/models/refresh'); + assertEquals(await response.text(), 'done'); +}); diff --git a/apps/platform-cloudflare/__tests__/durable-object-channel-broker_test.ts b/apps/platform-cloudflare/__tests__/execution-cell-channel-broker_test.ts similarity index 75% rename from apps/platform-cloudflare/__tests__/durable-object-channel-broker_test.ts rename to apps/platform-cloudflare/__tests__/execution-cell-channel-broker_test.ts index 32f72c4c5..3c08ba352 100644 --- a/apps/platform-cloudflare/__tests__/durable-object-channel-broker_test.ts +++ b/apps/platform-cloudflare/__tests__/execution-cell-channel-broker_test.ts @@ -1,7 +1,7 @@ import { test } from 'vitest'; -import { DurableObjectChannelBroker, type BroadcastNamespace } from '../src/durable-object-channel-broker.ts'; -import type { ChannelCodec } from '@floway-dev/platform'; +import { ExecutionCellChannelBroker } from '../src/execution-cell-channel-broker.ts'; +import type { ChannelCodec, ExecutionCellNamespace } from '@floway-dev/platform'; import { assertEquals } from '@floway-dev/test-utils'; // String codec: encode passes through, decode rejects payloads prefixed with @@ -51,31 +51,32 @@ const buildNamespace = ( closeAlls: string[] = [], fetches?: { count: number }, ) => { - const ns: BroadcastNamespace = { - idFromName(_name) { return {}; }, - get(_id) { - return { - broadcast: async payload => { broadcasts.push(payload); }, - closeAll: async reason => { closeAlls.push(reason); }, - fetch: async () => { - if (fetches) fetches.count += 1; - // Real CF returns 101; Node's `Response` rejects status 101 in its - // constructor, so synthesise it by overriding `status` after the - // fact. The broker only reads `status` and `webSocket`. - const response = new Response(null, { status: 200 }); - Object.defineProperty(response, 'status', { value: 101, configurable: true }); - Object.defineProperty(response, 'webSocket', { value: socket, configurable: true }); - return response; - }, - }; + const ns: ExecutionCellNamespace = { + async fetch(_cellId, request) { + if (fetches) fetches.count += 1; + const url = new URL(request.url); + if (url.pathname === '/broadcast' && request.method === 'POST') { + broadcasts.push(await request.text()); + return new Response(null, { status: 204 }); + } + if (url.pathname === '/broadcast/close' && request.method === 'POST') { + closeAlls.push(await request.text()); + return new Response(null, { status: 204 }); + } + // Real CF returns 101; Node's `Response` rejects status 101 in its + // constructor, so synthesise it by overriding `status` after the fact. + const response = new Response(null, { status: 200 }); + Object.defineProperty(response, 'status', { value: 101, configurable: true }); + Object.defineProperty(response, 'webSocket', { value: socket, configurable: true }); + return response; }, }; return ns; }; -test('DurableObjectChannelBroker.subscribe drives payloads through the broadcast socket', async () => { +test('ExecutionCellChannelBroker.subscribe drives payloads through the broadcast socket', async () => { const socket = new FakeServerSocket(); - const broker = new DurableObjectChannelBroker(buildNamespace(socket), stringCodec); + const broker = new ExecutionCellChannelBroker(buildNamespace(socket), stringCodec); const controller = new AbortController(); const iter = broker.subscribe('k', controller.signal)[Symbol.asyncIterator](); @@ -98,10 +99,10 @@ test('DurableObjectChannelBroker.subscribe drives payloads through the broadcast assertEquals(socket.closed?.code, 1000); }); -test('DurableObjectChannelBroker.subscribe does not open a socket for an already-aborted signal', async () => { +test('ExecutionCellChannelBroker.subscribe does not open a socket for an already-aborted signal', async () => { const socket = new FakeServerSocket(); const fetches = { count: 0 }; - const broker = new DurableObjectChannelBroker(buildNamespace(socket, [], [], fetches), stringCodec); + const broker = new ExecutionCellChannelBroker(buildNamespace(socket, [], [], fetches), stringCodec); const controller = new AbortController(); controller.abort(); @@ -112,9 +113,9 @@ test('DurableObjectChannelBroker.subscribe does not open a socket for an already assertEquals(socket.closed, null); }); -test('DurableObjectChannelBroker.subscribe resolves concurrent reads in socket order', async () => { +test('ExecutionCellChannelBroker.subscribe resolves concurrent reads in socket order', async () => { const socket = new FakeServerSocket(); - const broker = new DurableObjectChannelBroker(buildNamespace(socket), stringCodec); + const broker = new ExecutionCellChannelBroker(buildNamespace(socket), stringCodec); const controller = new AbortController(); const iter = broker.subscribe('k', controller.signal)[Symbol.asyncIterator](); const first = iter.next(); @@ -130,27 +131,27 @@ test('DurableObjectChannelBroker.subscribe resolves concurrent reads in socket o controller.abort(); }); -test('DurableObjectChannelBroker.publish encodes the payload through the codec', async () => { +test('ExecutionCellChannelBroker.publish encodes the payload through the codec', async () => { const broadcasts: string[] = []; const ns = buildNamespace(new FakeServerSocket(), broadcasts); - const broker = new DurableObjectChannelBroker(ns, stringCodec); + const broker = new ExecutionCellChannelBroker(ns, stringCodec); await broker.publish('k', 'frame-a'); assertEquals(broadcasts.length, 1); assertEquals(broadcasts[0], 'frame-a'); }); -test('DurableObjectChannelBroker.closeChannel forwards the reason to the actor', async () => { +test('ExecutionCellChannelBroker.closeChannel forwards the reason to the actor', async () => { const closeAlls: string[] = []; const ns = buildNamespace(new FakeServerSocket(), [], closeAlls); - const broker = new DurableObjectChannelBroker(ns, stringCodec); + const broker = new ExecutionCellChannelBroker(ns, stringCodec); await broker.closeChannel('k', 'custom-reason'); assertEquals(closeAlls.length, 1); assertEquals(closeAlls[0], 'custom-reason'); }); -test('DurableObjectChannelBroker.subscribe rejects every pending read when decoding fails', async () => { +test('ExecutionCellChannelBroker.subscribe rejects every pending read when decoding fails', async () => { const socket = new FakeServerSocket(); - const broker = new DurableObjectChannelBroker(buildNamespace(socket), stringCodec); + const broker = new ExecutionCellChannelBroker(buildNamespace(socket), stringCodec); const controller = new AbortController(); const iter = broker.subscribe('k', controller.signal)[Symbol.asyncIterator](); @@ -167,9 +168,9 @@ test('DurableObjectChannelBroker.subscribe rejects every pending read when decod assertEquals((results[1] as PromiseRejectedResult).reason.message, 'stringCodec rejected payload: bad:payload'); }); -test('DurableObjectChannelBroker.subscribe drains buffered payloads before surfacing a decode failure', async () => { +test('ExecutionCellChannelBroker.subscribe drains buffered payloads before surfacing a decode failure', async () => { const socket = new FakeServerSocket(); - const broker = new DurableObjectChannelBroker(buildNamespace(socket), stringCodec); + const broker = new ExecutionCellChannelBroker(buildNamespace(socket), stringCodec); const controller = new AbortController(); const iter = broker.subscribe('k', controller.signal)[Symbol.asyncIterator](); @@ -184,9 +185,9 @@ test('DurableObjectChannelBroker.subscribe drains buffered payloads before surfa assertEquals((failed[0] as PromiseRejectedResult).reason.message, 'stringCodec rejected payload: bad:payload'); }); -test('DurableObjectChannelBroker.subscribe ends the iterator on a server-initiated socket close', async () => { +test('ExecutionCellChannelBroker.subscribe ends the iterator on a server-initiated socket close', async () => { const socket = new FakeServerSocket(); - const broker = new DurableObjectChannelBroker(buildNamespace(socket), stringCodec); + const broker = new ExecutionCellChannelBroker(buildNamespace(socket), stringCodec); const controller = new AbortController(); const iter = broker.subscribe('k', controller.signal)[Symbol.asyncIterator](); @@ -198,9 +199,9 @@ test('DurableObjectChannelBroker.subscribe ends the iterator on a server-initiat assertEquals(result.done, true); }); -test('DurableObjectChannelBroker.subscribe surfaces a server-side socket error by throwing from .next()', async () => { +test('ExecutionCellChannelBroker.subscribe surfaces a server-side socket error by throwing from .next()', async () => { const socket = new FakeServerSocket(); - const broker = new DurableObjectChannelBroker(buildNamespace(socket), stringCodec); + const broker = new ExecutionCellChannelBroker(buildNamespace(socket), stringCodec); const controller = new AbortController(); const iter = broker.subscribe('k', controller.signal)[Symbol.asyncIterator](); @@ -215,12 +216,12 @@ test('DurableObjectChannelBroker.subscribe surfaces a server-side socket error b caught = err; } assertEquals(caught instanceof Error, true); - assertEquals((caught as Error).message, 'BroadcastDO socket error'); + assertEquals((caught as Error).message, 'ExecutionDO socket error'); }); -test('DurableObjectChannelBroker.subscribe delivers a frame buffered before the first .next() call', async () => { +test('ExecutionCellChannelBroker.subscribe delivers a frame buffered before the first .next() call', async () => { const socket = new FakeServerSocket(); - const broker = new DurableObjectChannelBroker(buildNamespace(socket), stringCodec); + const broker = new ExecutionCellChannelBroker(buildNamespace(socket), stringCodec); const controller = new AbortController(); const iter = broker.subscribe('k', controller.signal)[Symbol.asyncIterator](); @@ -234,9 +235,9 @@ test('DurableObjectChannelBroker.subscribe delivers a frame buffered before the assertEquals(first.value, 'pre-buffered'); }); -test('DurableObjectChannelBroker.subscribe closes the socket when the iterator returns', async () => { +test('ExecutionCellChannelBroker.subscribe closes the socket when the iterator returns', async () => { const socket = new FakeServerSocket(); - const broker = new DurableObjectChannelBroker(buildNamespace(socket), stringCodec); + const broker = new ExecutionCellChannelBroker(buildNamespace(socket), stringCodec); const controller = new AbortController(); const iter = broker.subscribe('k', controller.signal)[Symbol.asyncIterator](); diff --git a/apps/platform-cloudflare/__tests__/broadcast-do_test.ts b/apps/platform-cloudflare/__tests__/execution-do_test.ts similarity index 63% rename from apps/platform-cloudflare/__tests__/broadcast-do_test.ts rename to apps/platform-cloudflare/__tests__/execution-do_test.ts index 64472bc12..710ef9f69 100644 --- a/apps/platform-cloudflare/__tests__/broadcast-do_test.ts +++ b/apps/platform-cloudflare/__tests__/execution-do_test.ts @@ -1,7 +1,7 @@ import { DurableObject } from 'cloudflare:workers'; import { test } from 'vitest'; -import { BroadcastDO } from '../src/broadcast-do.ts'; +import { ExecutionDO } from '../src/execution-do.ts'; import { assertEquals } from '@floway-dev/test-utils'; // Minimal stub of the CF DurableObject runtime surface the actor touches. @@ -27,10 +27,7 @@ class FakeWebSocket implements WebSocket { closed: { code: number; reason: string } | null = null; send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void { - // BroadcastDO's `broadcast(payload: string)` contract forbids non-string - // payloads; throw loud on any binary input so a regression that sneaks - // ArrayBuffer/Blob through surfaces here instead of becoming a silent - // empty frame on the wire. + // Broadcast protocol payloads are text; reject binary test input loudly. if (typeof data !== 'string') throw new Error('FakeWebSocket.send: expected string payload'); this.sent.push(data); } @@ -43,6 +40,11 @@ class FakeWebSocket implements WebSocket { class FakeState { readonly sockets: FakeWebSocket[] = []; + readonly exports = { + ExecutionOperationEntrypoint: { + fetch: async (_request: Request) => new Response('executed'), + }, + }; acceptWebSocket(ws: WebSocket): void { this.sockets.push(ws as FakeWebSocket); } @@ -54,69 +56,65 @@ class FakeState { } } -test('BroadcastDO extends DurableObject so the runtime gates RPC dispatch on it', () => { - // BroadcastDO must extend DurableObject so the CF runtime gates RPC - // dispatch on the subclass; without the extends declaration, direct method - // invocation (`stub.broadcast(...)`, `stub.closeAll(...)`) fails with - // "the receiving Durable Object does not support RPC". The unit-test - // surface doesn't reach the runtime RPC machinery, so the prototype-chain - // check pins that the extends declaration is present. - assertEquals(Object.getPrototypeOf(BroadcastDO.prototype) === DurableObject.prototype, true); +test('ExecutionDO extends the platform DurableObject base', () => { + assertEquals(Object.getPrototypeOf(ExecutionDO.prototype) === DurableObject.prototype, true); }); -test('BroadcastDO.broadcast sends the payload verbatim to every registered socket', async () => { +test('ExecutionDO broadcast request sends the payload verbatim to every registered socket', async () => { const state = new FakeState(); const ws1 = new FakeWebSocket(); const ws2 = new FakeWebSocket(); state.push(ws1); state.push(ws2); - const actor = new BroadcastDO(state, {}); + const actor = new ExecutionDO(state, {}); - await actor.broadcast('hello world'); + const response = await actor.fetch(new Request('https://execution.do/broadcast', { method: 'POST', body: 'hello world' })); + assertEquals(response.status, 204); assertEquals(ws1.sent.length, 1); assertEquals(ws1.sent[0], 'hello world'); assertEquals(ws2.sent[0], 'hello world'); }); -test('BroadcastDO.closeAll closes every socket with the given reason and code 1000', async () => { +test('ExecutionDO close request closes every socket with the given reason and code 1000', async () => { const state = new FakeState(); const ws1 = new FakeWebSocket(); const ws2 = new FakeWebSocket(); state.push(ws1); state.push(ws2); - const actor = new BroadcastDO(state, {}); + const actor = new ExecutionDO(state, {}); - await actor.closeAll('reason of the day'); + const response = await actor.fetch(new Request('https://execution.do/broadcast/close', { method: 'POST', body: 'reason of the day' })); + assertEquals(response.status, 204); assertEquals(ws1.closed?.code, 1000); assertEquals(ws1.closed?.reason, 'reason of the day'); assertEquals(ws2.closed?.code, 1000); assertEquals(ws2.closed?.reason, 'reason of the day'); }); -test('BroadcastDO.webSocketClose calls ws.close to complete the close handshake', async () => { - const actor = new BroadcastDO(new FakeState(), {}); +test('ExecutionDO.webSocketClose calls ws.close to complete the close handshake', async () => { + const actor = new ExecutionDO(new FakeState(), {}); const ws = new FakeWebSocket(); await actor.webSocketClose(ws, 1001, 'going away', true); assertEquals(ws.closed?.code, 1001); assertEquals(ws.closed?.reason, 'going away'); }); -test('BroadcastDO.webSocketError exists so the runtime delivers close events', async () => { +test('ExecutionDO.webSocketError exists so the runtime delivers close events', async () => { // The hook's mere presence is what gates close-event delivery; assert the // method is declared on the class itself so the gating contract survives a // refactor that mistakes the no-op body for dead code. - assertEquals(typeof BroadcastDO.prototype.webSocketError, 'function'); - assertEquals(Object.prototype.hasOwnProperty.call(BroadcastDO.prototype, 'webSocketError'), true); - const actor = new BroadcastDO(new FakeState(), {}); + assertEquals(typeof ExecutionDO.prototype.webSocketError, 'function'); + assertEquals(Object.prototype.hasOwnProperty.call(ExecutionDO.prototype, 'webSocketError'), true); + const actor = new ExecutionDO(new FakeState(), {}); const ws = new FakeWebSocket(); await actor.webSocketError(ws, new Error('whatever')); // No side effect — the runtime drops the socket from getWebSockets() on its own. assertEquals(ws.closed, null); }); -test('BroadcastDO.fetch upgrades to a WebSocket and registers the server side', async () => { +test('ExecutionDO.fetch upgrades to a WebSocket and registers the server side', async () => { // The actor's fetch path is the subscriber entry point: it must mint a // WebSocketPair, hand the server side to the runtime via acceptWebSocket, // and return a 101 response carrying the client side. Stub the CF-only @@ -142,9 +140,9 @@ test('BroadcastDO.fetch upgrades to a WebSocket and registers the server side', try { const state = new FakeState(); - const actor = new BroadcastDO(state, {}); + const actor = new ExecutionDO(state, {}); - const response = await actor.fetch(new Request('https://broadcast.do/subscribe')); + const response = await actor.fetch(new Request('https://execution.do/broadcast', { headers: { Upgrade: 'websocket' } })); assertEquals(response.status, 101); assertEquals(response.webSocket !== undefined, true); @@ -158,3 +156,25 @@ test('BroadcastDO.fetch upgrades to a WebSocket and registers the server side', } } }); + +test('ExecutionDO coalesces concurrent operations and returns independent responses', async () => { + let calls = 0; + const state = new FakeState(); + state.exports.ExecutionOperationEntrypoint.fetch = async () => { + calls += 1; + return new Response('models refreshed', { status: 202, headers: { 'x-execution': 'done' } }); + }; + const actor = new ExecutionDO(state, {}); + + const first = actor.fetch(new Request('https://execution.do/models/refresh', { method: 'POST' })); + const second = actor.fetch(new Request('https://execution.do/models/refresh', { method: 'POST' })); + + assertEquals((await first).status, 202); + const secondResponse = await second; + assertEquals(await secondResponse.text(), 'models refreshed'); + assertEquals(secondResponse.headers.get('x-execution'), 'done'); + assertEquals(calls, 1); + + await actor.fetch(new Request('https://execution.do/models/refresh', { method: 'POST' })); + assertEquals(calls, 2); +}); diff --git a/apps/platform-cloudflare/entry.ts b/apps/platform-cloudflare/entry.ts index 7ff6960d8..084bd3a01 100644 --- a/apps/platform-cloudflare/entry.ts +++ b/apps/platform-cloudflare/entry.ts @@ -1,18 +1,27 @@ +import { WorkerEntrypoint } from 'cloudflare:workers'; import type { ExecutionContext } from 'hono'; import { bootstrapCloudflarePlatform, type CloudflareEnv } from './src/bootstrap.ts'; import { app, + handleExecutionRequest, initBackgroundSchedulerResolver, initRepo, runScheduledMaintenance, SqlRepo, } from '@floway-dev/gateway'; -// Re-exported here because the CF runtime resolves the DO class by its -// exported name on the Worker module. The wrangler `migrations.new_sqlite_classes` -// entry must match this export. -export { BroadcastDO } from './src/broadcast-do.ts'; +// Re-exported here because the current binding and the rename migration's +// target resolve the class by its Worker-module export name. +export { ExecutionDO } from './src/execution-do.ts'; + +export class ExecutionOperationEntrypoint extends WorkerEntrypoint { + async fetch(request: Request): Promise { + const { db } = bootstrapCloudflarePlatform(this.env); + initRepo(new SqlRepo(db)); + return await handleExecutionRequest(request); + } +} initBackgroundSchedulerResolver(c => promise => c.executionCtx.waitUntil(promise)); diff --git a/apps/platform-cloudflare/src/bootstrap.ts b/apps/platform-cloudflare/src/bootstrap.ts index c2fd1147e..ae89bcee4 100644 --- a/apps/platform-cloudflare/src/bootstrap.ts +++ b/apps/platform-cloudflare/src/bootstrap.ts @@ -1,4 +1,5 @@ -import { DurableObjectChannelBroker, type BroadcastNamespace } from './durable-object-channel-broker.ts'; +import { DurableObjectExecutionCellNamespace, type ExecutionDurableObjectNamespace } from './durable-object-execution-cell.ts'; +import { ExecutionCellChannelBroker } from './execution-cell-channel-broker.ts'; import { createCloudflareExternalResourceFetcher } from './external-resource-fetcher.ts'; import { createCloudflareImageProcessor, type ImagesBinding } from './image-processor.ts'; import { KvImageCacheStore, type KvNamespace } from './kv-image-cache-store.ts'; @@ -6,7 +7,7 @@ import { R2FileStore, type R2BucketLike } from './r2-file-store.ts'; import { cloudflareRuntimeRootCAs } from './runtime-root-cas.ts'; import { cloudflareSocketDial } from './socket-dial.ts'; import { timingSafeEqual } from './timing-safe-equal.ts'; -import { FileDumpStore, initDumpBroker, initDumpStore } from '@floway-dev/gateway'; +import { FileDumpStore, initDumpBroker, initDumpStore, initExecutionCellNamespace } from '@floway-dev/gateway'; import { dumpCodec } from '@floway-dev/gateway/dump-codec'; import type { DumpMetadata } from '@floway-dev/gateway/dump-types'; import { addTrustedRootCAs } from '@floway-dev/http'; @@ -28,16 +29,17 @@ export interface CloudflareEnv { FILES: R2BucketLike; IMAGES: ImagesBinding; KV: KvNamespace; - BROADCAST_DO: BroadcastNamespace; + EXECUTION_DO: ExecutionDurableObjectNamespace; [key: string]: unknown; } // Every binding declared on `CloudflareEnv` is load-bearing — D1 holds all // config and telemetry, R2 stores file-backed response payloads and dump bodies, -// Images re-encodes images, and KV memoises the results. A missing binding means +// Images re-encodes images, KV memoises the results, and EXECUTION_DO hosts +// WebSocket fan-out plus per-use execution cells. A missing binding means // wrangler.jsonc drifted from the code, so we refuse to initialise rather // than 503 on first use of the absent binding. -const REQUIRED_BINDINGS = ['DB', 'FILES', 'IMAGES', 'KV', 'BROADCAST_DO'] as const; +const REQUIRED_BINDINGS = ['DB', 'FILES', 'IMAGES', 'KV', 'EXECUTION_DO'] as const; export const bootstrapCloudflarePlatform = (env: CloudflareEnv): { db: SqlDatabase } => { const missing = REQUIRED_BINDINGS.filter(name => env[name] === undefined); @@ -63,6 +65,8 @@ export const bootstrapCloudflarePlatform = (env: CloudflareEnv): { db: SqlDataba initSocketDial(cloudflareSocketDial); addTrustedRootCAs(cloudflareRuntimeRootCAs); initDumpStore(new FileDumpStore(env.DB, files)); - initDumpBroker(new DurableObjectChannelBroker(env.BROADCAST_DO, dumpCodec)); + const executionCells = new DurableObjectExecutionCellNamespace(env.EXECUTION_DO); + initExecutionCellNamespace(executionCells); + initDumpBroker(new ExecutionCellChannelBroker(executionCells, dumpCodec)); return { db: env.DB }; }; diff --git a/apps/platform-cloudflare/src/broadcast-do.ts b/apps/platform-cloudflare/src/broadcast-do.ts deleted file mode 100644 index 1dd9a9d1c..000000000 --- a/apps/platform-cloudflare/src/broadcast-do.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { DurableObject } from 'cloudflare:workers'; - -// `extends DurableObject` is load-bearing: the CF runtime gates RPC method -// dispatch (`stub.broadcast(...)`, `stub.closeAll(...)`) on the actor -// extending this base class. Without it the runtime rejects the call with -// "the receiving Durable Object does not support RPC" and any caller using -// direct method invocation silently fails. - -export class BroadcastDO extends DurableObject { - // Declared explicitly so the type-check sees `(ctx, env)` even when the - // `cloudflare:workers` types resolve to a parameterless base. - constructor(ctx: DurableObjectState, env: unknown) { - super(ctx, env); - } - - async fetch(_request: Request): Promise { - const pair = new WebSocketPair(); - const client = pair[0]; - const server = pair[1]; - this.ctx.acceptWebSocket(server); - return new Response(null, { status: 101, webSocket: client }); - } - - async broadcast(payload: string): Promise { - for (const ws of this.ctx.getWebSockets()) ws.send(payload); - } - - async closeAll(reason: string): Promise { - for (const ws of this.ctx.getWebSockets()) ws.close(1000, reason); - } - - // Hibernation hooks. With compatibility_date < 2026-04-07 the runtime - // delivers close events only when these hooks are declared on the actor, - // and `webSocketClose` must call `ws.close(code, reason)` to complete the - // close handshake from the actor side — without it the client sees a - // `1006 abnormal closure` and the actor holds the dead socket until the - // hibernation timeout. - async webSocketClose(ws: WebSocket, code: number, reason: string, _wasClean: boolean): Promise { - ws.close(code, reason); - } - async webSocketError(_ws: WebSocket, _err: unknown): Promise {} -} diff --git a/apps/platform-cloudflare/src/cloudflare-workers.d.ts b/apps/platform-cloudflare/src/cloudflare-workers.d.ts index 5b702c202..828dad405 100644 --- a/apps/platform-cloudflare/src/cloudflare-workers.d.ts +++ b/apps/platform-cloudflare/src/cloudflare-workers.d.ts @@ -1,28 +1,36 @@ -// Hand-rolled ambient declaration for the subset of `cloudflare:workers` that -// `BroadcastDO` reaches for at runtime; the workspace intentionally does not +// Hand-rolled ambient declaration for the subset of `cloudflare:workers` used +// by the Cloudflare composition root; the workspace intentionally does not // depend on the full `@cloudflare/workers-types` (sibling files follow the // same pattern for `cloudflare:sockets` and the WebSocket surface). // -// Production code at `apps/platform-cloudflare/src/broadcast-do.ts` +// Production code at `apps/platform-cloudflare/src/execution-do.ts` // does `import { DurableObject } from 'cloudflare:workers'` so the CF runtime -// gates RPC dispatch on the subclass extending this base; the tests resolve +// recognizes the subclass as a Durable Object; tests resolve // the same import through `__tests__/test-utils/cloudflare-workers-stub.ts` // via the alias in `apps/platform-cloudflare/vitest.config.ts`. declare module 'cloudflare:workers' { - // The base class's only role for our actor is to mark the subclass as - // RPC-eligible. The runtime stores `(ctx, env)` on `this` for us; we + // The runtime stores `(ctx, env)` on `this` for us; we // declare them as `protected` so the actor body can read `this.ctx`. export abstract class DurableObject { protected ctx: DurableObjectState; protected env: Env; constructor(ctx: DurableObjectState, env: Env); } + + export abstract class WorkerEntrypoint { + protected env: Env; + } } -// The runtime's `DurableObjectState` surface the actor touches — just the -// WebSocket Hibernation entry points. +// The runtime surface used for hibernatable WebSockets and the loopback +// WorkerEntrypoint that executes database-owning operations outside the DO. interface DurableObjectState { + readonly exports: { + readonly ExecutionOperationEntrypoint: { + fetch(request: Request): Promise; + }; + }; acceptWebSocket(server: WebSocket): void; getWebSockets(): WebSocket[]; } diff --git a/apps/platform-cloudflare/src/durable-object-execution-cell.ts b/apps/platform-cloudflare/src/durable-object-execution-cell.ts new file mode 100644 index 000000000..e326512e4 --- /dev/null +++ b/apps/platform-cloudflare/src/durable-object-execution-cell.ts @@ -0,0 +1,14 @@ +import type { ExecutionCellNamespace } from '@floway-dev/platform'; + +export interface ExecutionDurableObjectNamespace { + idFromName(name: string): unknown; + get(id: unknown): { fetch(request: Request): Promise }; +} + +export class DurableObjectExecutionCellNamespace implements ExecutionCellNamespace { + constructor(private readonly namespace: ExecutionDurableObjectNamespace) {} + + fetch(cellId: string, request: Request): Promise { + return this.namespace.get(this.namespace.idFromName(cellId)).fetch(request); + } +} diff --git a/apps/platform-cloudflare/src/durable-object-channel-broker.ts b/apps/platform-cloudflare/src/execution-cell-channel-broker.ts similarity index 66% rename from apps/platform-cloudflare/src/durable-object-channel-broker.ts rename to apps/platform-cloudflare/src/execution-cell-channel-broker.ts index 18c42c2de..fda372f42 100644 --- a/apps/platform-cloudflare/src/durable-object-channel-broker.ts +++ b/apps/platform-cloudflare/src/execution-cell-channel-broker.ts @@ -1,46 +1,35 @@ -import { iterateReadableStream, type ChannelBroker, type ChannelCodec } from '@floway-dev/platform'; +import { iterateReadableStream, type ChannelBroker, type ChannelCodec, type ExecutionCellNamespace } from '@floway-dev/platform'; -// Minimal namespace surface for BROADCAST_DO — declared locally so this -// file stays off `@cloudflare/workers-types`. -export interface BroadcastNamespace { - idFromName(name: string): unknown; - get(id: unknown): BroadcastStub; -} - -interface BroadcastStub { - broadcast(payload: string): Promise; - closeAll(reason: string): Promise; - fetch(request: Request): Promise; -} - -export class DurableObjectChannelBroker implements ChannelBroker { +export class ExecutionCellChannelBroker implements ChannelBroker { constructor( - private readonly namespace: BroadcastNamespace, + private readonly cells: ExecutionCellNamespace, private readonly codec: ChannelCodec, ) {} - private stub(channelId: string): BroadcastStub { - return this.namespace.get(this.namespace.idFromName(channelId)); - } - async publish(channelId: string, payload: T): Promise { - await this.stub(channelId).broadcast(this.codec.encode(payload)); + const response = await this.cells.fetch(channelId, new Request('https://execution.do/broadcast', { + method: 'POST', + body: this.codec.encode(payload), + })); + if (!response.ok) throw new Error(`ExecutionDO broadcast returned HTTP ${response.status}`); } async closeChannel(channelId: string, reason: string): Promise { - await this.stub(channelId).closeAll(reason); + const response = await this.cells.fetch(channelId, new Request('https://execution.do/broadcast/close', { + method: 'POST', + body: reason, + })); + if (!response.ok) throw new Error(`ExecutionDO close returned HTTP ${response.status}`); } subscribe(channelId: string, signal: AbortSignal): AsyncIterable { - return iterateReadableStream(iterateFromBroadcastSocket(this.stub(channelId), signal, this.codec)); + return iterateReadableStream(iterateFromExecutionSocket(this.cells, channelId, signal, this.codec)); } } -// Listener registration and socket open run eagerly so a broadcast that races -// against the iterator drain still buffers into the queue and lands on the -// next read. -const iterateFromBroadcastSocket = ( - stub: BroadcastStub, +const iterateFromExecutionSocket = ( + cells: ExecutionCellNamespace, + channelId: string, signal: AbortSignal, codec: ChannelCodec, ): ReadableStream => { @@ -64,9 +53,6 @@ const iterateFromBroadcastSocket = ( socket?.removeEventListener('close', onClose); socket?.removeEventListener('error', onError); }; - // Subscriber termination must remove its WebSocket from the Durable - // Object hibernation registry. Waiting for the eager open also covers a - // cancellation or error that arrives while the handshake is in flight. const closeSocket = async (): Promise => { await openPromise.catch(() => {}); socket?.close(1000, 'subscriber done'); @@ -103,7 +89,7 @@ const iterateFromBroadcastSocket = ( } }; const onClose = (): void => close(false); - const onError = (): void => fail(new Error('BroadcastDO socket error')); + const onError = (): void => fail(new Error('ExecutionDO socket error')); const onAbort = (): void => close(); cancel = async (): Promise => { @@ -115,14 +101,14 @@ const iterateFromBroadcastSocket = ( pull = flushError; const openPromise = (async (): Promise => { - const response = await stub.fetch(new Request('https://broadcast.do/subscribe', { + const response = await cells.fetch(channelId, new Request('https://execution.do/broadcast', { headers: { Upgrade: 'websocket' }, })); if (response.status !== 101) { - throw new Error(`BroadcastDO subscribe returned HTTP ${response.status} instead of 101`); + throw new Error(`ExecutionDO subscribe returned HTTP ${response.status} instead of 101`); } const openedSocket = response.webSocket; - if (!openedSocket) throw new Error('BroadcastDO returned 101 without a webSocket'); + if (!openedSocket) throw new Error('ExecutionDO returned 101 without a webSocket'); socket = openedSocket; if (!terminated) { diff --git a/apps/platform-cloudflare/src/execution-do.ts b/apps/platform-cloudflare/src/execution-do.ts new file mode 100644 index 000000000..205a6db08 --- /dev/null +++ b/apps/platform-cloudflare/src/execution-do.ts @@ -0,0 +1,59 @@ +import { DurableObject } from 'cloudflare:workers'; + +import { responseFromExecutionSnapshot, snapshotExecutionResponse, type ExecutionResponseSnapshot } from '@floway-dev/platform'; + +export class ExecutionDO extends DurableObject { + private execution: Promise | null = null; + + constructor(ctx: DurableObjectState, env: unknown) { + super(ctx, env); + } + + async fetch(request: Request): Promise { + const url = new URL(request.url); + if (url.pathname === '/broadcast' && request.method === 'GET') return this.subscribe(request); + if (url.pathname === '/broadcast' && request.method === 'POST') return await this.broadcast(request); + if (url.pathname === '/broadcast/close' && request.method === 'POST') return await this.closeAll(request); + return await this.execute(request); + } + + private subscribe(request: Request): Response { + if (request.headers.get('Upgrade')?.toLowerCase() !== 'websocket') { + return new Response('WebSocket upgrade required', { status: 426 }); + } + const [client, server] = new WebSocketPair(); + this.ctx.acceptWebSocket(server); + return new Response(null, { status: 101, webSocket: client }); + } + + private async broadcast(request: Request): Promise { + const payload = await request.text(); + for (const ws of this.ctx.getWebSockets()) ws.send(payload); + return new Response(null, { status: 204 }); + } + + private async closeAll(request: Request): Promise { + const reason = await request.text(); + for (const ws of this.ctx.getWebSockets()) ws.close(1000, reason); + return new Response(null, { status: 204 }); + } + + private async execute(request: Request): Promise { + if (this.execution === null) { + const operation = this.ctx.exports.ExecutionOperationEntrypoint.fetch(request).then(snapshotExecutionResponse); + this.execution = operation; + void operation.then( + () => { if (this.execution === operation) this.execution = null; }, + () => { if (this.execution === operation) this.execution = null; }, + ); + void operation.catch(error => console.error('ExecutionDO operation failed', error)); + } + return responseFromExecutionSnapshot(await this.execution); + } + + async webSocketClose(ws: WebSocket, code: number, reason: string, _wasClean: boolean): Promise { + ws.close(code, reason); + } + + async webSocketError(_ws: WebSocket, _err: unknown): Promise {} +} diff --git a/apps/platform-node/entry.ts b/apps/platform-node/entry.ts index c9734d88d..a0998a79f 100644 --- a/apps/platform-node/entry.ts +++ b/apps/platform-node/entry.ts @@ -27,13 +27,15 @@ import { bootstrapNodePlatform } from './src/bootstrap.ts'; import { applyMigrations } from './src/migrate.ts'; import { app, + handleExecutionRequest, initBackgroundSchedulerResolver, + initExecutionCellNamespace, initRepo, initResponsesWebSocketUpgradeResolver, runScheduledMaintenance, SqlRepo, } from '@floway-dev/gateway'; -import { getEnvOptional } from '@floway-dev/platform'; +import { getEnvOptional, InProcessExecutionCellNamespace } from '@floway-dev/platform'; // In Node we don't have Workers' executionCtx.waitUntil — there's no request // lifecycle to attach background work to — so the resolver fire-and-forgets @@ -64,6 +66,7 @@ const SCHEDULED_INTERVAL_MS = 60 * 60 * 1000; await applyMigrations(db); initRepo(new SqlRepo(db)); +initExecutionCellNamespace(new InProcessExecutionCellNamespace(handleExecutionRequest)); // Run the scheduled maintenance job once after a short startup delay and // then every hour. Without the startup run, a process that restarts more diff --git a/packages/gateway/__tests__/scheduled/models-refresh_test.ts b/packages/gateway/__tests__/scheduled/models-refresh_test.ts index 828437910..ccfb7bb46 100644 --- a/packages/gateway/__tests__/scheduled/models-refresh_test.ts +++ b/packages/gateway/__tests__/scheduled/models-refresh_test.ts @@ -76,8 +76,9 @@ test('one malformed upstream does not prevent later refreshes from being schedul () => Response.json({ data: [{ id: 'healthy-model' }] }), async () => { await scheduleModelsCacheRefreshes('TEST', promise => { background.push(promise); }); - expect(background).toHaveLength(1); - await background[0]; + expect(background).toHaveLength(2); + const settled = await Promise.allSettled(background); + expect(settled.map(result => result.status)).toEqual(['rejected', 'fulfilled']); }, ); } finally { diff --git a/packages/gateway/__tests__/vitest.setup.ts b/packages/gateway/__tests__/vitest.setup.ts index 5167a765c..fb0828316 100644 --- a/packages/gateway/__tests__/vitest.setup.ts +++ b/packages/gateway/__tests__/vitest.setup.ts @@ -3,8 +3,10 @@ import type { DumpBroker } from '../src/dump/broker.ts'; import { initDumpBroker, initDumpStore } from '../src/dump/registry.ts'; import type { DumpStore } from '../src/dump/store-contract.ts'; import type { DumpMetadata, StoredDumpRecord, DumpRecordId } from '../src/dump/types.ts'; +import { handleExecutionRequest } from '../src/execution/handler.ts'; import { initBackgroundSchedulerResolver } from '../src/runtime/background.ts'; -import { initEnv, initRuntimeKind, initTimingSafeEqual } from '@floway-dev/platform'; +import { initExecutionCellNamespace } from '../src/runtime/execution.ts'; +import { initEnv, initRuntimeKind, initTimingSafeEqual, InProcessExecutionCellNamespace } from '@floway-dev/platform'; // Production always initializes the environment getter at boot. Mirror that // here with a neutral default; tests needing real values (RUNTIME_LOCATION, @@ -16,6 +18,7 @@ initRuntimeKind('node'); initTimingSafeEqual((a, b) => a.every((byte, index) => byte === b[index])); initBackgroundSchedulerResolver(_c => trackBackground); +initExecutionCellNamespace(new InProcessExecutionCellNamespace(handleExecutionRequest)); // Default no-op dump bindings keep tests that do not exercise dump persistence // independent of that subsystem. Dump-specific tests install real or recording diff --git a/packages/gateway/src/execution/handler.ts b/packages/gateway/src/execution/handler.ts new file mode 100644 index 000000000..960862c1f --- /dev/null +++ b/packages/gateway/src/execution/handler.ts @@ -0,0 +1,26 @@ +import { executeModelsRefresh, type ModelsRefreshExecutionInput } from './models-refresh.ts'; + +export const handleExecutionRequest = async (request: Request): Promise => { + const url = new URL(request.url); + if (url.pathname !== '/models/refresh' || request.method !== 'POST') { + return new Response('Execution operation not found', { status: 404 }); + } + const input = parseModelsRefreshInput(await request.json()); + await executeModelsRefresh(input); + return new Response(null, { status: 204 }); +}; + +const parseModelsRefreshInput = (value: unknown): ModelsRefreshExecutionInput => { + if (typeof value !== 'object' || value === null) throw new TypeError('Models refresh execution input must be an object'); + const input = value as Record; + if (typeof input.upstreamId !== 'string' || input.upstreamId === '') throw new TypeError('Models refresh upstreamId must be a non-empty string'); + if (!Number.isSafeInteger(input.configVersion) || (input.configVersion as number) < 1) throw new TypeError('Models refresh configVersion must be a positive integer'); + if (input.cacheEpoch !== null && (!Number.isSafeInteger(input.cacheEpoch) || (input.cacheEpoch as number) < 0)) throw new TypeError('Models refresh cacheEpoch must be a non-negative integer or null'); + if (input.runtimeLocation !== null && typeof input.runtimeLocation !== 'string') throw new TypeError('Models refresh runtimeLocation must be a string or null'); + return { + upstreamId: input.upstreamId, + configVersion: input.configVersion as number, + cacheEpoch: input.cacheEpoch as number | null, + runtimeLocation: input.runtimeLocation as string | null, + }; +}; diff --git a/packages/gateway/src/execution/models-refresh.ts b/packages/gateway/src/execution/models-refresh.ts new file mode 100644 index 000000000..4f39343f0 --- /dev/null +++ b/packages/gateway/src/execution/models-refresh.ts @@ -0,0 +1,50 @@ +import { warmUpstreamModels } from '../data-plane/providers/models-refresh.ts'; +import { createProvider } from '../data-plane/providers/registry.ts'; +import { createPerRequestFetcher } from '../dial/per-request.ts'; +import { getRepo } from '../repo/index.ts'; +import type { StoredUpstreamRecord } from '../repo/types.ts'; +import { getExecutionCellNamespace } from '../runtime/execution.ts'; +import type { BackgroundScheduler } from '@floway-dev/platform'; + +export interface ModelsRefreshExecutionInput { + upstreamId: string; + configVersion: number; + cacheEpoch: number | null; + runtimeLocation: string | null; +} + +export const executeModelsRefresh = async (input: ModelsRefreshExecutionInput): Promise => { + const record = await getRepo().upstreams.getById(input.upstreamId); + if (record === null + || record.configVersion !== input.configVersion + || (record.modelsCache?.fetchedAt ?? null) !== input.cacheEpoch) return; + const fetcherForUpstream = await createPerRequestFetcher(input.runtimeLocation, [record]); + await warmUpstreamModels(createProvider(record), fetcherForUpstream(record.id)); +}; + +export const scheduleModelsRefreshExecution = ( + record: StoredUpstreamRecord, + runtimeLocation: string | null, + scheduler: BackgroundScheduler, +): void => { + const cacheEpoch = record.modelsCache?.fetchedAt ?? null; + const cellId = `models:${record.id}:${record.configVersion}:${cacheEpoch ?? 'cold'}`; + const input: ModelsRefreshExecutionInput = { + upstreamId: record.id, + configVersion: record.configVersion, + cacheEpoch, + runtimeLocation, + }; + const execution = getExecutionCellNamespace().fetch(cellId, new Request('https://execution.floway/models/refresh', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(input), + })).then(async response => { + if (response.ok) return; + throw new Error(`Models refresh execution failed with HTTP ${response.status}: ${await response.text()}`); + }).catch(error => { + console.error(`[models] refresh execution failed for ${record.id}`, error); + throw error; + }); + scheduler(execution); +}; diff --git a/packages/gateway/src/index.ts b/packages/gateway/src/index.ts index 25511037a..ab4b1bb70 100644 --- a/packages/gateway/src/index.ts +++ b/packages/gateway/src/index.ts @@ -4,6 +4,8 @@ export { FileDumpStore } from './repo/dump-store.ts'; export { SqlRepo } from './repo/sql.ts'; export { MODEL_CATALOG_REVISION, modelsCacheGeneration } from './repo/models-cache-contract.ts'; export { initBackgroundSchedulerResolver } from './runtime/background.ts'; +export { initExecutionCellNamespace } from './runtime/execution.ts'; export { initDumpBroker, initDumpStore } from './dump/registry.ts'; export { initResponsesWebSocketUpgradeResolver } from './data-plane/chat/responses/websocket.ts'; export { runScheduledMaintenance } from './scheduled.ts'; +export { handleExecutionRequest } from './execution/handler.ts'; diff --git a/packages/gateway/src/runtime/execution.ts b/packages/gateway/src/runtime/execution.ts new file mode 100644 index 000000000..ca5a5e5b9 --- /dev/null +++ b/packages/gateway/src/runtime/execution.ts @@ -0,0 +1,12 @@ +import type { ExecutionCellNamespace } from '@floway-dev/platform'; + +let executionCells: ExecutionCellNamespace | null = null; + +export const initExecutionCellNamespace = (namespace: ExecutionCellNamespace): void => { + executionCells = namespace; +}; + +export const getExecutionCellNamespace = (): ExecutionCellNamespace => { + if (executionCells === null) throw new Error('Execution cell namespace not initialized'); + return executionCells; +}; diff --git a/packages/gateway/src/scheduled/models-refresh.ts b/packages/gateway/src/scheduled/models-refresh.ts index 65466992d..e4b1ebc37 100644 --- a/packages/gateway/src/scheduled/models-refresh.ts +++ b/packages/gateway/src/scheduled/models-refresh.ts @@ -1,6 +1,4 @@ -import { scheduleUpstreamModelsRefresh } from '../data-plane/providers/models-refresh.ts'; -import { createProvider } from '../data-plane/providers/registry.ts'; -import { createPerRequestFetcher } from '../dial/per-request.ts'; +import { scheduleModelsRefreshExecution } from '../execution/models-refresh.ts'; import { getRepo } from '../repo/index.ts'; import { hasLocationIndependentEgress } from '../repo/proxy-fallback-list.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; @@ -8,11 +6,9 @@ import type { BackgroundScheduler } from '@floway-dev/platform'; export const scheduleModelsCacheRefreshes = async (runtimeLocation: string | null, scheduler: BackgroundScheduler): Promise => { const upstreams = (await getRepo().upstreams.list()).filter(upstream => upstream.enabled && (runtimeLocation !== null || hasLocationIndependentEgress(upstream.proxyFallbackList))); - const fetcherForUpstream = await createPerRequestFetcher(runtimeLocation, upstreams); - for (const upstream of upstreams) { try { - scheduleUpstreamModelsRefresh(createProvider(upstream), scheduler, fetcherForUpstream(upstream.id)); + scheduleModelsRefreshExecution(upstream, runtimeLocation, scheduler); } catch (error) { console.error(`[scheduled] models.refresh failed for ${upstream.id}`, error); } diff --git a/packages/platform/src/execution-cell.ts b/packages/platform/src/execution-cell.ts new file mode 100644 index 000000000..588fd5aad --- /dev/null +++ b/packages/platform/src/execution-cell.ts @@ -0,0 +1,45 @@ +// Addressable request/response execution cell. The platform decides how a +// cell id maps to an isolated owner (Durable Object, process-local actor, or +// worker); operation protocols remain in their owning package. +export interface ExecutionCellNamespace { + fetch(cellId: string, request: Request): Promise; +} + +export class InProcessExecutionCellNamespace implements ExecutionCellNamespace { + private readonly executions = new Map>(); + + constructor(private readonly handler: (request: Request) => Promise) {} + + async fetch(cellId: string, request: Request): Promise { + let execution = this.executions.get(cellId); + if (execution === undefined) { + execution = this.handler(request).then(snapshotExecutionResponse); + this.executions.set(cellId, execution); + void execution.then( + () => this.executions.delete(cellId), + () => this.executions.delete(cellId), + ); + } + return responseFromExecutionSnapshot(await execution); + } +} + +export interface ExecutionResponseSnapshot { + status: number; + statusText: string; + headers: [string, string][]; + body: ArrayBuffer | null; +} + +export const snapshotExecutionResponse = async (response: Response): Promise => ({ + status: response.status, + statusText: response.statusText, + headers: [...response.headers], + body: response.body === null ? null : await response.arrayBuffer(), +}); + +export const responseFromExecutionSnapshot = (snapshot: ExecutionResponseSnapshot): Response => new Response(snapshot.body?.slice(0) ?? null, { + status: snapshot.status, + statusText: snapshot.statusText, + headers: snapshot.headers, +}); diff --git a/packages/platform/src/index.ts b/packages/platform/src/index.ts index 48ec128e3..1fd609f46 100644 --- a/packages/platform/src/index.ts +++ b/packages/platform/src/index.ts @@ -1,6 +1,7 @@ export * from './background.ts'; export * from './channel-broker.ts'; export * from './env.ts'; +export * from './execution-cell.ts'; export * from './external-resource-fetcher.ts'; export * from './file-store.ts'; export * from './image-cache-store.ts'; diff --git a/wrangler.example.jsonc b/wrangler.example.jsonc index 5997a59e7..e8569fec7 100644 --- a/wrangler.example.jsonc +++ b/wrangler.example.jsonc @@ -32,7 +32,7 @@ // `@reclaimprotocol/tls/webcrypto` (used by packages/http and the proxy // REALITY dialer) still imports `webcrypto` from `crypto`, so the Worker // cannot resolve those modules at cold start without the Node compat flag. - "compatibility_flags": ["nodejs_compat"], + "compatibility_flags": ["nodejs_compat", "enable_ctx_exports"], "triggers": { "crons": ["17 * * * *"] }, @@ -108,13 +108,14 @@ "id": "" } ], - // Per-key WebSocket fan-out actor. Content-agnostic — the actor never - // inspects the payload; callers layer their own framing via a codec. + // Per-use execution cells. The initial broadcast protocol provides + // hibernatable WebSocket fan-out; model refresh operations share this + // actor host without placing provider data in Durable Object storage. "durable_objects": { "bindings": [ { - "name": "BROADCAST_DO", - "class_name": "BroadcastDO" + "name": "EXECUTION_DO", + "class_name": "ExecutionDO" } ] }, @@ -125,13 +126,22 @@ // alternative `new_classes` picks the legacy KV-backed storage path which // CF is phasing out and which Workers Free cannot create. // - // BroadcastDO has no `ctx.storage.*` calls — the SQLite db is provisioned + // ExecutionDO has no `ctx.storage.*` calls — the SQLite db is provisioned // but unused. The name reflects the BACKEND choice, not whether the actor // touches it. "migrations": [ { "tag": "v1", "new_sqlite_classes": ["BroadcastDO"] + }, + { + "tag": "v2", + "renamed_classes": [ + { + "from": "BroadcastDO", + "to": "ExecutionDO" + } + ] } ] } From 41c9bdbca9755ab3c9d68e9ea3918aef201cea80 Mon Sep 17 00:00:00 2001 From: Menci Date: Fri, 7 Aug 2026 04:34:41 +0800 Subject: [PATCH 45/46] refactor(gateway): move model refresh ownership into execution cells Route explicit listing, save-time warmup, stale reads, and scheduled refreshes through the platform execution-cell namespace. Remove the isolate coordinator and SQLite claim/lease/token state; D1 now retains only retry backoff and config-version publication fencing. Replace ownership tests with execution coalescing, backoff, and generation-fence coverage. --- .../__tests__/node-sqlite-repo_test.ts | 6 +- .../data-transfer/routes_test.ts | 5 +- .../upstreams/copilot-device-login_test.ts | 8 +- .../data-plane/providers/catalog_test.ts | 21 +- .../data-plane/providers/models-cache_test.ts | 557 ++++-------------- .../data-plane/providers/resolution_test.ts | 39 +- .../shared/listing/addressable_test.ts | 10 +- packages/gateway/__tests__/repo/memory.ts | 51 +- .../__tests__/repo/models-cache-fixture.ts | 10 +- .../__tests__/repo/models-refresh_test.ts | 268 +++------ packages/gateway/__tests__/repo/sql_test.ts | 2 +- packages/gateway/__tests__/test-utils/app.ts | 14 +- .../0080_simplify_models_refresh.sql | 12 + .../src/control-plane/models/routes.ts | 7 +- .../shared/save-upstream-for-models.ts | 8 +- .../src/control-plane/upstreams/models.ts | 31 +- .../gateway/src/data-plane/codex/models.ts | 3 +- .../gateway/src/data-plane/models/gemini.ts | 9 +- .../gateway/src/data-plane/models/http.ts | 7 +- .../gateway/src/data-plane/models/load.ts | 5 +- .../src/data-plane/providers/catalog.ts | 6 +- .../src/data-plane/providers/models-cache.ts | 15 +- .../data-plane/providers/models-refresh.ts | 224 ------- .../src/data-plane/providers/resolution.ts | 10 +- .../data-plane/shared/listing/addressable.ts | 5 +- packages/gateway/src/execution/handler.ts | 13 +- .../gateway/src/execution/models-refresh.ts | 138 ++++- .../gateway/src/repo/models-cache-contract.ts | 4 +- .../src/repo/models-refresh-backoff.ts | 4 + .../src/repo/models-refresh-contract.ts | 10 - packages/gateway/src/repo/sql.ts | 102 +--- packages/gateway/src/repo/types.ts | 31 +- .../gateway/src/scheduled/models-refresh.ts | 4 +- 33 files changed, 451 insertions(+), 1188 deletions(-) create mode 100644 packages/gateway/migrations/0080_simplify_models_refresh.sql delete mode 100644 packages/gateway/src/data-plane/providers/models-refresh.ts create mode 100644 packages/gateway/src/repo/models-refresh-backoff.ts delete mode 100644 packages/gateway/src/repo/models-refresh-contract.ts diff --git a/apps/platform-node/__tests__/node-sqlite-repo_test.ts b/apps/platform-node/__tests__/node-sqlite-repo_test.ts index 4120adfaf..a4a5d2f8e 100644 --- a/apps/platform-node/__tests__/node-sqlite-repo_test.ts +++ b/apps/platform-node/__tests__/node-sqlite-repo_test.ts @@ -98,13 +98,9 @@ test('repository JSON codecs round-trip upstream, alias, and Responses state thr const storedUpstream = await repo.upstreams.getById(upstreamRecord.id); if (storedUpstream === null) throw new Error('expected stored upstream fixture'); const cacheGeneration = modelsCacheGeneration(storedUpstream); - const cacheToken = 'node-cache-fixture'; - const cacheClaim = await repo.upstreams.claimModelsRefresh({ id: 'up_node', generation: cacheGeneration, token: cacheToken, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, bypassBackoff: true, observedActiveToken: null }); - if (cacheClaim.kind !== 'claimed') throw new Error('expected model-cache fixture claim'); - await repo.upstreams.finalizeModelsRefreshSuccess({ + await repo.upstreams.publishModelsRefresh({ id: 'up_node', generation: cacheGeneration, - token: cacheToken, cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_786_000_000_000, diff --git a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts index 3241397ba..a84b24fbe 100644 --- a/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/data-transfer/routes_test.ts @@ -7,8 +7,9 @@ import { expect, test, vi } from 'vitest'; // until the vitest timeout. Stub the cache layer to a no-op so the import // path's own behavior (upserts, identity validation, etc.) is what the tests // exercise — the warm itself has dedicated coverage in models-cache_test.ts. -vi.mock('../../../src/data-plane/providers/models-refresh.ts', () => ({ - warmUpstreamModels: () => Promise.resolve([]), +vi.mock('../../../src/execution/models-refresh.ts', async importOriginal => ({ + ...await importOriginal(), + refreshModels: () => Promise.resolve({ kind: 'refreshed' }), })); import { exportData, importData } from '../../../src/control-plane/data-transfer/routes.ts'; diff --git a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts index e2c7d6d76..50d0c17a4 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts @@ -5,14 +5,14 @@ import { afterEach, expect, test, vi } from 'vitest'; // exchange and persistence. const modelsCacheMock = vi.hoisted<{ calls: number; error: Error | null; pending: Promise | null }>(() => ({ calls: 0, error: null, pending: null })); -vi.mock('../../../src/data-plane/providers/models-refresh.ts', () => ({ - warmUpstreamModels: async () => { +vi.mock('../../../src/execution/models-refresh.ts', async importOriginal => ({ + ...await importOriginal(), + refreshModels: async () => { modelsCacheMock.calls++; if (modelsCacheMock.pending) await modelsCacheMock.pending; if (modelsCacheMock.error) throw modelsCacheMock.error; - return []; + return { kind: 'refreshed' }; }, - clearModelsRefreshesForTesting: () => {}, })); import { seedModelsCache, storedModelsCacheGeneration } from '../../repo/models-cache-fixture.ts'; diff --git a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts index b6dbaade0..af89693fa 100644 --- a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts @@ -1,7 +1,6 @@ import { describe, expect, test, vi } from 'vitest'; import { compareModelIds, getModelsFromProviders } from '../../../src/data-plane/providers/catalog.ts'; -import { clearModelsRefreshesForTesting } from '../../../src/data-plane/providers/models-refresh.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; import { buildCustomUpstreamRecord, copilotModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; @@ -146,7 +145,7 @@ test('catalog assembly returns the merged catalog plus the per-id upstream index throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const { models, upstreamsByPublicId } = await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler); + const { models, upstreamsByPublicId } = await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST'); const model = models.find(candidate => candidate.id === 'shared-model'); assertEquals(model?.display_name, 'Shared Model'); @@ -234,7 +233,7 @@ test('disabledPublicModelIds hides models from the catalog and routing, per upst })); await warmModelsForTest(); - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; + const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; assertEquals([...catalog.map(m => m.id)].sort(), ['gpt-keep', 'gpt-shared']); // The solo and override ids resolve to nothing (hidden + unroutable). @@ -254,7 +253,6 @@ test('disabledPublicModelIds hides models from the catalog and routing, per upst // directly observes concurrency without a wall-clock threshold that load can // satisfy or violate independently of execution order. test('catalog refresh triggers fan out per upstream in parallel', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); @@ -291,7 +289,7 @@ test('catalog refresh triggers fan out per upstream in parallel', async () => { await vi.waitFor(() => expect(started.toSorted()).toEqual(upstreams.map(upstream => upstream.host).toSorted())); for (const release of releases.values()) release(); await warming; - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; + const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; assertEquals([...catalog.map(m => m.id)].sort(), ['p1-model', 'p2-model', 'p3-model']); }, @@ -302,7 +300,6 @@ test('catalog refresh triggers fan out per upstream in parallel', async () => { // recorded against `sawSuccess === true`; the public catalog still includes // every successful upstream's models. test('catalog assembly: a rejected provider does not block other providers', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); @@ -340,7 +337,7 @@ test('catalog assembly: a rejected provider does not block other providers', asy throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; + const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; assertEquals([...catalog.map(m => m.id)].sort(), ['ok-1-model', 'ok-2-model']); }, ); @@ -368,7 +365,7 @@ describe('catalog listing under modelPrefix', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; + const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; assertEquals(catalog.map(m => m.id), ['gpt-4o']); }, ); @@ -393,7 +390,7 @@ describe('catalog listing under modelPrefix', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; + const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; assertEquals(catalog.map(m => m.id), ['or/gpt-4o']); // Prefixed surface gets a synthesized display_name prepending the // upstream's display name so the dashboard tells the operator at a @@ -440,7 +437,7 @@ describe('catalog listing under modelPrefix', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; + const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; assertEquals(catalog.map(m => m.id), ['or/gpt-4o']); const bare = await enumerateModelCandidates({ upstreamIds: null, model: 'gpt-4o', kind: 'chat', scheduler: testScheduler, runtimeLocation: 'TEST' }); @@ -488,7 +485,7 @@ describe('catalog listing under modelPrefix', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; + const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; assertEquals([...catalog.map(m => m.id)].sort(), ['gpt-4o', 'or/gpt-4o']); // Both upstreams enumerate against the bare id: up_plain via its only @@ -580,7 +577,7 @@ describe('catalog listing under modelPrefix', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler)).models; + const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; assertEquals([...catalog.map(m => m.id)].sort(), ['gpt-mini', 'or/gpt-mini']); }, ); diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index f235f0a44..111c02422 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -1,78 +1,23 @@ -import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { expect, test, vi } from 'vitest'; import { readUpstreamModelsSnapshotAndScheduleRefresh, MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; -import { clearModelsRefreshesForTesting, fetchUpstreamModels, warmUpstreamModels } from '../../../src/data-plane/providers/models-refresh.ts'; -import type { GatewayProvider } from '../../../src/data-plane/providers/registry.ts'; -import { initRepo } from '../../../src/repo/index.ts'; +import { createProvider } from '../../../src/data-plane/providers/registry.ts'; +import { modelsRefreshTarget, refreshModels } from '../../../src/execution/models-refresh.ts'; import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; -import { SqlRepo } from '../../../src/repo/sql.ts'; -import type { ModelsCacheGeneration } from '../../../src/repo/types.ts'; -import { InMemoryRepo } from '../../repo/memory.ts'; import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; -import { createSqliteTestDb } from '../../repo/test-sqlite.ts'; -import { directFetcher, type ProviderModel, type UpstreamModelsCache } from '@floway-dev/provider'; -import { stubProvider, stubProviderModel } from '@floway-dev/test-utils'; - -const UPSTREAM_ID = 'up_a'; -const CACHE_CONFIG = { identity: 'old' }; -const CACHE_GENERATION: ModelsCacheGeneration = { - configVersion: 1, -}; - -const aModel = (id: string): ProviderModel => stubProviderModel({ id }); - -const stubInstance = ( - fetchFn: () => Promise, - modelsCache: UpstreamModelsCache | null = null, - generation: ModelsCacheGeneration = CACHE_GENERATION, -): GatewayProvider => ({ - upstreamId: UPSTREAM_ID, - kind: 'custom', - name: UPSTREAM_ID, - inboundHeaderAllowlist: [], - disabledPublicModelIds: [], - modelPrefix: null, - modelsCache, - instance: stubProvider({ getProvidedModels: fetchFn }), - modelsCacheGeneration: generation, -}); - -const setupRepo = async (): Promise => { - const repo = new InMemoryRepo(); - initRepo(repo); - await repo.upstreams.save({ - id: UPSTREAM_ID, - kind: 'custom', - name: 'Upstream A', - enabled: true, - sortOrder: 0, - createdAt: '2026-08-01T00:00:00.000Z', - updatedAt: '2026-08-01T00:00:00.000Z', - config: CACHE_CONFIG, - state: null, - modelsCache: null, - flagOverrides: {}, - disabledPublicModelIds: [], - proxyFallbackList: [], - modelPrefix: null, - hue: 210, - }); - return repo; -}; - -const seedCache = async ( - repo: InMemoryRepo, - cache: { revision: number; fetchedAt: number; models: ProviderModel[] }, -): Promise => { - await seedModelsCache(repo.upstreams, UPSTREAM_ID, CACHE_GENERATION, cache); - const stored = (await repo.upstreams.getById(UPSTREAM_ID))?.modelsCache; - if (!stored) throw new Error('the seeded catalog did not land on the upstream row'); - return stored; +import { buildCustomUpstreamRecord, setupAppTest } from '../../test-utils/app.ts'; +import { ProviderModelsUnavailableError } from '@floway-dev/provider'; +import { jsonResponse, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; + +const setupCustom = async () => { + const { repo } = await setupAppTest(); + await repo.upstreams.deleteAll(); + await repo.upstreams.save(buildCustomUpstreamRecord()); + const record = await repo.upstreams.getById('up_custom'); + if (record === null) throw new Error('custom upstream missing'); + return { repo, record }; }; -const storedCache = async (repo: InMemoryRepo): Promise => - (await repo.upstreams.getById(UPSTREAM_ID))?.modelsCache ?? null; - const captureScheduled = () => { const promises: Promise[] = []; return { @@ -81,406 +26,108 @@ const captureScheduled = () => { }; }; -beforeEach(() => { - vi.restoreAllMocks(); - clearModelsRefreshesForTesting(); -}); - -describe('readUpstreamModelsSnapshotAndScheduleRefresh', () => { - test('cold cache returns immediately and refreshes in the background', async () => { - const repo = await setupRepo(); - let resolveFetch: ((models: ProviderModel[]) => void) | null = null; - const fetchFn = vi.fn(() => new Promise(resolve => { resolveFetch = resolve; })); - const scheduled = captureScheduled(); - - const result = readUpstreamModelsSnapshotAndScheduleRefresh( - stubInstance(fetchFn), - { scheduler: scheduled.scheduler, fetcher: directFetcher }, - ); - - expect(result.models).toEqual([]); - expect(scheduled.promises).toHaveLength(1); - await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); - resolveFetch!([aModel('m1')]); - await scheduled.promises[0]; - expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['m1']); - }); - - test('within SOFT returns the stored catalog without scheduling a refresh', async () => { - const repo = await setupRepo(); - const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 1000, models: [aModel('cached')] }); - const fetchFn = vi.fn(async () => [aModel('fresh')]); - const scheduled = captureScheduled(); - - const result = readUpstreamModelsSnapshotAndScheduleRefresh( - stubInstance(fetchFn, cache), - { scheduler: scheduled.scheduler, fetcher: directFetcher }, - ); - - expect(result.models.map(model => model.id)).toEqual(['cached']); - expect(scheduled.promises).toEqual([]); - expect(fetchFn).not.toHaveBeenCalled(); - }); - - test('snapshots remain usable regardless of age while refresh runs separately', async () => { - const repo = await setupRepo(); - const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 365 * 24 * 60 * 60_000, models: [aModel('stale')] }); - let resolveFetch: ((models: ProviderModel[]) => void) | null = null; - const fetchFn = vi.fn(() => new Promise(resolve => { resolveFetch = resolve; })); - const scheduled = captureScheduled(); - - const result = readUpstreamModelsSnapshotAndScheduleRefresh( - stubInstance(fetchFn, cache), - { scheduler: scheduled.scheduler, fetcher: directFetcher }, - ); - - expect(result.models.map(model => model.id)).toEqual(['stale']); - await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); - resolveFetch!([aModel('fresh')]); - await scheduled.promises[0]; - expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['fresh']); - }); - - test('explicit fetch blocks for a fresh result', async () => { - const repo = await setupRepo(); - const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 1000, models: [aModel('stored')] }); - const fetchFn = vi.fn(async () => [aModel('fresh')]); - const result = await fetchUpstreamModels(stubInstance(fetchFn, cache), directFetcher); - - expect(result.map(model => model.id)).toEqual(['fresh']); - expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['fresh']); - }); - - test('concurrent cold callers join one background refresh', async () => { - await setupRepo(); - let resolveFetch: ((models: ProviderModel[]) => void) | null = null; - const fetchFn = vi.fn(() => new Promise(resolve => { resolveFetch = resolve; })); - const instance = stubInstance(fetchFn); - const scheduled = captureScheduled(); - - const [first, second] = [ - readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: scheduled.scheduler, fetcher: directFetcher }), - readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: scheduled.scheduler, fetcher: directFetcher }), - ]; - - expect(first.models).toEqual([]); - expect(second.models).toEqual([]); - await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); - resolveFetch!([aModel('m1')]); - await Promise.all(scheduled.promises); - }); - - test('a failed refresh preserves stale data and activates persistent backoff', async () => { - const repo = await setupRepo(); - const now = 1_800_000_000_000; - vi.spyOn(Date, 'now').mockReturnValue(now); - const cache = await seedCache(repo, { revision: MODEL_CATALOG_REVISION, fetchedAt: now - 20 * 60_000, models: [aModel('stale')] }); - const fetchFn = vi.fn(async () => { throw new Error('boom'); }); - const instance = stubInstance(fetchFn, cache); - const firstScheduled = captureScheduled(); - - expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: firstScheduled.scheduler, fetcher: directFetcher }).models.map(model => model.id)).toEqual(['stale']); - await expect(firstScheduled.promises[0]).rejects.toThrow('boom'); - - clearModelsRefreshesForTesting(); - const secondScheduled = captureScheduled(); - expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: secondScheduled.scheduler, fetcher: directFetcher }).models.map(model => model.id)).toEqual(['stale']); - await expect(secondScheduled.promises[0]).resolves.toBeUndefined(); - expect(fetchFn).toHaveBeenCalledTimes(1); - expect((await storedCache(repo))?.lastError?.message).toContain('boom'); - }); - - test('cold failures return empty and retry after the persisted backoff expires', async () => { - const repo = await setupRepo(); - let now = 1_800_000_000_000; - vi.spyOn(Date, 'now').mockImplementation(() => now); - const fetchFn = vi.fn(async () => { throw new Error('boom'); }); - const instance = stubInstance(fetchFn); - - const firstScheduled = captureScheduled(); - expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: firstScheduled.scheduler, fetcher: directFetcher }).models).toEqual([]); - await expect(firstScheduled.promises[0]).rejects.toThrow('boom'); - expect(await storedCache(repo)).toMatchObject({ fetchedAt: 0, models: [], lastError: { message: 'boom' } }); - - clearModelsRefreshesForTesting(); - now += 59_999; - const backedOff = captureScheduled(); - expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: backedOff.scheduler, fetcher: directFetcher }).models).toEqual([]); - await expect(backedOff.promises[0]).resolves.toBeUndefined(); - expect(fetchFn).toHaveBeenCalledTimes(1); - - clearModelsRefreshesForTesting(); - now += 1; - const retry = captureScheduled(); - expect(readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler: retry.scheduler, fetcher: directFetcher }).models).toEqual([]); - await expect(retry.promises[0]).rejects.toThrow('boom'); - expect(fetchFn).toHaveBeenCalledTimes(2); - }); - - test('synchronous warm respects backoff while explicit fetch bypasses it', async () => { - const repo = await setupRepo(); - const now = 1_800_000_000_000; - vi.spyOn(Date, 'now').mockReturnValue(now); - const failing = stubInstance(async () => { throw new Error('boom'); }); - const scheduled = captureScheduled(); - readUpstreamModelsSnapshotAndScheduleRefresh(failing, { scheduler: scheduled.scheduler, fetcher: directFetcher }); - await expect(scheduled.promises[0]).rejects.toThrow('boom'); - clearModelsRefreshesForTesting(); - - const fetchFn = vi.fn(async () => [aModel('recovered')]); - const cache = await storedCache(repo); - const warming = stubInstance(fetchFn, cache); - await expect(warmUpstreamModels(warming, directFetcher)).resolves.toBeUndefined(); - expect(fetchFn).not.toHaveBeenCalled(); - - await expect(fetchUpstreamModels(warming, directFetcher)) - .resolves.toEqual([aModel('recovered')]); - expect(fetchFn).toHaveBeenCalledTimes(1); - }); - - test('synchronous warm waits for a refresh owned by another runtime', async () => { - const repo = await setupRepo(); - const now = Date.now(); - await expect(repo.upstreams.claimModelsRefresh({ id: UPSTREAM_ID, generation: CACHE_GENERATION, token: 'remote-owner', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null })) - .resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - const localFetch = vi.fn(async () => [aModel('duplicate-local-model')]); - const warming = warmUpstreamModels(stubInstance(localFetch), directFetcher); - - let settled = false; - void warming.finally(() => { settled = true; }); - await new Promise(resolve => setTimeout(resolve, 20)); - expect(settled).toBe(false); - - await repo.upstreams.finalizeModelsRefreshSuccess({ - id: UPSTREAM_ID, - generation: CACHE_GENERATION, - token: 'remote-owner', - cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [aModel('remote-model')] }, - }); - - await warming; - expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['remote-model']); - expect(localFetch).not.toHaveBeenCalled(); +test('a fresh snapshot returns without scheduling work', async () => { + const { repo, record } = await setupCustom(); + await seedModelsCache(repo.upstreams, record.id, modelsCacheGeneration(record), { + revision: MODEL_CATALOG_REVISION, + fetchedAt: Date.now(), + models: [stubProviderModel({ id: 'cached' })], }); + const cached = await repo.upstreams.getById(record.id); + if (cached === null) throw new Error('cached upstream missing'); + const scheduled = captureScheduled(); - test('explicit fetch follows the durable owner already awaited by a local warm', async () => { - const repo = await setupRepo(); - const now = Date.now(); - await repo.upstreams.claimModelsRefresh({ id: UPSTREAM_ID, generation: CACHE_GENERATION, token: 'remote-owner', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); - const fetchFn = vi.fn(async () => [aModel('duplicate-local-model')]); - const instance = stubInstance(fetchFn); - const warming = warmUpstreamModels(instance, directFetcher); - await new Promise(resolve => setTimeout(resolve, 20)); - - const explicit = fetchUpstreamModels(instance, directFetcher); - await new Promise(resolve => setTimeout(resolve, 20)); - expect(fetchFn).not.toHaveBeenCalled(); - await repo.upstreams.finalizeModelsRefreshSuccess({ - id: UPSTREAM_ID, - generation: CACHE_GENERATION, - token: 'remote-owner', - cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [aModel('remote-model')] }, - }); - expect((await explicit).map(model => model.id)).toEqual(['remote-model']); - await warming; - expect(instance.modelsCache?.models.map(model => model.id)).toEqual(['remote-model']); + const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(createProvider(cached), { + scheduler: scheduled.scheduler, + runtimeLocation: 'TEST', }); - test('explicit fetch retries after a remote owner records failure', async () => { - const repo = await setupRepo(); - const now = Date.now(); - await repo.upstreams.claimModelsRefresh({ id: UPSTREAM_ID, generation: CACHE_GENERATION, token: 'remote-owner', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); - const fetchFn = vi.fn(async () => [aModel('explicit-recovery-model')]); - const explicit = fetchUpstreamModels(stubInstance(fetchFn), directFetcher); - await new Promise(resolve => setTimeout(resolve, 20)); - - await repo.upstreams.finalizeModelsRefreshFailure({ - id: UPSTREAM_ID, - generation: CACHE_GENERATION, - token: 'remote-owner', - error: { message: 'remote failure', at: now + 1 }, - previousFailureCount: 0, - failedAt: now + 1, - }); - - await expect(explicit).resolves.toEqual([aModel('explicit-recovery-model')]); - expect(fetchFn).toHaveBeenCalledOnce(); - }); - - test('explicit fetch joins a warm that already owns the durable refresh', async () => { - await setupRepo(); - let resolveWarm: ((models: ProviderModel[]) => void) | null = null; - const fetchFn = vi.fn(() => new Promise(resolve => { resolveWarm = resolve; })); - const instance = stubInstance(fetchFn); - const warming = warmUpstreamModels(instance, directFetcher); - await vi.waitFor(() => expect(fetchFn).toHaveBeenCalledTimes(1)); - const explicit = fetchUpstreamModels(instance, directFetcher); - resolveWarm!([aModel('warm-owner-model')]); - expect((await explicit).map(model => model.id)).toEqual(['warm-owner-model']); - await warming; - expect(instance.modelsCache?.models.map(model => model.id)).toEqual(['warm-owner-model']); - expect(fetchFn).toHaveBeenCalledTimes(1); - }); - - test('a transient success-finalize error is retried without installing failure backoff', async () => { - const repo = await setupRepo(); - const finalizeFailure = vi.spyOn(repo.upstreams, 'finalizeModelsRefreshFailure'); - const finalizeSuccess = vi.spyOn(repo.upstreams, 'finalizeModelsRefreshSuccess').mockRejectedValueOnce(new Error('finalize failed')); - const instance = stubInstance(async () => [aModel('published-model')]); - - await expect(fetchUpstreamModels(instance, directFetcher)).resolves.toEqual([aModel('published-model')]); - expect(finalizeSuccess).toHaveBeenCalledTimes(2); - expect(finalizeFailure).not.toHaveBeenCalled(); - expect((await storedCache(repo))?.models).toEqual([aModel('published-model')]); - }); - - test('persistent finalize errors release the claim for a later refresh', async () => { - const repo = await setupRepo(); - vi.spyOn(repo.upstreams, 'finalizeModelsRefreshSuccess').mockRejectedValue(new Error('storage unavailable')); - - await expect(fetchUpstreamModels(stubInstance(async () => [aModel('unpublished')]), directFetcher)) - .rejects.toThrow('Failed to finalize models refresh'); - await expect(repo.upstreams.claimModelsRefresh({ - id: UPSTREAM_ID, - generation: CACHE_GENERATION, - token: 'next-owner', - now: Date.now(), - staleClaimedBefore: Number.MIN_SAFE_INTEGER, - bypassBackoff: false, - observedActiveToken: null, - })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - }); - - test('a superseded generation neither joins nor overwrites the current catalog', async () => { - const repo = await setupRepo(); - let resolveOld: ((models: ProviderModel[]) => void) | null = null; - const oldFetch = vi.fn(() => new Promise(resolve => { resolveOld = resolve; })); - const oldScheduled = captureScheduled(); - readUpstreamModelsSnapshotAndScheduleRefresh( - stubInstance(oldFetch), - { scheduler: oldScheduled.scheduler, fetcher: directFetcher }, - ); - await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); - - const nextConfig = { identity: 'new' }; - const nextGeneration = { configVersion: CACHE_GENERATION.configVersion + 1 }; - const current = await repo.upstreams.getById(UPSTREAM_ID); - if (!current) throw new Error('upstream row missing'); - await repo.upstreams.replaceForModels({ previous: current, upstream: { ...current, config: nextConfig } }); - const newFetch = vi.fn(async () => [aModel('new-tenant-model')]); - const newResult = await fetchUpstreamModels(stubInstance(newFetch, null, nextGeneration), directFetcher); + expect(snapshot.models.map(model => model.id)).toEqual(['cached']); + expect(scheduled.promises).toEqual([]); +}); - expect(newResult.map(model => model.id)).toEqual(['new-tenant-model']); - resolveOld!([aModel('old-tenant-model')]); - await oldScheduled.promises[0]; - expect(oldFetch).toHaveBeenCalledTimes(1); - expect(newFetch).toHaveBeenCalledTimes(1); - expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['new-tenant-model']); - }); +test('a stale snapshot returns immediately and refreshes through the execution cell', async () => { + const { repo, record } = await setupCustom(); + await seedModelsCache(repo.upstreams, record.id, modelsCacheGeneration(record), { + revision: MODEL_CATALOG_REVISION, + fetchedAt: Date.now() - 11 * 60_000, + models: [stubProviderModel({ id: 'stale' })], + }); + const stale = await repo.upstreams.getById(record.id); + if (stale === null) throw new Error('stale upstream missing'); + const scheduled = captureScheduled(); + + await withMockedFetch( + () => jsonResponse({ object: 'list', data: [{ id: 'fresh' }] }), + async () => { + const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(createProvider(stale), { + scheduler: scheduled.scheduler, + runtimeLocation: 'TEST', + }); + expect(snapshot.models.map(model => model.id)).toEqual(['stale']); + await Promise.all(scheduled.promises); + }, + ); + + expect((await repo.upstreams.getById(record.id))?.modelsCache?.models.map(model => model.id)).toEqual(['fresh']); +}); - test('explicit fetch joins an older background refresh instead of preempting it', async () => { - const repo = await setupRepo(); - let resolveOld: ((models: ProviderModel[]) => void) | null = null; - const oldFetch = vi.fn(() => new Promise(resolve => { resolveOld = resolve; })); - const oldScheduled = captureScheduled(); - readUpstreamModelsSnapshotAndScheduleRefresh( - stubInstance(oldFetch), - { scheduler: oldScheduled.scheduler, fetcher: directFetcher }, - ); - await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); +test('concurrent callers share one upstream fetch', async () => { + const { record } = await setupCustom(); + let release: ((response: Response) => void) | undefined; + const fetch = vi.fn(() => new Promise(resolve => { release = resolve; })); - const explicitFetch = vi.fn(async () => [aModel('duplicate-explicit-model')]); - const explicit = fetchUpstreamModels(stubInstance(explicitFetch), directFetcher); - resolveOld!([aModel('late-old-model')]); - expect((await explicit).map(model => model.id)).toEqual(['late-old-model']); - await oldScheduled.promises[0]; - expect(explicitFetch).not.toHaveBeenCalled(); - expect((await storedCache(repo))?.models.map(model => model.id)).toEqual(['late-old-model']); + await withMockedFetch(fetch, async () => { + const target = modelsRefreshTarget(record); + const first = refreshModels(target, 'TEST', { bypassBackoff: true, includeDiscovered: false }); + const second = refreshModels(target, 'TEST', { bypassBackoff: true, includeDiscovered: false }); + await vi.waitFor(() => expect(fetch).toHaveBeenCalledTimes(1)); + release!(jsonResponse({ object: 'list', data: [{ id: 'shared' }] })); + await expect(Promise.all([first, second])).resolves.toEqual([{ kind: 'refreshed' }, { kind: 'refreshed' }]); }); +}); - test('explicit fetch retries with its own transport after an older background owner fails', async () => { - const repo = await setupRepo(); - let rejectOld: ((error: Error) => void) | null = null; - const oldFetch = vi.fn(() => new Promise((_resolve, reject) => { rejectOld = reject; })); - const oldScheduled = captureScheduled(); - readUpstreamModelsSnapshotAndScheduleRefresh( - stubInstance(oldFetch), - { scheduler: oldScheduled.scheduler, fetcher: directFetcher }, - ); - await vi.waitFor(() => expect(oldFetch).toHaveBeenCalledTimes(1)); +test('background refreshes honor backoff and explicit refreshes bypass it', async () => { + const { record } = await setupCustom(); + const fetch = vi.fn(() => new Response('unavailable', { status: 503 })); - const explicitFetch = vi.fn(async () => [aModel('explicit-recovery-model')]); - const explicitInstance = stubInstance(explicitFetch); - const firstExplicit = fetchUpstreamModels(explicitInstance, directFetcher); - const secondExplicit = fetchUpstreamModels(explicitInstance, directFetcher); - rejectOld!(new Error('late old failure')); - await expect(oldScheduled.promises[0]).rejects.toThrow('late old failure'); - await expect(firstExplicit).resolves.toEqual([aModel('explicit-recovery-model')]); - await expect(secondExplicit).resolves.toEqual([aModel('explicit-recovery-model')]); - expect(explicitFetch).toHaveBeenCalledOnce(); - expect(await storedCache(repo)).toMatchObject({ models: [{ id: 'explicit-recovery-model' }], lastError: null }); + await withMockedFetch(fetch, async () => { + const target = modelsRefreshTarget(record); + await expect(refreshModels(target, 'TEST', { bypassBackoff: false, includeDiscovered: false })) + .rejects.toBeInstanceOf(ProviderModelsUnavailableError); + await expect(refreshModels(target, 'TEST', { bypassBackoff: false, includeDiscovered: false })) + .resolves.toEqual({ kind: 'backoff' }); + await expect(refreshModels(target, 'TEST', { bypassBackoff: true, includeDiscovered: false })) + .rejects.toBeInstanceOf(ProviderModelsUnavailableError); }); - test('catalog revision mismatch is cold and refreshes without blocking', async () => { - const repo = await setupRepo(); - const cache = await seedCache(repo, { - revision: MODEL_CATALOG_REVISION - 1, - fetchedAt: Date.now() - 1000, - models: [aModel('old-catalog')], - }); - const fetchFn = vi.fn(async () => [aModel('current-catalog')]); - const scheduled = captureScheduled(); + expect(fetch).toHaveBeenCalledTimes(2); +}); - const result = readUpstreamModelsSnapshotAndScheduleRefresh( - stubInstance(fetchFn, cache), - { scheduler: scheduled.scheduler, fetcher: directFetcher }, - ); +test('a changed config fences an old execution target before fetching', async () => { + const { repo, record } = await setupCustom(); + await repo.upstreams.save(buildCustomUpstreamRecord({ + config: { ...record.config as Record, apiKey: 'changed' }, + })); + const fetch = vi.fn(() => jsonResponse({ object: 'list', data: [] })); - expect(result.models).toEqual([]); - await scheduled.promises[0]; - expect((await storedCache(repo))?.revision).toBe(MODEL_CATALOG_REVISION); + await withMockedFetch(fetch, async () => { + await expect(refreshModels(modelsRefreshTarget(record), 'TEST', { bypassBackoff: true, includeDiscovered: false })) + .resolves.toEqual({ kind: 'generation-mismatch' }); }); + expect(fetch).not.toHaveBeenCalled(); +}); - test('an obsolete SQL cache hydrates cold and is replaced in the background', async () => { - const db = await createSqliteTestDb(); - const repo = new SqlRepo(db); - initRepo(repo); - await repo.upstreams.save({ - id: UPSTREAM_ID, - kind: 'custom', - name: 'Upstream A', - enabled: true, - sortOrder: 0, - createdAt: '2026-08-01T00:00:00.000Z', - updatedAt: '2026-08-01T00:00:00.000Z', - config: {}, - state: null, - modelsCache: null, - flagOverrides: {}, - disabledPublicModelIds: [], - proxyFallbackList: [], - modelPrefix: null, - hue: 210, - }); - await db.prepare('UPDATE upstreams SET models_cache_json = ? WHERE id = ?').bind(JSON.stringify({ - revision: MODEL_CATALOG_REVISION - 1, - fetchedAt: Date.now() - 1_000, - models: [{ id: 'old-catalog', enabledFlags: [] }], - lastError: null, - }), UPSTREAM_ID).run(); - - const hydrated = await repo.upstreams.getById(UPSTREAM_ID); - if (!hydrated) throw new Error('upstream row missing'); - expect(hydrated.modelsCache).toBeNull(); - const fetchFn = vi.fn(async () => [aModel('current-catalog')]); - const scheduled = captureScheduled(); - const result = readUpstreamModelsSnapshotAndScheduleRefresh( - stubInstance(fetchFn, hydrated.modelsCache, modelsCacheGeneration(hydrated)), - { scheduler: scheduled.scheduler, fetcher: directFetcher }, - ); - - expect(result.models).toEqual([]); - await scheduled.promises[0]; - expect((await repo.upstreams.getById(UPSTREAM_ID))?.modelsCache?.revision).toBe(MODEL_CATALOG_REVISION); - }); +test('custom explicit refresh returns discovered dashboard models from the same fetch', async () => { + const { record } = await setupCustom(); + await withMockedFetch( + () => jsonResponse({ object: 'list', data: [{ id: 'discovered', display_name: 'Discovered' }] }), + async () => { + const result = await refreshModels(modelsRefreshTarget(record), 'TEST', { bypassBackoff: true, includeDiscovered: true }); + expect(result).toMatchObject({ + kind: 'refreshed', + discovered: [{ upstreamModelId: 'discovered', publicModelId: 'discovered', display_name: 'Discovered' }], + }); + }, + ); }); diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index 59843307b..286f36be8 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -1,8 +1,8 @@ import { describe, expect, test, vi } from 'vitest'; -import { clearModelsRefreshesForTesting, fetchUpstreamModels } from '../../../src/data-plane/providers/models-refresh.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; import { enumerateModelCandidates, enumerateRealModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { modelsRefreshTarget, refreshModels } from '../../../src/execution/models-refresh.ts'; import { buildCustomUpstreamRecord, copilotModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; import { directFetcher, type InternalModel, type ProviderModel } from '@floway-dev/provider'; import { assertEquals, jsonResponse, withMockedFetch as withMockedFetchRaw } from '@floway-dev/test-utils'; @@ -66,15 +66,15 @@ test('enumerateModelCandidates blocks a cold catalog fetch after client disconne ); }); -test('a scheduled cold refresh survives disconnect before its claim dispatches', async () => { +test('a scheduled cold refresh survives disconnect after execution starts', async () => { const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(buildCustomUpstreamRecord()); - const originalClaim = repo.upstreams.claimModelsRefresh.bind(repo.upstreams); - let releaseClaim: (() => void) | null = null; - vi.spyOn(repo.upstreams, 'claimModelsRefresh').mockImplementation(async input => { - await new Promise(resolve => { releaseClaim = resolve; }); - return await originalClaim(input); + const originalBegin = repo.upstreams.beginModelsRefresh.bind(repo.upstreams); + let releaseBegin: (() => void) | null = null; + vi.spyOn(repo.upstreams, 'beginModelsRefresh').mockImplementation(async input => { + await new Promise(resolve => { releaseBegin = resolve; }); + return await originalBegin(input); }); const controller = new AbortController(); const background: Promise[] = []; @@ -94,9 +94,9 @@ test('a scheduled cold refresh survives disconnect before its claim dispatches', runtimeLocation: 'TEST', clientDisconnectSignal: controller.signal, }); - await vi.waitFor(() => expect(releaseClaim).not.toBeNull()); + await vi.waitFor(() => expect(releaseBegin).not.toBeNull()); controller.abort(new Error('client disconnected')); - releaseClaim!(); + releaseBegin!(); await Promise.all(background); }, ); @@ -267,10 +267,12 @@ test('enumerateRealModelCandidates only loads the selected providers\' catalogs' throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - await fetchUpstreamModels(providers[0], directFetcher); + const first = await repo.upstreams.getById(providers[0].upstreamId); + if (first === null) throw new Error('first upstream missing'); + await refreshModels(modelsRefreshTarget(first), 'TEST', { bypassBackoff: true, includeDiscovered: false }); const warmed = (await listModelProviders(null)).find(provider => provider.upstreamId === 'up_first'); if (!warmed) throw new Error('warmed provider missing'); - const { candidates } = await enumerateRealModelCandidates('target-model', 'chat', [warmed], { fetcherForUpstream: () => directFetcher, scheduler: testScheduler }); + const { candidates } = await enumerateRealModelCandidates('target-model', 'chat', [warmed], { fetcherForUpstream: () => directFetcher, scheduler: testScheduler, runtimeLocation: 'TEST' }); assertEquals(candidates[0]?.model.id, 'target-model'); assertEquals(candidates[0]?.provider.upstreamId, 'up_first'); @@ -314,8 +316,8 @@ test('enumerateRealModelCandidates rejects a model id disabled on that upstream await warmModelsForTest(); const providers = await listModelProviders(null); - const enabled = await enumerateRealModelCandidates('enabled-model', 'chat', providers, { fetcherForUpstream: () => directFetcher, scheduler: testScheduler }); - const disabled = await enumerateRealModelCandidates('disabled-model', 'chat', providers, { fetcherForUpstream: () => directFetcher, scheduler: testScheduler }); + const enabled = await enumerateRealModelCandidates('enabled-model', 'chat', providers, { fetcherForUpstream: () => directFetcher, scheduler: testScheduler, runtimeLocation: 'TEST' }); + const disabled = await enumerateRealModelCandidates('disabled-model', 'chat', providers, { fetcherForUpstream: () => directFetcher, scheduler: testScheduler, runtimeLocation: 'TEST' }); assertEquals(enabled.candidates[0]?.model.id, 'enabled-model'); assertEquals(disabled.candidates.length, 0); }); @@ -352,7 +354,6 @@ test('a recorded refresh failure is irrelevant when the prefix policy cannot add // upstream's display name flows back via `failedUpstreams` while its empty // or last-known-good snapshot stays independent of the current request. test('enumerateModelCandidates: healthy upstream still resolves alongside a rejecting one, with failedUpstreams reported', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); @@ -404,7 +405,6 @@ test('enumerateModelCandidates: healthy upstream still resolves alongside a reje // attempt, so the resolver returns immediately rather than walking the // stripped form. test('enumerateModelCandidates does NOT trigger the dated-suffix retry on a wrong-kind sawAnyId match', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(buildCustomUpstreamRecord({ @@ -445,7 +445,6 @@ test('enumerateModelCandidates does NOT trigger the dated-suffix retry on a wron // failedUpstreams across the two retry attempts must dedupe: a single broken // upstream that rejects both walks reports its name once, not twice. test('enumerateModelCandidates deduplicates failedUpstreams across the dated-suffix retry attempts', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(buildCustomUpstreamRecord({ @@ -481,7 +480,6 @@ test('enumerateModelCandidates deduplicates failedUpstreams across the dated-suf }); test('an AbortError from background catalog refresh does not abort model resolution', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(buildCustomUpstreamRecord({ @@ -519,7 +517,6 @@ test('an AbortError from background catalog refresh does not abort model resolut // upstream fetch. The failure renderer surfaces this as a model-missing 404 // without re-deriving the empty-cap branch. test('enumerateModelCandidates returns the empty triple when the visible upstream list is empty', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); // A populated catalog is the case under test: the empty cap, not an empty @@ -577,7 +574,6 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }; test('flattens across targets in declaration order for first-available', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ @@ -605,7 +601,6 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }); test('shuffles the outer walk for random selection but keeps intra-target order', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ @@ -638,7 +633,6 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }); test('dedups (model, upstream, rules) when two targets hit the same binding with identical rules', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ @@ -665,7 +659,6 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }); test('keeps the first representative in its original position when duplicate bindings are interleaved', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ @@ -697,7 +690,6 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }); test('keeps two entries for the same (model, upstream) with distinct rules', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ @@ -723,7 +715,6 @@ describe('enumerateModelCandidates alias walk (flat + dedup)', () => { }); test('falls through to a later target when an earlier one has no kind-matching binding', async () => { - clearModelsRefreshesForTesting(); const { repo } = await setupAppTest(); await seedUpstreams(repo); await repo.modelAliases.insert({ diff --git a/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts b/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts index fa442b546..115f2b502 100644 --- a/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts @@ -1,6 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { clearModelsRefreshesForTesting } from '../../../../src/data-plane/providers/models-refresh.ts'; import { enumerateAddressableModelIds } from '../../../../src/data-plane/shared/listing/addressable.ts'; import { buildCustomUpstreamRecord, setupAppTest, warmModelsForTest } from '../../../test-utils/app.ts'; import { directFetcher } from '@floway-dev/provider'; @@ -15,7 +14,6 @@ describe('enumerateAddressableModelIds', () => { const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); await repo.upstreams.save(buildCustomUpstreamRecord()); - clearModelsRefreshesForTesting(); await withMockedFetch( request => { @@ -27,7 +25,7 @@ describe('enumerateAddressableModelIds', () => { }, async () => { await warmModelsForTest(); - const surface = await enumerateAddressableModelIds(null, () => directFetcher, noBackground); + const surface = await enumerateAddressableModelIds(null, () => directFetcher, noBackground, 'TEST'); expect(surface.map(e => ({ id: e.id, unlisted: e.unlisted }))).toEqual([ { id: 'shared-model', unlisted: undefined }, ]); @@ -45,7 +43,6 @@ describe('enumerateAddressableModelIds', () => { // public id. modelPrefix: { prefix: 'cust/', addressable: ['unprefixed', 'prefixed'], listed: ['prefixed'] }, })); - clearModelsRefreshesForTesting(); await withMockedFetch( request => { @@ -57,7 +54,7 @@ describe('enumerateAddressableModelIds', () => { }, async () => { await warmModelsForTest(); - const surface = await enumerateAddressableModelIds(null, () => directFetcher, noBackground); + const surface = await enumerateAddressableModelIds(null, () => directFetcher, noBackground, 'TEST'); const byId = new Map(surface.map(e => [e.id, e])); expect(byId.get('cust/gpt-5.4')?.unlisted).toBeUndefined(); expect(byId.get('gpt-5.4')?.unlisted).toBe(true); @@ -71,9 +68,8 @@ describe('enumerateAddressableModelIds', () => { test('throws "no upstream configured" when the upstream cap is empty — surfacing the same hint /v1/models has always raised', async () => { const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); - clearModelsRefreshesForTesting(); - await expect(enumerateAddressableModelIds(null, () => directFetcher, noBackground)) + await expect(enumerateAddressableModelIds(null, () => directFetcher, noBackground, 'TEST')) .rejects.toThrow('No upstream provider configured'); }); }); diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index 422a7ed3e..4b266dc8c 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -4,7 +4,7 @@ import { buildKeyToUserMap } from '../../src/control-plane/shared/key-to-user.ts import { normalizeDisabledPublicModelIds } from '../../src/repo/disabled-public-models.ts'; import { normalizeFlagOverrides } from '../../src/repo/flag-overrides.ts'; import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; -import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; +import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-backoff.ts'; import { normalizeProxyFallbackList } from '../../src/repo/proxy-fallback-list.ts'; import { assertSameStoredResponsesItem, @@ -29,10 +29,9 @@ import type { AgentSetupRenewal, AgentSetupRepository, BackoffRow, - ModelsRefreshClaimInput, - ModelsRefreshClaimResult, + ModelsRefreshBeginInput, + ModelsRefreshBeginResult, ModelsRefreshFailureInput, - ModelsRefreshOwnerInput, ModelsRefreshSuccessInput, ModelAliasesRepo, ModelAliasRecord, @@ -738,7 +737,7 @@ class MemoryWebSearchConfigRepo implements WebSearchConfigRepo { class MemoryUpstreamRepo implements UpstreamRepo { private store = new Map(); - private modelsRefreshes = new Map(); + private modelsRefreshes = new Map(); list(): Promise { return Promise.resolve([...this.store.values()].map(cloneUpstreamRecord).sort((a, b) => a.sortOrder - b.sortOrder || a.createdAt.localeCompare(b.createdAt))); @@ -833,9 +832,8 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(); } - finalizeModelsRefreshSuccess(input: ModelsRefreshSuccessInput): Promise { - const { id, generation, token, cache } = input; - if (this.modelsRefreshes.get(id)?.claimToken !== token) return Promise.resolve(false); + publishModelsRefresh(input: ModelsRefreshSuccessInput): Promise { + const { id, generation, cache } = input; const existing = this.store.get(id); if (!existing || existing.configVersion !== generation.configVersion) return Promise.resolve(false); existing.modelsCache = { revision: cache.revision, fetchedAt: cache.fetchedAt, models: [...cache.models], lastError: null }; @@ -843,48 +841,27 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(true); } - finalizeModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise { - const { id, generation, token, error, previousFailureCount, failedAt } = input; + recordModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise { + const { id, generation, error, previousFailureCount, failedAt } = input; const failureCount = previousFailureCount + 1; const retryAt = modelsRefreshRetryAt(failedAt, previousFailureCount); const refresh = this.modelsRefreshes.get(id); - if (refresh?.claimToken !== token || refresh.failCount !== previousFailureCount) return Promise.resolve(false); + if ((refresh?.failureCount ?? 0) !== previousFailureCount) return Promise.resolve(false); const existing = this.store.get(id); if (!existing || existing.configVersion !== generation.configVersion) return Promise.resolve(false); if (existing.modelsCache) existing.modelsCache.lastError = error; else existing.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }; - this.modelsRefreshes.set(id, { failCount: failureCount, retryAt, claimToken: null, claimedAt: null }); + this.modelsRefreshes.set(id, { failureCount, retryAt }); return Promise.resolve(true); } - abandonModelsRefresh(input: ModelsRefreshOwnerInput): Promise { - const { id, generation, token } = input; - const existing = this.store.get(id); - const refresh = this.modelsRefreshes.get(id); - if (!existing - || existing.configVersion !== generation.configVersion - || refresh?.claimToken !== token) return Promise.resolve(false); - this.modelsRefreshes.set(id, { ...refresh, claimToken: null, claimedAt: null }); - return Promise.resolve(true); - } - - claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { - const { id, generation, token, now, staleClaimedBefore, bypassBackoff, observedActiveToken } = input; + beginModelsRefresh(input: ModelsRefreshBeginInput): Promise { + const { id, generation, now, bypassBackoff } = input; const stored = this.store.get(id); if (!stored || stored.configVersion !== generation.configVersion) return Promise.resolve({ kind: 'generation-mismatch' }); const existing = this.modelsRefreshes.get(id); - if (observedActiveToken !== null && existing === undefined) return Promise.resolve({ kind: 'completed' }); - if (existing !== undefined) { - if (existing.claimToken !== null && existing.claimedAt! > staleClaimedBefore) return Promise.resolve({ kind: 'active', token: existing.claimToken }); - if (!bypassBackoff && existing.retryAt > now) return Promise.resolve({ kind: 'backoff' }); - } - this.modelsRefreshes.set(id, { - failCount: existing?.failCount ?? 0, - retryAt: existing?.retryAt ?? 0, - claimToken: token, - claimedAt: now, - }); - return Promise.resolve({ kind: 'claimed', failureCount: existing?.failCount ?? 0 }); + if (!bypassBackoff && existing !== undefined && existing.retryAt > now) return Promise.resolve({ kind: 'backoff' }); + return Promise.resolve({ kind: 'ready', failureCount: existing?.failureCount ?? 0 }); } } diff --git a/packages/gateway/__tests__/repo/models-cache-fixture.ts b/packages/gateway/__tests__/repo/models-cache-fixture.ts index 53e067ba4..c765b0550 100644 --- a/packages/gateway/__tests__/repo/models-cache-fixture.ts +++ b/packages/gateway/__tests__/repo/models-cache-fixture.ts @@ -17,10 +17,7 @@ export const seedModelsCache = async ( generation: ModelsCacheGeneration, cache: Omit, ): Promise => { - const token = crypto.randomUUID(); - const claim = await repo.claimModelsRefresh({ id, generation, token, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, bypassBackoff: true, observedActiveToken: null }); - if (claim.kind !== 'claimed') return false; - return await repo.finalizeModelsRefreshSuccess({ id, generation, token, cache }); + return await repo.publishModelsRefresh({ id, generation, cache }); }; export const seedModelsCacheError = async ( @@ -29,8 +26,5 @@ export const seedModelsCacheError = async ( generation: ModelsCacheGeneration, error: NonNullable, ): Promise => { - const token = crypto.randomUUID(); - const claim = await repo.claimModelsRefresh({ id, generation, token, now: Date.now(), staleClaimedBefore: Number.MIN_SAFE_INTEGER, bypassBackoff: true, observedActiveToken: null }); - if (claim.kind !== 'claimed') return false; - return await repo.finalizeModelsRefreshFailure({ id, generation, token, error, previousFailureCount: 0, failedAt: -60_000 }); + return await repo.recordModelsRefreshFailure({ id, generation, error, previousFailureCount: 0, failedAt: -60_000 }); }; diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index 79d4517f3..52a034868 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -3,9 +3,9 @@ import { describe, expect, test } from 'vitest'; import { InMemoryRepo } from './memory.ts'; import { createSqliteTestDb } from './test-sqlite.ts'; import { MODEL_CATALOG_REVISION, modelsCacheGeneration } from '../../src/repo/models-cache-contract.ts'; -import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-contract.ts'; +import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-backoff.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; -import type { ModelsCacheGeneration, Repo, StoredUpstreamRecord } from '../../src/repo/types.ts'; +import type { Repo, StoredUpstreamRecord } from '../../src/repo/types.ts'; const record: StoredUpstreamRecord = { id: 'up_refresh', @@ -26,228 +26,94 @@ const record: StoredUpstreamRecord = { hue: 210, }; -const generation: ModelsCacheGeneration = modelsCacheGeneration(record); - const factories: [string, () => Promise][] = [ ['memory', async () => new InMemoryRepo()], ['SQL', async () => new SqlRepo(await createSqliteTestDb())], ]; -describe.each(factories)('%s models refresh coordination', (_name, createRepo) => { - test('catalog refresh inputs advance the generation while state writes do not', async () => { - const repo = await createRepo(); - await repo.upstreams.save(record); - await repo.upstreams.saveState(record.id, () => ({ accessToken: 'rotated' })); - expect((await repo.upstreams.getById(record.id))?.configVersion).toBe(1); - - const current = await repo.upstreams.getById(record.id); - if (current === null) throw new Error('upstream row missing'); - await repo.upstreams.save({ ...current, config: { tenant: 'next' } }); - const changed = await repo.upstreams.getById(record.id); - expect(changed?.configVersion).toBe(2); - expect(changed?.state).toEqual({ accessToken: 'rotated' }); - if (changed === null) throw new Error('upstream row missing after config update'); - await repo.upstreams.save({ ...changed, flagOverrides: { 'vendor-deepseek': true } }); - const flagged = await repo.upstreams.getById(record.id); - expect(flagged?.configVersion).toBe(3); - if (flagged === null) throw new Error('upstream row missing after flag update'); - await repo.upstreams.save({ ...flagged, proxyFallbackList: [{ id: 'direct_fetch' }] }); - expect((await repo.upstreams.getById(record.id))?.configVersion).toBe(4); - }); - - test('claims atomically, applies one backoff schedule, and lets force bypass cooldown', async () => { - const repo = await createRepo(); - await repo.upstreams.save(record); +describe.each(factories)('%s models refresh persistence', (_name, createRepo) => { + test('applies retry backoff and lets an explicit refresh bypass it', async () => { + const repo = (await createRepo()).upstreams; + await repo.save(record); + const generation = modelsCacheGeneration(record); let now = 1_800_000_000_000; - const first = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'claim-0', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); - expect(first).toEqual({ kind: 'claimed', failureCount: 0 }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'racer', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'claim-0' }); - - const delays = [1, 2, 4, 8, 16, 32, 60, 60].map(minutes => minutes * 60_000); - if (first.kind !== 'claimed') throw new Error('expected refresh claim'); - let claim = first; - for (const [index, delay] of delays.entries()) { - const retryAt = modelsRefreshRetryAt(now, claim.failureCount); - expect(retryAt - now).toBe(delay); - await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: `claim-${index}`, error: { message: 'failure', at: now }, previousFailureCount: claim.failureCount, failedAt: now }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: `early-${index}`, now: retryAt - 1, staleClaimedBefore: retryAt - 900_001, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'backoff' }); + for (const [failureCount, minutes] of [1, 5, 30, 120, 120].entries()) { + await expect(repo.beginModelsRefresh({ id: record.id, generation, now, bypassBackoff: false })) + .resolves.toEqual({ kind: 'ready', failureCount }); + await expect(repo.recordModelsRefreshFailure({ + id: record.id, + generation, + error: { message: 'failure', at: now }, + previousFailureCount: failureCount, + failedAt: now, + })).resolves.toBe(true); + const retryAt = modelsRefreshRetryAt(now, failureCount); + expect(retryAt - now).toBe(minutes * 60_000); + await expect(repo.beginModelsRefresh({ id: record.id, generation, now: retryAt - 1, bypassBackoff: false })) + .resolves.toEqual({ kind: 'backoff' }); + await expect(repo.beginModelsRefresh({ id: record.id, generation, now: retryAt - 1, bypassBackoff: true })) + .resolves.toEqual({ kind: 'ready', failureCount: failureCount + 1 }); now = retryAt; - const nextClaim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: `claim-${index + 1}`, now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); - if (nextClaim.kind !== 'claimed') throw new Error('expected refresh claim'); - claim = nextClaim; - expect(claim.failureCount).toBe(index + 1); } - - await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: `claim-${delays.length}`, error: { message: 'failure', at: now }, previousFailureCount: claim.failureCount, failedAt: now }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'forced', now: now + 1, staleClaimedBefore: now - 899_999, bypassBackoff: true, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: delays.length + 1 }); - await repo.upstreams.finalizeModelsRefreshSuccess({ id: record.id, generation, token: 'forced', cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [] } }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'after-success', now: now + 2, staleClaimedBefore: now - 899_998, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - }); - - test('a waiter acquires an abandoned claim rather than treating it as completed', async () => { - const repo = await createRepo(); - await repo.upstreams.save(record); - const now = 1_800_000_000_000; - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'owner', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - await expect(repo.upstreams.abandonModelsRefresh({ id: record.id, generation, token: 'owner' })).resolves.toBe(true); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'waiter', now: now + 1, staleClaimedBefore: now - 899_999, bypassBackoff: true, observedActiveToken: 'owner' })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); }); - test('recovers abandoned claims and fences tokens and config versions', async () => { - const repo = await createRepo(); - await repo.upstreams.save(record); + test('success publishes the catalog and clears failure backoff', async () => { + const repo = (await createRepo()).upstreams; + await repo.save(record); + const generation = modelsCacheGeneration(record); const now = 1_800_000_000_000; + await repo.recordModelsRefreshFailure({ id: record.id, generation, error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'abandoned', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'replacement', now: now + 900_001, staleClaimedBefore: now + 1, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - await repo.upstreams.finalizeModelsRefreshSuccess({ id: record.id, generation, token: 'abandoned', cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 900_001, models: [] } }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'racer', now: now + 900_002, staleClaimedBefore: now + 2, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'replacement' }); - - const next = { ...record, config: { tenant: 'next' } }; - await repo.upstreams.replaceForModels({ previous: record, upstream: next }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'old-config', now: now + 900_003, staleClaimedBefore: now + 3, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'generation-mismatch' }); - const storedNext = await repo.upstreams.getById(record.id); - if (storedNext === null) throw new Error('upstream row missing'); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(storedNext), token: 'current', now: now + 900_003, staleClaimedBefore: now + 3, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - - const renamed = { ...storedNext, name: 'Renamed', updatedAt: '2026-08-01T00:01:00.000Z' }; - await repo.upstreams.replaceForModels({ previous: storedNext, upstream: renamed }); - await expect(repo.upstreams.claimModelsRefresh({ id: record.id, generation: modelsCacheGeneration(storedNext), token: 'after-rename', now: now + 900_004, staleClaimedBefore: now + 4, bypassBackoff: false, observedActiveToken: null })).resolves.toEqual({ kind: 'active', token: 'current' }); - }); - - test('metadata saves preserve an active refresh owner', async () => { - const repo = await createRepo(); - await repo.upstreams.save(record); - const now = 1_800_000_000_000; - const claim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'active', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); - if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); - - const next = { ...record, name: 'Renamed', updatedAt: '2026-08-01T00:01:00.000Z' }; - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next })).resolves.not.toBeNull(); - await expect(repo.upstreams.claimModelsRefresh({ - id: record.id, - generation: modelsCacheGeneration(next), - token: 'racer', - now: now + 1, - staleClaimedBefore: now - 899_999, - bypassBackoff: false, - observedActiveToken: null, - })).resolves.toEqual({ kind: 'active', token: 'active' }); - await expect(repo.upstreams.finalizeModelsRefreshSuccess({ + await expect(repo.publishModelsRefresh({ id: record.id, generation, - token: 'active', cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [] }, })).resolves.toBe(true); + await expect(repo.beginModelsRefresh({ id: record.id, generation, now: now + 2, bypassBackoff: false })) + .resolves.toEqual({ kind: 'ready', failureCount: 0 }); + expect((await repo.getById(record.id))?.modelsCache).toMatchObject({ fetchedAt: now + 1, lastError: null }); }); - test('state changes preserve the snapshot generation and refresh cooldown', async () => { - const repo = await createRepo(); - await repo.upstreams.save(record); - const now = 1_800_000_000_000; - const claim = await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'failed', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); - if (claim.kind !== 'claimed') throw new Error('expected refresh claim'); - await repo.upstreams.finalizeModelsRefreshFailure({ id: record.id, generation, token: 'failed', error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); - - const next = { ...record, state: { credential: 'rotated' }, updatedAt: '2026-08-01T00:01:00.000Z' }; - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: next })).resolves.not.toBeNull(); - expect((await repo.upstreams.getById(record.id))?.modelsCache?.lastError?.message).toBe('failure'); - await expect(repo.upstreams.claimModelsRefresh({ - id: record.id, - generation: modelsCacheGeneration(next), - token: 'new-credential', - now: now + 1, - staleClaimedBefore: now - 899_999, - bypassBackoff: false, - observedActiveToken: null, - })).resolves.toEqual({ kind: 'backoff' }); - }); - - test('provider-managed credential state can rotate without invalidating its own owner', async () => { - const repo = await createRepo(); - await repo.upstreams.save(record); - const claim = await repo.upstreams.claimModelsRefresh({ - id: record.id, - generation, - token: 'state-owner', - now: 1_800_000_000_000, - staleClaimedBefore: 1_799_999_100_000, - bypassBackoff: false, - observedActiveToken: null, - }); - expect(claim.kind).toBe('claimed'); - await repo.upstreams.saveState(record.id, () => ({ credential: 'rotated' })); - - await expect(repo.upstreams.finalizeModelsRefreshSuccess({ - id: record.id, - generation, - token: 'state-owner', - cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_800_000_000_001, models: [] }, - })).resolves.toBe(true); - await expect(repo.upstreams.claimModelsRefresh({ - id: record.id, - generation, - token: 'stale-generation', - now: 1_800_000_000_002, - staleClaimedBefore: 1_799_999_100_002, - bypassBackoff: false, - observedActiveToken: null, - })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); - }); - - test('catalog-aware replacement rejects a stale control-plane writer', async () => { - const repo = await createRepo(); - await repo.upstreams.save(record); - await repo.upstreams.saveState(record.id, () => ({ providerManaged: 'newer' })); - const winner = { ...record, name: 'Winner', updatedAt: '2026-08-01T00:01:00.000Z' }; - const stale = { ...record, name: 'Stale', updatedAt: '2026-08-01T00:02:00.000Z' }; - - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: winner })).resolves.not.toBeNull(); - await expect(repo.upstreams.replaceForModels({ previous: record, upstream: stale })).resolves.toBeNull(); - expect((await repo.upstreams.getById(record.id))?.name).toBe('Winner'); - expect((await repo.upstreams.getById(record.id))?.state).toEqual({ providerManaged: 'newer' }); - }); - - test('catalog-aware insertion never overwrites a concurrent winner', async () => { - const repo = await createRepo(); - await expect(repo.upstreams.insertForModels(record)).resolves.not.toBeNull(); - await expect(repo.upstreams.insertForModels({ ...record, name: 'Loser' })).resolves.toBeNull(); - expect((await repo.upstreams.getById(record.id))?.name).toBe(record.name); + test('config changes fence stale success and failure publication', async () => { + const repo = (await createRepo()).upstreams; + await repo.save(record); + const generation = modelsCacheGeneration(record); + const current = await repo.getById(record.id); + if (current === null) throw new Error('upstream row missing'); + await repo.replaceForModels({ previous: current, upstream: { ...current, config: { tenant: 'next' } } }); + + await expect(repo.beginModelsRefresh({ id: record.id, generation, now: 1, bypassBackoff: true })) + .resolves.toEqual({ kind: 'generation-mismatch' }); + await expect(repo.publishModelsRefresh({ id: record.id, generation, cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: 1, models: [] } })) + .resolves.toBe(false); + await expect(repo.recordModelsRefreshFailure({ id: record.id, generation, error: { message: 'old', at: 1 }, previousFailureCount: 0, failedAt: 1 })) + .resolves.toBe(false); }); - test('a waiter can reclaim a replacement owner after its lease also expires', async () => { - const repo = await createRepo(); - await repo.upstreams.save(record); - const firstNow = 1_800_000_000_000; - await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'owner-a', now: firstNow, staleClaimedBefore: firstNow - 900_000, bypassBackoff: false, observedActiveToken: null }); - const secondNow = firstNow + 900_001; - await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'owner-b', now: secondNow, staleClaimedBefore: firstNow + 1, bypassBackoff: false, observedActiveToken: null }); + test('state and metadata changes preserve the generation and backoff', async () => { + const repo = (await createRepo()).upstreams; + await repo.save(record); + const generation = modelsCacheGeneration(record); + const now = 1_800_000_000_000; + await repo.recordModelsRefreshFailure({ id: record.id, generation, error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); + await repo.saveState(record.id, () => ({ credential: 'rotated' })); + const current = await repo.getById(record.id); + if (current === null) throw new Error('upstream row missing'); + await repo.replaceForModels({ previous: current, upstream: { ...current, name: 'Renamed' } }); - await expect(repo.upstreams.claimModelsRefresh({ - id: record.id, - generation, - token: 'waiter', - now: secondNow + 900_001, - staleClaimedBefore: secondNow + 1, - bypassBackoff: false, - observedActiveToken: 'owner-a', - })).resolves.toEqual({ kind: 'claimed', failureCount: 0 }); + expect((await repo.getById(record.id))?.configVersion).toBe(1); + await expect(repo.beginModelsRefresh({ id: record.id, generation, now: now + 1, bypassBackoff: false })) + .resolves.toEqual({ kind: 'backoff' }); }); - test('failure finalization rejects a count not issued with the claim', async () => { - const repo = await createRepo(); - await repo.upstreams.save(record); - const now = 1_800_000_000_000; - await repo.upstreams.claimModelsRefresh({ id: record.id, generation, token: 'owner', now, staleClaimedBefore: now - 900_000, bypassBackoff: false, observedActiveToken: null }); - - await expect(repo.upstreams.finalizeModelsRefreshFailure({ - id: record.id, - generation, - token: 'owner', - error: { message: 'failure', at: now }, - previousFailureCount: 99, - failedAt: now, - })).resolves.toBe(false); + test('catalog-aware writes reject stale and duplicate control-plane writers', async () => { + const repo = (await createRepo()).upstreams; + await expect(repo.insertForModels(record)).resolves.not.toBeNull(); + await expect(repo.insertForModels({ ...record, name: 'Loser' })).resolves.toBeNull(); + const current = await repo.getById(record.id); + if (current === null) throw new Error('upstream row missing'); + await expect(repo.replaceForModels({ previous: current, upstream: { ...current, name: 'Winner' } })).resolves.not.toBeNull(); + await expect(repo.replaceForModels({ previous: current, upstream: { ...current, name: 'Stale' } })).resolves.toBeNull(); }); }); diff --git a/packages/gateway/__tests__/repo/sql_test.ts b/packages/gateway/__tests__/repo/sql_test.ts index 6905c2f3c..46bc13cad 100644 --- a/packages/gateway/__tests__/repo/sql_test.ts +++ b/packages/gateway/__tests__/repo/sql_test.ts @@ -230,7 +230,7 @@ test('SQL rejects malformed persisted model refresh state', async () => { await assertRejects( () => db.prepare('UPDATE upstreams SET models_refresh_json = ? WHERE id = ?') - .bind(JSON.stringify({ failCount: 0, retryAt: 0, claimToken: 'owner', claimedAt: null }), 'up_test') + .bind(JSON.stringify({ failureCount: -1, retryAt: 0 }), 'up_test') .run(), ); }); diff --git a/packages/gateway/__tests__/test-utils/app.ts b/packages/gateway/__tests__/test-utils/app.ts index fc4387c34..476299d40 100644 --- a/packages/gateway/__tests__/test-utils/app.ts +++ b/packages/gateway/__tests__/test-utils/app.ts @@ -1,10 +1,8 @@ import { trackBackground } from './background-tracker.ts'; import { app } from '../../src/app.ts'; -import { clearModelsRefreshesForTesting, warmUpstreamModels } from '../../src/data-plane/providers/models-refresh.ts'; -import { listModelProviders } from '../../src/data-plane/providers/registry.ts'; import type { WebSearchConfig } from '../../src/data-plane/tools/web-search/types.ts'; -import { createPerRequestFetcher } from '../../src/dial/per-request.ts'; -import { initRepo } from '../../src/repo/index.ts'; +import { modelsRefreshTarget, refreshModels } from '../../src/execution/models-refresh.ts'; +import { getRepo, initRepo } from '../../src/repo/index.ts'; import type { ApiKey } from '../../src/repo/types.ts'; import { initBackgroundSchedulerResolver } from '../../src/runtime/background.ts'; import { InMemoryRepo } from '../repo/memory.ts'; @@ -140,7 +138,6 @@ export async function setupAppTest(options: SetupOptions = {}): Promise => { - const providers = await listModelProviders(null); - const fetcherForUpstream = await createPerRequestFetcher('TEST'); - await Promise.allSettled(providers.map(async provider => - await warmUpstreamModels(provider, fetcherForUpstream(provider.upstreamId)))); + const upstreams = await getRepo().upstreams.list(); + await Promise.allSettled(upstreams.map(async upstream => + await refreshModels(modelsRefreshTarget(upstream), 'TEST', { bypassBackoff: false, includeDiscovered: false }))); }; export function parseSSEText(text: string): Array<{ event: string; data: string }> { diff --git a/packages/gateway/migrations/0080_simplify_models_refresh.sql b/packages/gateway/migrations/0080_simplify_models_refresh.sql new file mode 100644 index 000000000..ff374ad4c --- /dev/null +++ b/packages/gateway/migrations/0080_simplify_models_refresh.sql @@ -0,0 +1,12 @@ +-- Execution cells own in-flight coordination. D1 retains only retry backoff; +-- dropping the old column also removes its claim/lease shape constraint. +ALTER TABLE upstreams DROP COLUMN models_refresh_json; +ALTER TABLE upstreams ADD COLUMN models_refresh_json TEXT NULL CHECK ( + models_refresh_json IS NULL OR coalesce(( + json_valid(models_refresh_json) = 1 + AND json_type(models_refresh_json, '$.failureCount') = 'integer' + AND json_extract(models_refresh_json, '$.failureCount') >= 0 + AND json_type(models_refresh_json, '$.retryAt') = 'integer' + AND json_extract(models_refresh_json, '$.retryAt') >= 0 + ), 0) = 1 +); diff --git a/packages/gateway/src/control-plane/models/routes.ts b/packages/gateway/src/control-plane/models/routes.ts index 2a14675a7..568585970 100644 --- a/packages/gateway/src/control-plane/models/routes.ts +++ b/packages/gateway/src/control-plane/models/routes.ts @@ -87,17 +87,18 @@ export const controlPlaneModels = async (c: CtxWithQuery) => // `enumerateAddressableModelIds`, the hue-join map) so this request // pays a single `upstreams.list()` round-trip. const upstreamRows = await getRepo().upstreams.list(); - const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw), upstreamRows); + const runtimeLocation = getRuntimeLocation(c.req.raw); + const fetcherForUpstream = await createPerRequestFetcher(runtimeLocation, upstreamRows); // Two addressable surfaces: caller-scoped (drives visibility + // `aliasedFrom.targets` narrowing for non-admin) and gateway-wide // (drives the alias's metadata + endpoints + pricing — every caller // sees the same numbers for the same alias). For admin the two are // the same, so skip the second fetch. const [callerAddressable, gatewayAddressable, aliases] = await Promise.all([ - enumerateAddressableModelIds(upstreamScope, fetcherForUpstream, backgroundSchedulerFromContext(c), upstreamRows), + enumerateAddressableModelIds(upstreamScope, fetcherForUpstream, backgroundSchedulerFromContext(c), runtimeLocation, upstreamRows), isAdmin ? Promise.resolve(null) - : enumerateAddressableModelIds(null, fetcherForUpstream, backgroundSchedulerFromContext(c), upstreamRows), + : enumerateAddressableModelIds(null, fetcherForUpstream, backgroundSchedulerFromContext(c), runtimeLocation, upstreamRows), includeAliases ? getRepo().modelAliases.list() : Promise.resolve([]), ]); const hueByUpstream = new Map(upstreamRows.map(row => [row.id, row.hue])); diff --git a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts index bac0c2d71..231ba6edc 100644 --- a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts +++ b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts @@ -1,8 +1,6 @@ import type { Context } from 'hono'; -import { warmUpstreamModels } from '../../data-plane/providers/models-refresh.ts'; -import { createProvider } from '../../data-plane/providers/registry.ts'; -import { createPerRequestFetcher } from '../../dial/per-request.ts'; +import { modelsRefreshTarget, refreshModels } from '../../execution/models-refresh.ts'; import { getRepo } from '../../repo/index.ts'; import type { StoredUpstreamRecord } from '../../repo/types.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; @@ -46,10 +44,10 @@ export const saveUpstreamsAndWarmChangedModels = async ( const recordsToWarm = saved.filter(result => result.modelsChanged).map(result => result.record); if (recordsToWarm.length === 0) return new Map(saved.map(result => [result.record.id, result.record])); - const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw), recordsToWarm); + const runtimeLocation = getRuntimeLocation(c.req.raw); const warmedEntries = await Promise.all(recordsToWarm.map(async record => { try { - await warmUpstreamModels(createProvider(record), fetcherForUpstream(record.id)); + await refreshModels(modelsRefreshTarget(record), runtimeLocation, { bypassBackoff: false, includeDiscovered: false }); } catch (error) { logInfo('warm_models_cache_failed', { upstream_id: record.id, error: errorMessage(error) }); } diff --git a/packages/gateway/src/control-plane/upstreams/models.ts b/packages/gateway/src/control-plane/upstreams/models.ts index ef765e499..9d1c5c625 100644 --- a/packages/gateway/src/control-plane/upstreams/models.ts +++ b/packages/gateway/src/control-plane/upstreams/models.ts @@ -2,15 +2,15 @@ import { modelsCacheStatus } from './models-cache-status.ts'; import { resolveControlPlaneFetcher } from './proxy-resolution.ts'; import { isValidProviderKind, upstreamErrorMessage as errorMessage } from './shared.ts'; import { MODEL_LISTING_FAILURE_CODE, MODEL_LISTING_FAILURE_MESSAGE } from '../../data-plane/models/shared.ts'; -import { fetchUpstreamModels } from '../../data-plane/providers/models-refresh.ts'; -import { createPreviewProvider, createProvider } from '../../data-plane/providers/registry.ts'; +import { createPreviewProvider } from '../../data-plane/providers/registry.ts'; +import { modelsRefreshTarget, refreshModels } from '../../execution/models-refresh.ts'; import type { AuthedContext } from '../../middleware/auth.ts'; import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import type { previewModelsBody } from '../schemas.ts'; import { ProviderModelsUnavailableError, type Fetcher, type ProviderModel, type ProxyFallbackEntry, type UpstreamModelConfig, type UpstreamRecord } from '@floway-dev/provider'; -import { assertCustomUpstreamRecord, fetchCustomModels, projectCustomModels, projectCustomDiscoveredModels } from '@floway-dev/provider-custom'; +import { assertCustomUpstreamRecord, fetchCustomModels, projectCustomDiscoveredModels } from '@floway-dev/provider-custom'; // `upstreamModelId` is the wire-side identifier the provider will send when // a caller invokes the public `model.id` — Claude Code exposes @@ -106,29 +106,16 @@ export const fetchSavedModels = async (c: AuthedContext<'/:id/list-models'>) => const record = await getRepo().upstreams.getById(id); if (record === null) return c.json({ error: 'Upstream not found' }, 404); - let fetcher: Fetcher; try { - fetcher = await resolveControlPlaneFetcher({ - override: record.proxyFallbackList, - upstreamId: id, - runtimeLocation: getRuntimeLocation(c.req.raw), + const result = await refreshModels(modelsRefreshTarget(record), getRuntimeLocation(c.req.raw), { + bypassBackoff: true, + includeDiscovered: record.kind === 'custom', }); - } catch (err) { - return c.json({ error: errorMessage(err) }, 400); - } - - try { - let data: UpstreamModelConfig[]; - if (record.kind === 'custom') { - const config = assertCustomUpstreamRecord(record).config; - const result = await fetchCustomModels(config, fetcher); - await fetchUpstreamModels(createProvider(record), fetcher, async () => projectCustomModels(record, result)); - data = projectCustomDiscoveredModels(record, result); - } else { - data = (await fetchUpstreamModels(createProvider(record), fetcher)).map(reshapeModelForDashboard); - } + if (result.kind !== 'refreshed') throw new Error(`Upstream ${id} changed during models refresh`); const refreshed = await getRepo().upstreams.getById(id); if (refreshed === null) throw new Error(`Upstream ${id} disappeared after models refresh`); + const data = result.discovered ?? refreshed.modelsCache?.models.map(reshapeModelForDashboard); + if (data === undefined) throw new Error(`Upstream ${id} models refresh did not publish a catalog`); return c.json({ data, modelsCache: modelsCacheStatus(refreshed) }); } catch (e) { if (e instanceof ProviderModelsUnavailableError) { diff --git a/packages/gateway/src/data-plane/codex/models.ts b/packages/gateway/src/data-plane/codex/models.ts index 069715779..715428116 100644 --- a/packages/gateway/src/data-plane/codex/models.ts +++ b/packages/gateway/src/data-plane/codex/models.ts @@ -64,10 +64,11 @@ export const loadCodexCatalog = async ( upstreamIds: readonly string[] | null, fetcherForUpstream: (upstreamId: string) => Fetcher, scheduler: BackgroundScheduler, + runtimeLocation: string, ): Promise => { const [resolution, addressable] = await Promise.all([ resolveCodexCatalog(userAgent), - enumerateAddressableModelIds(upstreamIds, fetcherForUpstream, scheduler), + enumerateAddressableModelIds(upstreamIds, fetcherForUpstream, scheduler, runtimeLocation), ]); return assembleCodexCatalog(resolution.catalog, addressable, resolution.capabilities); }; diff --git a/packages/gateway/src/data-plane/models/gemini.ts b/packages/gateway/src/data-plane/models/gemini.ts index 353e7787d..d189ad83d 100644 --- a/packages/gateway/src/data-plane/models/gemini.ts +++ b/packages/gateway/src/data-plane/models/gemini.ts @@ -66,13 +66,14 @@ const loadGeminiModels = async ( upstreamFilter: readonly string[] | null, fetcherForUpstream: (upstreamId: string) => Fetcher, scheduler: BackgroundScheduler, + runtimeLocation: string, aliasRepo: ModelAliasesRepo, ): Promise => { const [callerAddressable, gatewayAddressable, aliases] = await Promise.all([ - enumerateAddressableModelIds(upstreamFilter, fetcherForUpstream, scheduler), + enumerateAddressableModelIds(upstreamFilter, fetcherForUpstream, scheduler, runtimeLocation), upstreamFilter === null ? Promise.resolve(null) - : enumerateAddressableModelIds(null, fetcherForUpstream, scheduler), + : enumerateAddressableModelIds(null, fetcherForUpstream, scheduler, runtimeLocation), aliasRepo.list(), ]); const gatewayAddressableModelIds = gatewayAddressable ?? callerAddressable; @@ -93,7 +94,7 @@ const loadGeminiModels = async ( export const serveGeminiModels = async (c: Context): Promise => { try { const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw)); - return Response.json({ models: await loadGeminiModels(effectiveUpstreamIdsFromContext(c), fetcherForUpstream, backgroundSchedulerFromContext(c), getRepo().modelAliases) }); + return Response.json({ models: await loadGeminiModels(effectiveUpstreamIdsFromContext(c), fetcherForUpstream, backgroundSchedulerFromContext(c), getRuntimeLocation(c.req.raw), getRepo().modelAliases) }); } catch (error) { return geminiModelLoadError(error); } @@ -106,7 +107,7 @@ export const serveGeminiModelInfo = async (c: Context): Promise => { const modelId = rawModelId.replace(/^models\//, ''); try { const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw)); - const model = (await loadGeminiModels(effectiveUpstreamIdsFromContext(c), fetcherForUpstream, backgroundSchedulerFromContext(c), getRepo().modelAliases)).find(candidate => candidate.baseModelId === modelId || candidate.name === `models/${modelId}`); + const model = (await loadGeminiModels(effectiveUpstreamIdsFromContext(c), fetcherForUpstream, backgroundSchedulerFromContext(c), getRuntimeLocation(c.req.raw), getRepo().modelAliases)).find(candidate => candidate.baseModelId === modelId || candidate.name === `models/${modelId}`); if (!model) return geminiError(404, `Model not found: ${modelId}`); return Response.json(model); } catch (error) { diff --git a/packages/gateway/src/data-plane/models/http.ts b/packages/gateway/src/data-plane/models/http.ts index 8aa06391a..f7f71cdcb 100644 --- a/packages/gateway/src/data-plane/models/http.ts +++ b/packages/gateway/src/data-plane/models/http.ts @@ -73,15 +73,16 @@ const isClaudeCodeUserAgent = (userAgent: string | undefined): boolean => export const serveModels = async (c: Context): Promise => { try { const userAgent = c.req.header('user-agent'); - const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw)); + const runtimeLocation = getRuntimeLocation(c.req.raw); + const fetcherForUpstream = await createPerRequestFetcher(runtimeLocation); const upstreamIds = effectiveUpstreamIdsFromContext(c); const scheduler = backgroundSchedulerFromContext(c); if (isCodexUserAgent(userAgent)) { - return Response.json(await loadCodexCatalog(userAgent, upstreamIds, fetcherForUpstream, scheduler)); + return Response.json(await loadCodexCatalog(userAgent, upstreamIds, fetcherForUpstream, scheduler, runtimeLocation)); } - const publicCatalog = await loadModels(upstreamIds, fetcherForUpstream, scheduler, getRepo().modelAliases); + const publicCatalog = await loadModels(upstreamIds, fetcherForUpstream, scheduler, runtimeLocation, getRepo().modelAliases); // The Claude Code CLI's model discovery request identifies itself with // a `claude-code/` User-Agent (built from the CLI's `n_()` // helper — verified in the v2.1.206 binary). The CLI's other request diff --git a/packages/gateway/src/data-plane/models/load.ts b/packages/gateway/src/data-plane/models/load.ts index 8ccbe09f7..607801f44 100644 --- a/packages/gateway/src/data-plane/models/load.ts +++ b/packages/gateway/src/data-plane/models/load.ts @@ -44,6 +44,7 @@ export const loadModels = async ( upstreamFilter: readonly string[] | null, fetcherForUpstream: (upstreamId: string) => Fetcher, scheduler: BackgroundScheduler, + runtimeLocation: string, aliasRepo: ModelAliasesRepo, ): Promise => { // Data-plane responses always narrow `aliasedFrom.targets` to the @@ -51,10 +52,10 @@ export const loadModels = async ( // ids), but the alias's metadata is still computed gateway-wide so // every caller sees the same numbers. const [callerAddressable, gatewayAddressable, aliases] = await Promise.all([ - enumerateAddressableModelIds(upstreamFilter, fetcherForUpstream, scheduler), + enumerateAddressableModelIds(upstreamFilter, fetcherForUpstream, scheduler, runtimeLocation), upstreamFilter === null ? Promise.resolve(null) - : enumerateAddressableModelIds(null, fetcherForUpstream, scheduler), + : enumerateAddressableModelIds(null, fetcherForUpstream, scheduler, runtimeLocation), aliasRepo.list(), ]); const gatewayAddressableModelIds = gatewayAddressable ?? callerAddressable; diff --git a/packages/gateway/src/data-plane/providers/catalog.ts b/packages/gateway/src/data-plane/providers/catalog.ts index c19eea081..fcad607a0 100644 --- a/packages/gateway/src/data-plane/providers/catalog.ts +++ b/packages/gateway/src/data-plane/providers/catalog.ts @@ -85,6 +85,7 @@ const collectProviderModels = async ( providers: readonly GatewayProvider[], fetcherForUpstream: (upstreamId: string) => Fetcher, scheduler: BackgroundScheduler, + runtimeLocation: string, ): Promise => { const byId = new Map(); const upstreamsByPublicId = new Map(); @@ -98,7 +99,7 @@ const collectProviderModels = async ( const fetchOne = (instance: GatewayProvider) => { const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(instance, { scheduler, - fetcher: fetcherForUpstream(instance.upstreamId), + runtimeLocation, }); return { instance, models: snapshot.models, lastError: snapshot.lastError }; }; @@ -222,12 +223,13 @@ export const getModelsFromProviders = async ( providers: readonly GatewayProvider[], fetcherForUpstream: (upstreamId: string) => Fetcher, scheduler: BackgroundScheduler, + runtimeLocation: string, ): Promise<{ models: InternalModel[]; upstreamsByPublicId: Map; failedUpstreams: readonly string[] }> => { if (providers.length === 0) { throw new Error('No upstream provider configured — connect GitHub Copilot or add a Custom/Azure upstream in the dashboard'); } - const { models, upstreamsByPublicId, sawSuccess, lastError, failedUpstreams } = await collectProviderModels(providers, fetcherForUpstream, scheduler); + const { models, upstreamsByPublicId, sawSuccess, lastError, failedUpstreams } = await collectProviderModels(providers, fetcherForUpstream, scheduler, runtimeLocation); // TODO: surface `failedUpstreams` on each listing endpoint's wire response // so partial-listing failures reach clients. diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index df3e2cb1c..7ced8d707 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -1,8 +1,8 @@ -import { scheduleUpstreamModelsRefresh } from './models-refresh.ts'; import type { GatewayProvider } from './registry.ts'; +import { scheduleModelsRefresh } from '../../execution/models-refresh.ts'; import { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; -import type { Fetcher, ProviderModel, UpstreamModelsCache } from '@floway-dev/provider'; +import type { ProviderModel, UpstreamModelsCache } from '@floway-dev/provider'; const SOFT_MS = 10 * 60 * 1000; @@ -15,7 +15,7 @@ export interface ModelsSnapshot { interface ModelsSnapshotReadOptions { scheduler: BackgroundScheduler; - fetcher: Fetcher; + runtimeLocation: string; } // Capture one immutable snapshot before scheduling any refresh work so its @@ -24,14 +24,19 @@ export const readUpstreamModelsSnapshotAndScheduleRefresh = ( instance: GatewayProvider, options: ModelsSnapshotReadOptions, ): ModelsSnapshot => { - const { scheduler, fetcher } = options; + const { scheduler, runtimeLocation } = options; const cached = instance.modelsCache?.revision === MODEL_CATALOG_REVISION ? instance.modelsCache : null; const snapshot = { models: cached?.models ?? [], lastError: cached?.lastError ?? null, }; if (!cached || Date.now() - cached.fetchedAt >= SOFT_MS) { - scheduleUpstreamModelsRefresh(instance, scheduler, fetcher); + const fetchedAt = instance.modelsCache?.fetchedAt; + scheduleModelsRefresh({ + upstreamId: instance.upstreamId, + configVersion: instance.modelsCacheGeneration.configVersion, + cacheEpoch: fetchedAt === undefined || fetchedAt === 0 ? null : fetchedAt, + }, runtimeLocation, scheduler); } return snapshot; }; diff --git a/packages/gateway/src/data-plane/providers/models-refresh.ts b/packages/gateway/src/data-plane/providers/models-refresh.ts deleted file mode 100644 index e77384b9e..000000000 --- a/packages/gateway/src/data-plane/providers/models-refresh.ts +++ /dev/null @@ -1,224 +0,0 @@ -import type { GatewayProvider } from './registry.ts'; -import { getRepo } from '../../repo/index.ts'; -import { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; -import { MODELS_REFRESH_CLAIM_LEASE_MS } from '../../repo/models-refresh-contract.ts'; -import type { BackgroundScheduler } from '@floway-dev/platform'; -import type { Fetcher, ProviderModel } from '@floway-dev/provider'; - -const ACTIVE_REFRESH_POLL_MS = 100; -const ACTIVE_REFRESH_POLL_CAP_MS = 1_000; -const ACTIVE_REFRESH_WAIT_MS = 60_000; - -// L1: per-isolate in-flight memoization. Saved upstreams join only within one -// persisted config generation; draft previews bypass this coordinator. Not a -// TTL cache — the entry is removed when the promise settles. The conditional -// delete defends against a stale removal racing a later replacement. -type RefreshMode = 'explicit' | 'warm' | 'background'; - -interface InFlightRefresh { - mode: RefreshMode; - promise: Promise; -} - -const inFlight = new Map(); - -const startInFlight = ( - key: string, - mode: RefreshMode, - fn: () => Promise, -): Promise => { - const entry: InFlightRefresh = { mode, promise: fn() }; - inFlight.set(key, entry); - entry.promise.finally(() => { - if (inFlight.get(key) === entry) inFlight.delete(key); - }).catch(() => {}); - return entry.promise; -}; - -const memoInFlight = ( - key: string, - mode: RefreshMode, - fn: () => Promise, -): Promise => { - const existing = inFlight.get(key); - return existing?.promise ?? startInFlight(key, mode, fn); -}; - -const errorMessage = (err: unknown): string => err instanceof Error ? err.message : String(err); - -const finalizeRefresh = async ( - finalize: () => Promise, - abandon: () => Promise, -): Promise => { - const errors: unknown[] = []; - for (let attempt = 0; attempt < 3; attempt++) { - try { - return await finalize(); - } catch (error) { - errors.push(error); - } - } - try { - if (!await abandon()) return false; - } catch (error) { - errors.push(error); - } - throw new AggregateError(errors, 'Failed to finalize models refresh'); -}; - -const runClaimedRefresh = async ( - instance: GatewayProvider, - fetcher: Fetcher, - mode: RefreshMode, - loadProvidedModels?: () => Promise, -): Promise => { - const repo = getRepo(); - const token = crypto.randomUUID(); - let observedActiveToken: string | null = null; - let pollMs = ACTIVE_REFRESH_POLL_MS; - const waitDeadline = Date.now() + ACTIVE_REFRESH_WAIT_MS; - while (true) { - const now = Date.now(); - const outcome = await repo.upstreams.claimModelsRefresh({ - id: instance.upstreamId, - generation: instance.modelsCacheGeneration, - token, - now, - staleClaimedBefore: now - MODELS_REFRESH_CLAIM_LEASE_MS, - bypassBackoff: mode === 'explicit', - observedActiveToken, - }); - if (outcome.kind === 'backoff' || outcome.kind === 'generation-mismatch') return null; - if (outcome.kind === 'completed') { - const current = await repo.upstreams.getById(instance.upstreamId); - if (current === null - || current.configVersion !== instance.modelsCacheGeneration.configVersion) return null; - if (current.modelsCache === null) throw new Error(`Completed models refresh for ${instance.upstreamId} has no cache`); - instance.modelsCache = current.modelsCache; - return current.modelsCache.models; - } - if (outcome.kind === 'active') { - if (mode === 'background') return null; - if (now >= waitDeadline) throw new Error(`Timed out waiting for models refresh owner for ${instance.upstreamId}`); - observedActiveToken = outcome.token; - await new Promise(resolve => setTimeout(resolve, pollMs)); - pollMs = Math.min(pollMs * 2, ACTIVE_REFRESH_POLL_CAP_MS); - continue; - } - - let models: ProviderModel[]; - try { - models = [...await (loadProvidedModels?.() ?? instance.instance.getProvidedModels(fetcher))]; - } catch (error) { - const failedAt = Date.now(); - const lastError = { message: errorMessage(error), at: failedAt }; - let finalized: boolean; - try { - finalized = await finalizeRefresh( - async () => await repo.upstreams.finalizeModelsRefreshFailure({ - id: instance.upstreamId, - generation: instance.modelsCacheGeneration, - token, - error: lastError, - previousFailureCount: outcome.failureCount, - failedAt, - }), - async () => await repo.upstreams.abandonModelsRefresh({ id: instance.upstreamId, generation: instance.modelsCacheGeneration, token }), - ); - } catch (backoffError) { - throw new AggregateError([error, backoffError], errorMessage(error)); - } - if (finalized) { - if (instance.modelsCache) instance.modelsCache.lastError = lastError; - else instance.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError }; - throw error; - } - if (mode === 'background') throw error; - observedActiveToken = token; - continue; - } - const entry = { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models, lastError: null }; - const finalized = await finalizeRefresh( - async () => await repo.upstreams.finalizeModelsRefreshSuccess({ - id: instance.upstreamId, - generation: instance.modelsCacheGeneration, - token, - cache: entry, - }), - async () => await repo.upstreams.abandonModelsRefresh({ id: instance.upstreamId, generation: instance.modelsCacheGeneration, token }), - ); - if (finalized) { - // The instance is reused across alias targets in one request, so publish - // the finalized snapshot locally as well as durably. - instance.modelsCache = entry; - return models; - } - if (mode === 'background') return models; - observedActiveToken = token; - } -}; - -const inFlightKey = (instance: GatewayProvider): string => { - const generation = instance.modelsCacheGeneration; - return `${instance.upstreamId}\0${generation.configVersion}`; -}; - -export const fetchUpstreamModels = async ( - instance: GatewayProvider, - fetcher: Fetcher, - loadProvidedModels?: () => Promise, -): Promise => { - const key = inFlightKey(instance); - while (true) { - const existing = inFlight.get(key); - if (existing?.mode === 'explicit') { - const joined = await existing.promise; - if (joined === null) throw new Error(`Models refresh generation changed for ${instance.upstreamId}`); - return joined; - } - if (existing?.mode === 'background') { - try { - const joined = await existing.promise; - if (joined !== null) return joined; - } catch { - // The operator request owns a distinct attempt after a background - // failure, and bypasses the cooldown that failure just established. - } - if (inFlight.get(key) === existing) inFlight.delete(key); - continue; - } - const models = await startInFlight(key, 'explicit', () => runClaimedRefresh(instance, fetcher, 'explicit', loadProvidedModels)); - if (models === null) throw new Error(`Failed to acquire models refresh for ${instance.upstreamId}`); - return models; - } -}; - -export const warmUpstreamModels = async ( - instance: GatewayProvider, - fetcher: Fetcher, -): Promise => { - const key = inFlightKey(instance); - const existing = inFlight.get(key); - if (existing) { - const joined = await existing.promise; - if (joined !== null || existing.mode === 'warm') return; - if (inFlight.get(key) === existing) inFlight.delete(key); - } - - await memoInFlight(key, 'warm', () => runClaimedRefresh(instance, fetcher, 'warm')); -}; - -export const scheduleUpstreamModelsRefresh = ( - instance: GatewayProvider, - scheduler: BackgroundScheduler, - fetcher: Fetcher, -): void => { - const key = inFlightKey(instance); - scheduler(memoInFlight(key, 'background', () => runClaimedRefresh(instance, fetcher, 'background')).then(() => {})); -}; - -// Test-only: drop the L1 map so a test's setup is independent of any -// promise the previous test left mid-settle. -export const clearModelsRefreshesForTesting = (): void => { - inFlight.clear(); -}; diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index b13beb692..f256b19a1 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -31,9 +31,10 @@ const enumerateOneUpstreamCandidates = async ( context: { fetcher: Fetcher; scheduler: BackgroundScheduler; + runtimeLocation: string; }, ): Promise<{ candidates: ModelCandidate[]; sawAnyId: boolean; modelsError: boolean }> => { - const { fetcher, scheduler } = context; + const { fetcher, scheduler, runtimeLocation } = context; const cfg = provider.modelPrefix; const lookupIds: string[] = []; if (cfg === null) { @@ -46,7 +47,7 @@ const enumerateOneUpstreamCandidates = async ( } if (lookupIds.length === 0) return { candidates: [], sawAnyId: false, modelsError: false }; - const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(provider, { scheduler, fetcher }); + const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(provider, { scheduler, runtimeLocation }); const disabled = new Set(provider.disabledPublicModelIds); const candidates: ModelCandidate[] = []; let sawAnyId = false; @@ -79,6 +80,7 @@ export const enumerateRealModelCandidates = async ( context: { fetcherForUpstream: (upstreamId: string) => Fetcher; scheduler: BackgroundScheduler; + runtimeLocation: string; clientDisconnectSignal?: AbortSignal; }, ): Promise<{ @@ -86,7 +88,7 @@ export const enumerateRealModelCandidates = async ( readonly sawAnyId: boolean; readonly failedUpstreams: readonly string[]; }> => { - const { fetcherForUpstream, scheduler, clientDisconnectSignal } = context; + const { fetcherForUpstream, scheduler, runtimeLocation, clientDisconnectSignal } = context; const settled = await Promise.allSettled(providers.map(provider => { clientDisconnectSignal?.throwIfAborted(); return enumerateOneUpstreamCandidates( @@ -96,6 +98,7 @@ export const enumerateRealModelCandidates = async ( { fetcher: fetcherForUpstream(provider.upstreamId), scheduler, + runtimeLocation, }, ); })); @@ -225,6 +228,7 @@ export const enumerateModelCandidates = async ({ const resolutionContext = { fetcherForUpstream: createFetcherForUpstream, scheduler, + runtimeLocation, clientDisconnectSignal, }; diff --git a/packages/gateway/src/data-plane/shared/listing/addressable.ts b/packages/gateway/src/data-plane/shared/listing/addressable.ts index 082bab686..5eebd482c 100644 --- a/packages/gateway/src/data-plane/shared/listing/addressable.ts +++ b/packages/gateway/src/data-plane/shared/listing/addressable.ts @@ -55,6 +55,7 @@ export const enumerateAddressableModelIds = async ( upstreamFilter: readonly string[] | null, fetcherForUpstream: (upstreamId: string) => Fetcher, scheduler: BackgroundScheduler, + runtimeLocation: string, preFetchedUpstreams?: readonly StoredUpstreamRecord[], ): Promise => { // Resolve providers once and thread them into the catalog assembly so @@ -65,7 +66,7 @@ export const enumerateAddressableModelIds = async ( // hint behavior on a brand-new gateway. `preFetchedUpstreams` avoids // an additional round-trip when the caller has the list already. const providers = await listModelProviders(upstreamFilter, preFetchedUpstreams); - const { models: realModels, upstreamsByPublicId } = await getModelsFromProviders(providers, fetcherForUpstream, scheduler); + const { models: realModels, upstreamsByPublicId } = await getModelsFromProviders(providers, fetcherForUpstream, scheduler, runtimeLocation); const byId = new Map(realModels.map(model => [model.id, model] as const)); const entries: AddressableIdEntry[] = []; @@ -88,7 +89,7 @@ export const enumerateAddressableModelIds = async ( const addressableOnly = cfg !== null ? cfg.addressable.filter(form => !cfg.listed.includes(form)) : []; if (cfg === null || addressableOnly.length === 0) return [] as AddressableIdEntry[]; - const upstreamModels = readUpstreamModelsSnapshotAndScheduleRefresh(provider, { scheduler, fetcher: fetcherForUpstream(provider.upstreamId) }).models; + const upstreamModels = readUpstreamModelsSnapshotAndScheduleRefresh(provider, { scheduler, runtimeLocation }).models; const disabled = new Set(provider.disabledPublicModelIds); const out: AddressableIdEntry[] = []; diff --git a/packages/gateway/src/execution/handler.ts b/packages/gateway/src/execution/handler.ts index 960862c1f..216214716 100644 --- a/packages/gateway/src/execution/handler.ts +++ b/packages/gateway/src/execution/handler.ts @@ -1,4 +1,4 @@ -import { executeModelsRefresh, type ModelsRefreshExecutionInput } from './models-refresh.ts'; +import { executeModelsRefresh, modelsRefreshExecutionError, type ModelsRefreshExecutionInput } from './models-refresh.ts'; export const handleExecutionRequest = async (request: Request): Promise => { const url = new URL(request.url); @@ -6,8 +6,11 @@ export const handleExecutionRequest = async (request: Request): Promise { @@ -17,10 +20,14 @@ const parseModelsRefreshInput = (value: unknown): ModelsRefreshExecutionInput => if (!Number.isSafeInteger(input.configVersion) || (input.configVersion as number) < 1) throw new TypeError('Models refresh configVersion must be a positive integer'); if (input.cacheEpoch !== null && (!Number.isSafeInteger(input.cacheEpoch) || (input.cacheEpoch as number) < 0)) throw new TypeError('Models refresh cacheEpoch must be a non-negative integer or null'); if (input.runtimeLocation !== null && typeof input.runtimeLocation !== 'string') throw new TypeError('Models refresh runtimeLocation must be a string or null'); + if (typeof input.bypassBackoff !== 'boolean') throw new TypeError('Models refresh bypassBackoff must be a boolean'); + if (typeof input.includeDiscovered !== 'boolean') throw new TypeError('Models refresh includeDiscovered must be a boolean'); return { upstreamId: input.upstreamId, configVersion: input.configVersion as number, cacheEpoch: input.cacheEpoch as number | null, runtimeLocation: input.runtimeLocation as string | null, + bypassBackoff: input.bypassBackoff, + includeDiscovered: input.includeDiscovered, }; }; diff --git a/packages/gateway/src/execution/models-refresh.ts b/packages/gateway/src/execution/models-refresh.ts index 4f39343f0..f7a74913d 100644 --- a/packages/gateway/src/execution/models-refresh.ts +++ b/packages/gateway/src/execution/models-refresh.ts @@ -1,50 +1,134 @@ -import { warmUpstreamModels } from '../data-plane/providers/models-refresh.ts'; import { createProvider } from '../data-plane/providers/registry.ts'; import { createPerRequestFetcher } from '../dial/per-request.ts'; import { getRepo } from '../repo/index.ts'; +import { MODEL_CATALOG_REVISION, modelsCacheGeneration } from '../repo/models-cache-contract.ts'; import type { StoredUpstreamRecord } from '../repo/types.ts'; import { getExecutionCellNamespace } from '../runtime/execution.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; +import { ProviderModelsUnavailableError, type ProviderModel, type UpstreamModelConfig } from '@floway-dev/provider'; +import { assertCustomUpstreamRecord, fetchCustomModels, projectCustomDiscoveredModels, projectCustomModels } from '@floway-dev/provider-custom'; export interface ModelsRefreshExecutionInput { upstreamId: string; configVersion: number; cacheEpoch: number | null; runtimeLocation: string | null; + bypassBackoff: boolean; + includeDiscovered: boolean; } -export const executeModelsRefresh = async (input: ModelsRefreshExecutionInput): Promise => { - const record = await getRepo().upstreams.getById(input.upstreamId); +export type ModelsRefreshExecutionResult = + | { kind: 'refreshed'; discovered?: UpstreamModelConfig[] } + | { kind: 'backoff' | 'generation-mismatch' }; + +interface ModelsRefreshExecutionError { + kind: 'provider-unavailable' | 'error'; + message: string; +} + +export type ModelsRefreshTarget = Pick; + +const cacheEpoch = (record: Pick): number | null => { + const fetchedAt = record.modelsCache?.fetchedAt; + return fetchedAt === undefined || fetchedAt === 0 ? null : fetchedAt; +}; + +export const modelsRefreshTarget = (record: StoredUpstreamRecord): ModelsRefreshTarget => ({ + upstreamId: record.id, + configVersion: record.configVersion, + cacheEpoch: cacheEpoch(record), +}); + +const errorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error); + +export const executeModelsRefresh = async (input: ModelsRefreshExecutionInput): Promise => { + const repo = getRepo().upstreams; + const record = await repo.getById(input.upstreamId); if (record === null || record.configVersion !== input.configVersion - || (record.modelsCache?.fetchedAt ?? null) !== input.cacheEpoch) return; - const fetcherForUpstream = await createPerRequestFetcher(input.runtimeLocation, [record]); - await warmUpstreamModels(createProvider(record), fetcherForUpstream(record.id)); + || cacheEpoch(record) !== input.cacheEpoch) return { kind: 'generation-mismatch' }; + + const generation = modelsCacheGeneration(record); + const beginning = await repo.beginModelsRefresh({ + id: record.id, + generation, + now: Date.now(), + bypassBackoff: input.bypassBackoff, + }); + if (beginning.kind !== 'ready') return beginning; + + try { + const fetcher = (await createPerRequestFetcher(input.runtimeLocation, [record]))(record.id); + let models: ProviderModel[]; + let discovered: UpstreamModelConfig[] | undefined; + if (record.kind === 'custom' && input.includeDiscovered) { + const response = await fetchCustomModels(assertCustomUpstreamRecord(record).config, fetcher); + models = projectCustomModels(record, response); + discovered = projectCustomDiscoveredModels(record, response); + } else { + models = [...await createProvider(record).instance.getProvidedModels(fetcher)]; + } + const published = await repo.publishModelsRefresh({ + id: record.id, + generation, + cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models }, + }); + return published ? { kind: 'refreshed', ...(discovered ? { discovered } : {}) } : { kind: 'generation-mismatch' }; + } catch (error) { + const failedAt = Date.now(); + await repo.recordModelsRefreshFailure({ + id: record.id, + generation, + error: { message: errorMessage(error), at: failedAt }, + previousFailureCount: beginning.failureCount, + failedAt, + }); + throw error; + } }; -export const scheduleModelsRefreshExecution = ( - record: StoredUpstreamRecord, +const executionInput = ( + target: ModelsRefreshTarget, runtimeLocation: string | null, - scheduler: BackgroundScheduler, -): void => { - const cacheEpoch = record.modelsCache?.fetchedAt ?? null; - const cellId = `models:${record.id}:${record.configVersion}:${cacheEpoch ?? 'cold'}`; - const input: ModelsRefreshExecutionInput = { - upstreamId: record.id, - configVersion: record.configVersion, - cacheEpoch, - runtimeLocation, - }; - const execution = getExecutionCellNamespace().fetch(cellId, new Request('https://execution.floway/models/refresh', { + options: Pick, +): ModelsRefreshExecutionInput => ({ + ...target, + runtimeLocation, + ...options, +}); + +const executeThroughCell = async (input: ModelsRefreshExecutionInput): Promise => { + const cellId = `models:${input.upstreamId}:${input.configVersion}:${input.cacheEpoch ?? 'cold'}`; + const response = await getExecutionCellNamespace().fetch(cellId, new Request('https://execution.floway/models/refresh', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(input), - })).then(async response => { - if (response.ok) return; - throw new Error(`Models refresh execution failed with HTTP ${response.status}: ${await response.text()}`); - }).catch(error => { - console.error(`[models] refresh execution failed for ${record.id}`, error); - throw error; - }); - scheduler(execution); + })); + if (response.ok) return await response.json() as ModelsRefreshExecutionResult; + const error = await response.json() as ModelsRefreshExecutionError; + if (error.kind === 'provider-unavailable') throw new ProviderModelsUnavailableError(null); + throw new Error(error.message); +}; + +export const refreshModels = async ( + target: ModelsRefreshTarget, + runtimeLocation: string | null, + options: Pick, +): Promise => { + const input = executionInput(target, runtimeLocation, options); + const result = await executeThroughCell(input); + return options.bypassBackoff && result.kind === 'backoff' ? await executeThroughCell(input) : result; }; + +export const scheduleModelsRefresh = ( + target: ModelsRefreshTarget, + runtimeLocation: string | null, + scheduler: BackgroundScheduler, +): void => { + scheduler(refreshModels(target, runtimeLocation, { bypassBackoff: false, includeDiscovered: false }).then(() => {})); +}; + +export const modelsRefreshExecutionError = (error: unknown): ModelsRefreshExecutionError => ({ + kind: error instanceof ProviderModelsUnavailableError ? 'provider-unavailable' : 'error', + message: errorMessage(error), +}); diff --git a/packages/gateway/src/repo/models-cache-contract.ts b/packages/gateway/src/repo/models-cache-contract.ts index 989e0498e..7ed72f7b4 100644 --- a/packages/gateway/src/repo/models-cache-contract.ts +++ b/packages/gateway/src/repo/models-cache-contract.ts @@ -5,8 +5,8 @@ import type { ModelsCacheGeneration, StoredUpstreamRecord } from './types.ts'; // its serialization changes so older rows become cold across deployments. export const MODEL_CATALOG_REVISION = 5; -// Fetch ownership survives provider-managed state writes such as token -// rotation, but changes whenever static request inputs or egress policy do. +// Refresh publication survives provider-managed state writes such as token +// rotation, but is fenced whenever static request inputs or egress policy change. export const modelsCacheGeneration = ( record: Pick, ): ModelsCacheGeneration => ({ diff --git a/packages/gateway/src/repo/models-refresh-backoff.ts b/packages/gateway/src/repo/models-refresh-backoff.ts new file mode 100644 index 000000000..462b0ca90 --- /dev/null +++ b/packages/gateway/src/repo/models-refresh-backoff.ts @@ -0,0 +1,4 @@ +const BACKOFF_DELAYS_MS = [60_000, 5 * 60_000, 30 * 60_000, 2 * 60 * 60_000] as const; + +export const modelsRefreshRetryAt = (failedAt: number, previousFailureCount: number): number => + failedAt + BACKOFF_DELAYS_MS[Math.min(previousFailureCount, BACKOFF_DELAYS_MS.length - 1)]; diff --git a/packages/gateway/src/repo/models-refresh-contract.ts b/packages/gateway/src/repo/models-refresh-contract.ts deleted file mode 100644 index 001bbbdb5..000000000 --- a/packages/gateway/src/repo/models-refresh-contract.ts +++ /dev/null @@ -1,10 +0,0 @@ -export const MODELS_REFRESH_BACKOFF_BASE_MS = 60_000; -export const MODELS_REFRESH_BACKOFF_CAP_MS = 60 * 60_000; -export const MODELS_REFRESH_BACKOFF_EXPONENT_CAP = 6; -export const MODELS_REFRESH_CLAIM_LEASE_MS = 15 * 60_000; - -export const modelsRefreshRetryAt = (now: number, previousFailureCount: number): number => - now + Math.min( - MODELS_REFRESH_BACKOFF_BASE_MS * (2 ** Math.min(previousFailureCount, MODELS_REFRESH_BACKOFF_EXPONENT_CAP)), - MODELS_REFRESH_BACKOFF_CAP_MS, - ); diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index ad0f14ab4..d8be48213 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -3,7 +3,7 @@ import { SqlExpirationSweepsRepo } from './expiration-sweeps-sql.ts'; import { normalizeFlagOverrides } from './flag-overrides.ts'; import { decodeAliasTargets, decodeAnnouncedMetadata, encodeAliasTargets, encodeAnnouncedMetadata } from './model-alias-codecs.ts'; import { MODEL_CATALOG_REVISION } from './models-cache-contract.ts'; -import { modelsRefreshRetryAt } from './models-refresh-contract.ts'; +import { modelsRefreshRetryAt } from './models-refresh-backoff.ts'; import { querySqlPerformanceOverview } from './performance-overview-sql.ts'; import { normalizeProxyFallbackList } from './proxy-fallback-list.ts'; import { SqlResponsesItemsRepo, SqlResponsesSnapshotsRepo } from './responses-state-sql.ts'; @@ -20,10 +20,9 @@ import type { AgentSetupRenewal, AgentSetupRepository, BackoffRow, - ModelsRefreshClaimInput, - ModelsRefreshClaimResult, + ModelsRefreshBeginInput, + ModelsRefreshBeginResult, ModelsRefreshFailureInput, - ModelsRefreshOwnerInput, ModelsRefreshSuccessInput, ModelAliasesRepo, ModelAliasRecord, @@ -1058,17 +1057,17 @@ class SqlUpstreamRepo implements UpstreamRepo { await this.db.prepare('DELETE FROM upstreams').run(); } - async finalizeModelsRefreshSuccess(input: ModelsRefreshSuccessInput): Promise { - const { id, generation, token, cache } = input; + async publishModelsRefresh(input: ModelsRefreshSuccessInput): Promise { + const { id, generation, cache } = input; const result = await this.db - .prepare("UPDATE upstreams SET models_cache_json = ?, models_refresh_json = NULL WHERE id = ? AND config_version = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") - .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, generation.configVersion, token) + .prepare('UPDATE upstreams SET models_cache_json = ?, models_refresh_json = NULL WHERE id = ? AND config_version = ?') + .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, generation.configVersion) .run(); return (result.meta.changes ?? 0) > 0; } - async finalizeModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise { - const { id, generation, token, error, previousFailureCount, failedAt } = input; + async recordModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise { + const { id, generation, error, previousFailureCount, failedAt } = input; const failureCount = previousFailureCount + 1; const retryAt = modelsRefreshRetryAt(failedAt, previousFailureCount); // A cold failure remains immediately stale while preserving the error for @@ -1078,81 +1077,26 @@ class SqlUpstreamRepo implements UpstreamRepo { .prepare( `UPDATE upstreams SET models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END, - models_refresh_json = json_object('failCount', ?, 'retryAt', ?, 'claimToken', NULL, 'claimedAt', NULL) + models_refresh_json = json_object('failureCount', CAST(? AS INTEGER), 'retryAt', CAST(? AS INTEGER)) WHERE id = ? AND config_version = ? - AND json_extract(models_refresh_json, '$.claimToken') = ? - AND coalesce(json_extract(models_refresh_json, '$.failCount'), 0) = ?`, + AND coalesce(json_extract(models_refresh_json, '$.failureCount'), 0) = ?`, ) - .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, generation.configVersion, token, previousFailureCount) + .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, generation.configVersion, previousFailureCount) .run(); return (result.meta.changes ?? 0) > 0; } - async abandonModelsRefresh(input: ModelsRefreshOwnerInput): Promise { - const { id, generation, token } = input; - const result = await this.db - .prepare("UPDATE upstreams SET models_refresh_json = json_set(models_refresh_json, '$.claimToken', NULL, '$.claimedAt', NULL) WHERE id = ? AND config_version = ? AND json_extract(models_refresh_json, '$.claimToken') = ?") - .bind(id, generation.configVersion, token) - .run(); - return (result.meta.changes ?? 0) > 0; - } - - async claimModelsRefresh(input: ModelsRefreshClaimInput): Promise { - const { id, generation, token, now, staleClaimedBefore, bypassBackoff } = input; - let observedActiveToken = input.observedActiveToken; - while (true) { - const row = await this.db - .prepare( - `UPDATE upstreams - SET models_refresh_json = json_object( - 'failCount', coalesce(json_extract(models_refresh_json, '$.failCount'), 0), - 'retryAt', coalesce(json_extract(models_refresh_json, '$.retryAt'), 0), - 'claimToken', ?, - 'claimedAt', ? - ) - WHERE id = ? AND config_version = ? AND ( - ? IS NULL AND ( - models_refresh_json IS NULL - OR ( - json_extract(models_refresh_json, '$.claimToken') IS NULL - AND (? = 1 OR coalesce(json_extract(models_refresh_json, '$.retryAt'), 0) <= ?) - ) - OR json_extract(models_refresh_json, '$.claimedAt') <= ? - ) - OR ( - ? IS NOT NULL - AND json_extract(models_refresh_json, '$.claimToken') IS NOT NULL - AND json_extract(models_refresh_json, '$.claimedAt') <= ? - ) - ) - RETURNING json_extract(models_refresh_json, '$.failCount') AS fail_count`, - ) - .bind(token, now, id, generation.configVersion, observedActiveToken, sqliteBoolean(bypassBackoff), now, staleClaimedBefore, observedActiveToken, staleClaimedBefore) - .first<{ fail_count: number }>(); - if (row !== null) return { kind: 'claimed', failureCount: row.fail_count }; - - const state = await this.db - .prepare( - `SELECT models_refresh_json, - json_extract(models_refresh_json, '$.retryAt') AS retry_at, - json_extract(models_refresh_json, '$.claimToken') AS claim_token, - json_extract(models_refresh_json, '$.claimedAt') AS claimed_at - FROM upstreams WHERE id = ? AND config_version = ?`, - ) - .bind(id, generation.configVersion) - .first<{ models_refresh_json: string | null; retry_at: number | null; claim_token: string | null; claimed_at: number | null }>(); - if (state === null) return { kind: 'generation-mismatch' }; - if (state.models_refresh_json === null) { - if (observedActiveToken !== null) return { kind: 'completed' }; - continue; - } - if (state.claim_token !== null && state.claimed_at !== null && state.claimed_at > staleClaimedBefore) return { kind: 'active', token: state.claim_token }; - if (observedActiveToken !== null && state.claim_token === null) { - observedActiveToken = null; - continue; - } - if (!bypassBackoff && state.retry_at !== null && state.retry_at > now) return { kind: 'backoff' }; - } + async beginModelsRefresh(input: ModelsRefreshBeginInput): Promise { + const { id, generation, now, bypassBackoff } = input; + const row = await this.db.prepare( + `SELECT + coalesce(json_extract(models_refresh_json, '$.failureCount'), 0) AS failure_count, + coalesce(json_extract(models_refresh_json, '$.retryAt'), 0) AS retry_at + FROM upstreams WHERE id = ? AND config_version = ?`, + ).bind(id, generation.configVersion).first<{ failure_count: number; retry_at: number }>(); + if (row === null) return { kind: 'generation-mismatch' }; + if (!bypassBackoff && row.retry_at > now) return { kind: 'backoff' }; + return { kind: 'ready', failureCount: row.failure_count }; } // Read-modify-write under optimistic concurrency, retried against the winner diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index 3291383b2..1dfe86785 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -362,56 +362,39 @@ export interface UpstreamRepo { // throws. See UpstreamsRepoSlim in @floway-dev/provider for why the change // is a function. saveState(id: string, mutate: (current: unknown) => unknown): Promise; - // Catalog-cache writes are conditional on the row generation that started - // the fetch. A superseded provider can finish serving its own request, but - // cannot publish models or errors under newer credentials/configuration. - finalizeModelsRefreshSuccess(input: ModelsRefreshSuccessInput): Promise; - finalizeModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise; - abandonModelsRefresh(input: ModelsRefreshOwnerInput): Promise; - claimModelsRefresh(input: ModelsRefreshClaimInput): Promise; + beginModelsRefresh(input: ModelsRefreshBeginInput): Promise; + publishModelsRefresh(input: ModelsRefreshSuccessInput): Promise; + recordModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise; } -export interface ModelsRefreshClaimInput { +export interface ModelsRefreshBeginInput { id: string; generation: ModelsCacheGeneration; - token: string; now: number; - staleClaimedBefore: number; bypassBackoff: boolean; - observedActiveToken: string | null; } export interface ModelsRefreshSuccessInput { id: string; generation: ModelsCacheGeneration; - token: string; cache: Omit; } -export interface ModelsRefreshOwnerInput { - id: string; - generation: ModelsCacheGeneration; - token: string; -} - export interface ModelsRefreshFailureInput { id: string; generation: ModelsCacheGeneration; - token: string; error: NonNullable; previousFailureCount: number; failedAt: number; } -export interface ModelsRefreshClaim { - kind: 'claimed'; +export interface ModelsRefreshReady { + kind: 'ready'; failureCount: number; } -export type ModelsRefreshClaimResult = ModelsRefreshClaim - | { kind: 'active'; token: string } +export type ModelsRefreshBeginResult = ModelsRefreshReady | { kind: 'backoff' } - | { kind: 'completed' } | { kind: 'generation-mismatch' }; export interface ModelsCacheGeneration { diff --git a/packages/gateway/src/scheduled/models-refresh.ts b/packages/gateway/src/scheduled/models-refresh.ts index e4b1ebc37..e3620c3d5 100644 --- a/packages/gateway/src/scheduled/models-refresh.ts +++ b/packages/gateway/src/scheduled/models-refresh.ts @@ -1,4 +1,4 @@ -import { scheduleModelsRefreshExecution } from '../execution/models-refresh.ts'; +import { modelsRefreshTarget, scheduleModelsRefresh } from '../execution/models-refresh.ts'; import { getRepo } from '../repo/index.ts'; import { hasLocationIndependentEgress } from '../repo/proxy-fallback-list.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; @@ -8,7 +8,7 @@ export const scheduleModelsCacheRefreshes = async (runtimeLocation: string | nul upstream.enabled && (runtimeLocation !== null || hasLocationIndependentEgress(upstream.proxyFallbackList))); for (const upstream of upstreams) { try { - scheduleModelsRefreshExecution(upstream, runtimeLocation, scheduler); + scheduleModelsRefresh(modelsRefreshTarget(upstream), runtimeLocation, scheduler); } catch (error) { console.error(`[scheduled] models.refresh failed for ${upstream.id}`, error); } From 10334a28fe7ba0eb25b36c70d15d4ad7ea75ae9f Mon Sep 17 00:00:00 2001 From: Menci Date: Fri, 7 Aug 2026 05:40:18 +0800 Subject: [PATCH 46/46] fix(gateway): converge model refresh execution Keep one execution cell per upstream/config/cache target across caller intent and runtime location, while preserving explicit discovery, proxy validation, retry, and automatic backoff semantics. Fence SQL completion by revision-aware cache epoch, advance epochs monotonically, collapse obsolete listing transport plumbing, and rewrite the unpublished refresh migration into its final shape. Add concurrency and error-path coverage from review-and-cleanup findings. --- .../execution-cell-channel-broker_test.ts | 13 +- .../src/execution-cell-channel-broker.ts | 8 +- .../__tests__/node-sqlite-repo_test.ts | 6 +- .../upstreams/copilot-device-login_test.ts | 4 +- .../control-plane/upstreams/routes_test.ts | 34 ++- .../data-plane/providers/catalog_test.ts | 22 +- .../data-plane/providers/models-cache_test.ts | 229 ++++++++++++++++-- .../data-plane/providers/registry_test.ts | 4 +- .../data-plane/providers/resolution_test.ts | 11 +- .../shared/listing/addressable_test.ts | 9 +- packages/gateway/__tests__/repo/memory.ts | 14 +- .../__tests__/repo/models-cache-fixture.ts | 24 +- .../__tests__/repo/models-refresh_test.ts | 76 ++++-- packages/gateway/__tests__/repo/sql_test.ts | 48 ++-- packages/gateway/__tests__/test-utils/app.ts | 2 +- .../0078_upstream_models_refresh.sql | 24 +- .../0080_simplify_models_refresh.sql | 12 - .../src/control-plane/models/routes.ts | 14 +- packages/gateway/src/control-plane/schemas.ts | 2 +- .../shared/save-upstream-for-models.ts | 2 +- .../src/control-plane/upstreams/models.ts | 11 +- .../gateway/src/data-plane/codex/models.ts | 9 +- .../gateway/src/data-plane/models/gemini.ts | 21 +- .../gateway/src/data-plane/models/http.ts | 9 +- .../gateway/src/data-plane/models/load.ts | 12 +- .../src/data-plane/providers/catalog.ts | 61 ++--- .../src/data-plane/providers/models-cache.ts | 22 +- .../src/data-plane/providers/registry.ts | 7 +- .../src/data-plane/providers/resolution.ts | 21 +- .../data-plane/shared/listing/addressable.ts | 46 ++-- packages/gateway/src/dial/per-request.ts | 53 +++- packages/gateway/src/execution/handler.ts | 19 +- .../gateway/src/execution/models-refresh.ts | 161 +++++++----- packages/gateway/src/index.ts | 2 +- .../gateway/src/repo/models-cache-contract.ts | 10 - packages/gateway/src/repo/sql.ts | 27 ++- packages/gateway/src/repo/types.ts | 33 +-- .../gateway/src/scheduled/models-refresh.ts | 5 +- 38 files changed, 655 insertions(+), 432 deletions(-) delete mode 100644 packages/gateway/migrations/0080_simplify_models_refresh.sql diff --git a/apps/platform-cloudflare/__tests__/execution-cell-channel-broker_test.ts b/apps/platform-cloudflare/__tests__/execution-cell-channel-broker_test.ts index 3c08ba352..f04d627ec 100644 --- a/apps/platform-cloudflare/__tests__/execution-cell-channel-broker_test.ts +++ b/apps/platform-cloudflare/__tests__/execution-cell-channel-broker_test.ts @@ -49,11 +49,14 @@ const buildNamespace = ( socket: FakeServerSocket, broadcasts: string[] = [], closeAlls: string[] = [], - fetches?: { count: number }, + fetches?: { count: number; cellIds?: string[] }, ) => { const ns: ExecutionCellNamespace = { - async fetch(_cellId, request) { - if (fetches) fetches.count += 1; + async fetch(cellId, request) { + if (fetches) { + fetches.count += 1; + fetches.cellIds?.push(cellId); + } const url = new URL(request.url); if (url.pathname === '/broadcast' && request.method === 'POST') { broadcasts.push(await request.text()); @@ -133,11 +136,13 @@ test('ExecutionCellChannelBroker.subscribe resolves concurrent reads in socket o test('ExecutionCellChannelBroker.publish encodes the payload through the codec', async () => { const broadcasts: string[] = []; - const ns = buildNamespace(new FakeServerSocket(), broadcasts); + const fetches = { count: 0, cellIds: [] as string[] }; + const ns = buildNamespace(new FakeServerSocket(), broadcasts, [], fetches); const broker = new ExecutionCellChannelBroker(ns, stringCodec); await broker.publish('k', 'frame-a'); assertEquals(broadcasts.length, 1); assertEquals(broadcasts[0], 'frame-a'); + assertEquals(fetches.cellIds, [JSON.stringify(['broadcast', 'k'])]); }); test('ExecutionCellChannelBroker.closeChannel forwards the reason to the actor', async () => { diff --git a/apps/platform-cloudflare/src/execution-cell-channel-broker.ts b/apps/platform-cloudflare/src/execution-cell-channel-broker.ts index fda372f42..2a2e5e087 100644 --- a/apps/platform-cloudflare/src/execution-cell-channel-broker.ts +++ b/apps/platform-cloudflare/src/execution-cell-channel-broker.ts @@ -1,5 +1,7 @@ import { iterateReadableStream, type ChannelBroker, type ChannelCodec, type ExecutionCellNamespace } from '@floway-dev/platform'; +const broadcastCellId = (channelId: string): string => JSON.stringify(['broadcast', channelId]); + export class ExecutionCellChannelBroker implements ChannelBroker { constructor( private readonly cells: ExecutionCellNamespace, @@ -7,7 +9,7 @@ export class ExecutionCellChannelBroker implements ChannelBroker { ) {} async publish(channelId: string, payload: T): Promise { - const response = await this.cells.fetch(channelId, new Request('https://execution.do/broadcast', { + const response = await this.cells.fetch(broadcastCellId(channelId), new Request('https://execution.do/broadcast', { method: 'POST', body: this.codec.encode(payload), })); @@ -15,7 +17,7 @@ export class ExecutionCellChannelBroker implements ChannelBroker { } async closeChannel(channelId: string, reason: string): Promise { - const response = await this.cells.fetch(channelId, new Request('https://execution.do/broadcast/close', { + const response = await this.cells.fetch(broadcastCellId(channelId), new Request('https://execution.do/broadcast/close', { method: 'POST', body: reason, })); @@ -101,7 +103,7 @@ const iterateFromExecutionSocket = ( pull = flushError; const openPromise = (async (): Promise => { - const response = await cells.fetch(channelId, new Request('https://execution.do/broadcast', { + const response = await cells.fetch(broadcastCellId(channelId), new Request('https://execution.do/broadcast', { headers: { Upgrade: 'websocket' }, })); if (response.status !== 101) { diff --git a/apps/platform-node/__tests__/node-sqlite-repo_test.ts b/apps/platform-node/__tests__/node-sqlite-repo_test.ts index a4a5d2f8e..3f02be7c8 100644 --- a/apps/platform-node/__tests__/node-sqlite-repo_test.ts +++ b/apps/platform-node/__tests__/node-sqlite-repo_test.ts @@ -2,7 +2,7 @@ import { test } from 'vitest'; import { applyMigrations } from '../src/migrate.ts'; import { createNodeSqliteDatabase } from '../src/node-sqlite-database.ts'; -import { MODEL_CATALOG_REVISION, modelsCacheGeneration, SqlRepo } from '@floway-dev/gateway'; +import { MODEL_CATALOG_REVISION, SqlRepo } from '@floway-dev/gateway'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; // The repo layer's own suite runs against sql.js, which — like D1 — coerces a @@ -97,10 +97,10 @@ test('repository JSON codecs round-trip upstream, alias, and Responses state thr await repo.upstreams.save(upstreamRecord); const storedUpstream = await repo.upstreams.getById(upstreamRecord.id); if (storedUpstream === null) throw new Error('expected stored upstream fixture'); - const cacheGeneration = modelsCacheGeneration(storedUpstream); await repo.upstreams.publishModelsRefresh({ id: 'up_node', - generation: cacheGeneration, + configVersion: storedUpstream.configVersion, + cacheEpoch: 0, cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_786_000_000_000, diff --git a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts index 50d0c17a4..ce01da893 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/copilot-device-login_test.ts @@ -15,7 +15,7 @@ vi.mock('../../../src/execution/models-refresh.ts', async importOriginal => ({ }, })); -import { seedModelsCache, storedModelsCacheGeneration } from '../../repo/models-cache-fixture.ts'; +import { seedModelsCache, storedModelsRefreshIdentity } from '../../repo/models-cache-fixture.ts'; import { buildCopilotUpstreamRecord, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, assertStringIncludes, jsonResponse, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; @@ -370,7 +370,7 @@ test('/api/upstreams/copilot/oauth/device-login/poll clears the previous identit const existing = buildCopilotUpstreamRecord(githubAccount, { id: 'up_switch_identity' }); await repo.upstreams.deleteAll(); await repo.upstreams.save(existing); - await seedModelsCache(repo.upstreams, existing.id, await storedModelsCacheGeneration(repo.upstreams, existing.id), { + await seedModelsCache(repo.upstreams, existing.id, await storedModelsRefreshIdentity(repo.upstreams, existing.id), { revision: 1, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'old-tenant-model' })], diff --git a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts index cc1bcf1b4..b513ca458 100644 --- a/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts +++ b/packages/gateway/__tests__/control-plane/upstreams/routes_test.ts @@ -3,12 +3,11 @@ import { test } from 'vitest'; import { blueprintUpstreamRecord, upstreamRecordToFullJson } from '../../../src/control-plane/upstreams/serialize.ts'; import { MODEL_LISTING_FAILURE_CODE } from '../../../src/data-plane/models/shared.ts'; import { MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; -import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; import type { StoredUpstreamRecord } from '../../../src/repo/types.ts'; -import { seedModelsCache, seedModelsCacheError, storedModelsCacheGeneration } from '../../repo/models-cache-fixture.ts'; -import { MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; +import { modelsRefreshIdentity, seedModelsCache, seedModelsCacheError, storedModelsRefreshIdentity } from '../../repo/models-cache-fixture.ts'; +import { buildCustomUpstreamRecord, MOCKED_FETCH_EGRESS, requestApp, setupAppTest } from '../../test-utils/app.ts'; import type { UpstreamProviderKind, UpstreamRecord } from '@floway-dev/provider'; -import { assertEquals, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; +import { assertEquals, assertStringIncludes, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; type JsonObject = Record; @@ -286,7 +285,7 @@ test('PATCH /api/upstreams preserves omitted secrets and re-warms the models cac // Plant a stale row so the post-PATCH read can verify the warm overwrote // it with the new upstream-supplied catalog rather than leaving the old // models in place. - await seedModelsCache(repo.upstreams, created.id, await getCacheGeneration(repo, created.id), { + await seedModelsCache(repo.upstreams, created.id, await getRefreshIdentity(repo, created.id), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1, models: [{ id: 'stale-model', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], @@ -448,18 +447,17 @@ test('GET /api/upstreams attaches models-cache freshness to every row', async () await repo.upstreams.save(warmRecord); await repo.upstreams.save(failedRecord); - await seedModelsCache(repo.upstreams, 'up_warm', await storedModelsCacheGeneration(repo.upstreams, 'up_warm'), { + await seedModelsCache(repo.upstreams, 'up_warm', await storedModelsRefreshIdentity(repo.upstreams, 'up_warm'), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [{ id: 'm1', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], }); - const failedGeneration = await storedModelsCacheGeneration(repo.upstreams, 'up_failed'); - await seedModelsCache(repo.upstreams, 'up_failed', failedGeneration, { + await seedModelsCache(repo.upstreams, 'up_failed', await storedModelsRefreshIdentity(repo.upstreams, 'up_failed'), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [{ id: 'm1', kind: 'chat', endpoints: {}, enabledFlags: new Set(), limits: {} }], }); - await seedModelsCacheError(repo.upstreams, 'up_failed', failedGeneration, { message: 'boom', at: 1_700_000_500_000 }); + await seedModelsCacheError(repo.upstreams, 'up_failed', await storedModelsRefreshIdentity(repo.upstreams, 'up_failed'), { message: 'boom', at: 1_700_000_500_000 }); const list = await requestApp('/api/upstreams', { headers: { 'x-floway-session': adminSession } }); assertEquals(list.status, 200); @@ -496,7 +494,7 @@ test('GET /api/upstream-options returns the minimal picker shape to admin and no }); // A disabled upstream is absent from the live catalog, so the picker's count // comes from the catalog it stored while it was on. - await seedModelsCache(repo.upstreams, 'up_disabled_custom', await getCacheGeneration(repo, 'up_disabled_custom'), { + await seedModelsCache(repo.upstreams, 'up_disabled_custom', await getRefreshIdentity(repo, 'up_disabled_custom'), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [ @@ -715,6 +713,18 @@ test('POST /api/upstreams/:id/list-models rejects a missing saved upstream', asy assertEquals(response.status, 404); }); +test('POST /api/upstreams/:id/list-models rejects an unknown saved proxy', async () => { + const { repo, adminSession } = await setupAppTest(); + await repo.upstreams.save(buildCustomUpstreamRecord({ id: 'up_bad_proxy', proxyFallbackList: [{ id: 'missing', colos: ['NRT'] }] })); + + const response = await requestApp('/api/upstreams/up_bad_proxy/list-models', { + method: 'POST', + headers: { 'x-floway-session': adminSession }, + }); + assertEquals(response.status, 400); + assertStringIncludes(JSON.stringify(await response.json()), 'unknown proxy id'); +}); + test('POST /api/upstreams/preview-models rejects an invalid kind with 400', async () => { const { adminSession } = await setupAppTest(); @@ -869,9 +879,9 @@ const getRecord = async (repo: { upstreams: { getById: (id: string) => Promise Promise } }, id: string) => { +const getRefreshIdentity = async (repo: { upstreams: { getById: (id: string) => Promise } }, id: string) => { const record = await getRecord(repo, id); - return modelsCacheGeneration(record); + return modelsRefreshIdentity(record); }; test('POST /api/upstreams/codex/oauth/authorize-url stamps SPA-provided challenge + state into the auth.openai.com URL', async () => { diff --git a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts index af89693fa..57e5ab16a 100644 --- a/packages/gateway/__tests__/data-plane/providers/catalog_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/catalog_test.ts @@ -3,8 +3,9 @@ import { describe, expect, test, vi } from 'vitest'; import { compareModelIds, getModelsFromProviders } from '../../../src/data-plane/providers/catalog.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; import { enumerateModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; +import { createModelsRefreshScheduler } from '../../../src/execution/models-refresh.ts'; import { buildCustomUpstreamRecord, copilotModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; -import { directFetcher, type InternalModel, type ProviderModel } from '@floway-dev/provider'; +import type { InternalModel, ProviderModel } from '@floway-dev/provider'; import { assertEquals, jsonResponse, withMockedFetch as withMockedFetchRaw } from '@floway-dev/test-utils'; const withMockedFetch = ( @@ -25,6 +26,7 @@ const sortedIds = (ids: readonly string[]): string[] => [...ids].sort(compareMod const testScheduler = (promise: Promise): void => { promise.catch(err => console.error('[background]', err)); }; +const scheduleRefresh = createModelsRefreshScheduler('TEST', testScheduler); test('compareModelIds pushes ids containing "/" to the tail', () => { assertEquals(sortedIds(['accounts/msft/x', 'gpt-4o', 'accounts/msft/y', 'claude-opus-4-7']), [ @@ -145,7 +147,7 @@ test('catalog assembly returns the merged catalog plus the per-id upstream index throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const { models, upstreamsByPublicId } = await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST'); + const { models, upstreamsByPublicId } = getModelsFromProviders(await listModelProviders(null), scheduleRefresh); const model = models.find(candidate => candidate.id === 'shared-model'); assertEquals(model?.display_name, 'Shared Model'); @@ -233,7 +235,7 @@ test('disabledPublicModelIds hides models from the catalog and routing, per upst })); await warmModelsForTest(); - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; + const catalog = getModelsFromProviders(await listModelProviders(null), scheduleRefresh).models; assertEquals([...catalog.map(m => m.id)].sort(), ['gpt-keep', 'gpt-shared']); // The solo and override ids resolve to nothing (hidden + unroutable). @@ -289,7 +291,7 @@ test('catalog refresh triggers fan out per upstream in parallel', async () => { await vi.waitFor(() => expect(started.toSorted()).toEqual(upstreams.map(upstream => upstream.host).toSorted())); for (const release of releases.values()) release(); await warming; - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; + const catalog = getModelsFromProviders(await listModelProviders(null), scheduleRefresh).models; assertEquals([...catalog.map(m => m.id)].sort(), ['p1-model', 'p2-model', 'p3-model']); }, @@ -337,7 +339,7 @@ test('catalog assembly: a rejected provider does not block other providers', asy throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; + const catalog = getModelsFromProviders(await listModelProviders(null), scheduleRefresh).models; assertEquals([...catalog.map(m => m.id)].sort(), ['ok-1-model', 'ok-2-model']); }, ); @@ -365,7 +367,7 @@ describe('catalog listing under modelPrefix', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; + const catalog = getModelsFromProviders(await listModelProviders(null), scheduleRefresh).models; assertEquals(catalog.map(m => m.id), ['gpt-4o']); }, ); @@ -390,7 +392,7 @@ describe('catalog listing under modelPrefix', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; + const catalog = getModelsFromProviders(await listModelProviders(null), scheduleRefresh).models; assertEquals(catalog.map(m => m.id), ['or/gpt-4o']); // Prefixed surface gets a synthesized display_name prepending the // upstream's display name so the dashboard tells the operator at a @@ -437,7 +439,7 @@ describe('catalog listing under modelPrefix', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; + const catalog = getModelsFromProviders(await listModelProviders(null), scheduleRefresh).models; assertEquals(catalog.map(m => m.id), ['or/gpt-4o']); const bare = await enumerateModelCandidates({ upstreamIds: null, model: 'gpt-4o', kind: 'chat', scheduler: testScheduler, runtimeLocation: 'TEST' }); @@ -485,7 +487,7 @@ describe('catalog listing under modelPrefix', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; + const catalog = getModelsFromProviders(await listModelProviders(null), scheduleRefresh).models; assertEquals([...catalog.map(m => m.id)].sort(), ['gpt-4o', 'or/gpt-4o']); // Both upstreams enumerate against the bare id: up_plain via its only @@ -577,7 +579,7 @@ describe('catalog listing under modelPrefix', () => { throw new Error(`Unhandled fetch ${request.url}`); }, async () => { - const catalog = (await getModelsFromProviders(await listModelProviders(null), () => directFetcher, testScheduler, 'TEST')).models; + const catalog = getModelsFromProviders(await listModelProviders(null), scheduleRefresh).models; assertEquals([...catalog.map(m => m.id)].sort(), ['gpt-mini', 'or/gpt-mini']); }, ); diff --git a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts index 111c02422..cd5363baf 100644 --- a/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/models-cache_test.ts @@ -2,9 +2,9 @@ import { expect, test, vi } from 'vitest'; import { readUpstreamModelsSnapshotAndScheduleRefresh, MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; import { createProvider } from '../../../src/data-plane/providers/registry.ts'; -import { modelsRefreshTarget, refreshModels } from '../../../src/execution/models-refresh.ts'; -import { modelsCacheGeneration } from '../../../src/repo/models-cache-contract.ts'; -import { seedModelsCache } from '../../repo/models-cache-fixture.ts'; +import { InvalidProxyConfigurationError } from '../../../src/dial/per-request.ts'; +import { createModelsRefreshScheduler, modelsRefreshTarget, refreshModels, refreshModelsExplicit } from '../../../src/execution/models-refresh.ts'; +import { modelsRefreshIdentity, seedModelsCache } from '../../repo/models-cache-fixture.ts'; import { buildCustomUpstreamRecord, setupAppTest } from '../../test-utils/app.ts'; import { ProviderModelsUnavailableError } from '@floway-dev/provider'; import { jsonResponse, stubProviderModel, withMockedFetch } from '@floway-dev/test-utils'; @@ -28,7 +28,7 @@ const captureScheduled = () => { test('a fresh snapshot returns without scheduling work', async () => { const { repo, record } = await setupCustom(); - await seedModelsCache(repo.upstreams, record.id, modelsCacheGeneration(record), { + await seedModelsCache(repo.upstreams, record.id, modelsRefreshIdentity(record), { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models: [stubProviderModel({ id: 'cached' })], @@ -37,10 +37,10 @@ test('a fresh snapshot returns without scheduling work', async () => { if (cached === null) throw new Error('cached upstream missing'); const scheduled = captureScheduled(); - const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(createProvider(cached), { - scheduler: scheduled.scheduler, - runtimeLocation: 'TEST', - }); + const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh( + createProvider(cached), + createModelsRefreshScheduler('TEST', scheduled.scheduler), + ); expect(snapshot.models.map(model => model.id)).toEqual(['cached']); expect(scheduled.promises).toEqual([]); @@ -48,7 +48,7 @@ test('a fresh snapshot returns without scheduling work', async () => { test('a stale snapshot returns immediately and refreshes through the execution cell', async () => { const { repo, record } = await setupCustom(); - await seedModelsCache(repo.upstreams, record.id, modelsCacheGeneration(record), { + await seedModelsCache(repo.upstreams, record.id, modelsRefreshIdentity(record), { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now() - 11 * 60_000, models: [stubProviderModel({ id: 'stale' })], @@ -60,10 +60,10 @@ test('a stale snapshot returns immediately and refreshes through the execution c await withMockedFetch( () => jsonResponse({ object: 'list', data: [{ id: 'fresh' }] }), async () => { - const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(createProvider(stale), { - scheduler: scheduled.scheduler, - runtimeLocation: 'TEST', - }); + const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh( + createProvider(stale), + createModelsRefreshScheduler('TEST', scheduled.scheduler), + ); expect(snapshot.models.map(model => model.id)).toEqual(['stale']); await Promise.all(scheduled.promises); }, @@ -79,12 +79,85 @@ test('concurrent callers share one upstream fetch', async () => { await withMockedFetch(fetch, async () => { const target = modelsRefreshTarget(record); - const first = refreshModels(target, 'TEST', { bypassBackoff: true, includeDiscovered: false }); - const second = refreshModels(target, 'TEST', { bypassBackoff: true, includeDiscovered: false }); + const first = refreshModels(target, 'TEST'); + const second = refreshModels(target, 'TEST'); + await vi.waitFor(() => expect(fetch).toHaveBeenCalledTimes(1)); + release!(jsonResponse({ object: 'list', data: [{ id: 'shared' }] })); + await expect(Promise.all([first, second])).resolves.toEqual([ + expect.objectContaining({ kind: 'refreshed' }), + expect.objectContaining({ kind: 'refreshed' }), + ]); + }); +}); + +test('automatic and explicit callers across locations share one base cell', async () => { + const { record } = await setupCustom(); + let release: ((response: Response) => void) | undefined; + const fetch = vi.fn(() => new Promise(resolve => { release = resolve; })); + + await withMockedFetch(fetch, async () => { + const target = modelsRefreshTarget(record); + const automatic = refreshModels(target, 'SIN'); + const explicit = refreshModelsExplicit(target, 'NRT', true); + await vi.waitFor(() => expect(fetch).toHaveBeenCalledTimes(1)); + release!(jsonResponse({ object: 'list', data: [{ id: 'shared' }] })); + await expect(Promise.all([automatic, explicit])).resolves.toEqual([ + expect.objectContaining({ kind: 'refreshed', discovered: [expect.objectContaining({ upstreamModelId: 'shared' })] }), + expect.objectContaining({ kind: 'refreshed', discovered: [expect.objectContaining({ upstreamModelId: 'shared' })] }), + ]); + }); +}); + +test('explicit join still validates proxy configuration excluded from the automatic owner location', async () => { + const { repo, record } = await setupCustom(); + await repo.upstreams.save({ + ...record, + proxyFallbackList: [{ id: 'missing', colos: ['NRT'] }, { id: 'direct_fetch' }], + }); + const configured = await repo.upstreams.getById(record.id); + if (configured === null) throw new Error('configured custom upstream missing'); + let release: ((response: Response) => void) | undefined; + const fetch = vi.fn(() => new Promise(resolve => { release = resolve; })); + + await withMockedFetch(fetch, async () => { + const target = modelsRefreshTarget(configured); + const automatic = refreshModels(target, 'SIN'); + const explicit = refreshModelsExplicit(target, 'NRT', true); await vi.waitFor(() => expect(fetch).toHaveBeenCalledTimes(1)); release!(jsonResponse({ object: 'list', data: [{ id: 'shared' }] })); - await expect(Promise.all([first, second])).resolves.toEqual([{ kind: 'refreshed' }, { kind: 'refreshed' }]); + await expect(automatic).resolves.toMatchObject({ kind: 'refreshed', mode: 'automatic' }); + await expect(explicit).rejects.toBeInstanceOf(InvalidProxyConfigurationError); + }); + expect(fetch).toHaveBeenCalledTimes(1); +}); + +test('explicit caller retries after joining a failed automatic refresh', async () => { + const { repo, record } = await setupCustom(); + let failAutomatic: ((response: Response) => void) | undefined; + let stealingAutomatic: Promise | undefined; + let reads = 0; + const getById = repo.upstreams.getById.bind(repo.upstreams); + vi.spyOn(repo.upstreams, 'getById').mockImplementation(async id => { + const current = await getById(id); + reads += 1; + if (reads === 2) stealingAutomatic = refreshModels(modelsRefreshTarget(record), 'HKG'); + return current; + }); + const fetch = vi.fn() + .mockImplementationOnce(() => new Promise(resolve => { failAutomatic = resolve; })) + .mockImplementationOnce(() => jsonResponse({ object: 'list', data: [{ id: 'retried' }] })); + + await withMockedFetch(fetch, async () => { + const target = modelsRefreshTarget(record); + const automatic = refreshModels(target, 'SIN'); + await vi.waitFor(() => expect(fetch).toHaveBeenCalledTimes(1)); + const explicit = refreshModelsExplicit(target, 'NRT', true); + failAutomatic!(new Response('unavailable', { status: 503 })); + await expect(automatic).rejects.toBeInstanceOf(ProviderModelsUnavailableError); + await expect(explicit).resolves.toMatchObject({ kind: 'refreshed', mode: 'explicit', discovered: [{ upstreamModelId: 'retried' }] }); + await stealingAutomatic; }); + expect(fetch).toHaveBeenCalledTimes(2); }); test('background refreshes honor backoff and explicit refreshes bypass it', async () => { @@ -93,17 +166,68 @@ test('background refreshes honor backoff and explicit refreshes bypass it', asyn await withMockedFetch(fetch, async () => { const target = modelsRefreshTarget(record); - await expect(refreshModels(target, 'TEST', { bypassBackoff: false, includeDiscovered: false })) + await expect(refreshModels(target, 'TEST')) .rejects.toBeInstanceOf(ProviderModelsUnavailableError); - await expect(refreshModels(target, 'TEST', { bypassBackoff: false, includeDiscovered: false })) - .resolves.toEqual({ kind: 'backoff' }); - await expect(refreshModels(target, 'TEST', { bypassBackoff: true, includeDiscovered: false })) + await expect(refreshModels(target, 'TEST')) + .resolves.toEqual({ kind: 'backoff', mode: 'automatic' }); + await expect(refreshModelsExplicit(target, 'TEST', false)) .rejects.toBeInstanceOf(ProviderModelsUnavailableError); }); expect(fetch).toHaveBeenCalledTimes(2); }); +test('a clean explicit failure makes one attempt and records one failure', async () => { + const { repo, record } = await setupCustom(); + const fetch = vi.fn(() => new Response('unavailable', { status: 503 })); + + await withMockedFetch(fetch, async () => { + await expect(refreshModelsExplicit(modelsRefreshTarget(record), 'TEST', true)) + .rejects.toBeInstanceOf(ProviderModelsUnavailableError); + }); + + expect(fetch).toHaveBeenCalledTimes(1); + await expect(repo.upstreams.beginModelsRefresh({ + id: record.id, + ...modelsRefreshIdentity(record), + now: Date.now(), + bypassBackoff: true, + })).resolves.toEqual({ kind: 'ready', failureCount: 1 }); +}); + +test('automatic proxy configuration failures are recorded and backed off', async () => { + const { repo, record } = await setupCustom(); + await repo.upstreams.save({ ...record, proxyFallbackList: [{ id: 'missing' }] }); + const invalid = await repo.upstreams.getById(record.id); + if (invalid === null) throw new Error('invalid-proxy upstream missing'); + const target = modelsRefreshTarget(invalid); + + await expect(refreshModels(target, 'TEST')).rejects.toBeInstanceOf(ProviderModelsUnavailableError); + expect((await repo.upstreams.getById(record.id))?.modelsCache?.lastError).not.toBeNull(); + await expect(refreshModels(target, 'TEST')).resolves.toEqual({ kind: 'backoff', mode: 'automatic' }); +}); + +test('failure persistence retains the upstream error when recording also fails', async () => { + const { repo, record } = await setupCustom(); + vi.spyOn(repo.upstreams, 'recordModelsRefreshFailure').mockRejectedValue(new Error('storage unavailable')); + + await withMockedFetch( + () => new Response('unavailable', { status: 503 }), + async () => { + try { + await refreshModels(modelsRefreshTarget(record), 'TEST'); + throw new Error('refresh unexpectedly succeeded'); + } catch (error) { + expect(error).toBeInstanceOf(AggregateError); + expect((error as AggregateError).errors).toEqual([ + expect.any(ProviderModelsUnavailableError), + expect.objectContaining({ message: 'storage unavailable' }), + ]); + } + }, + ); +}); + test('a changed config fences an old execution target before fetching', async () => { const { repo, record } = await setupCustom(); await repo.upstreams.save(buildCustomUpstreamRecord({ @@ -112,18 +236,38 @@ test('a changed config fences an old execution target before fetching', async () const fetch = vi.fn(() => jsonResponse({ object: 'list', data: [] })); await withMockedFetch(fetch, async () => { - await expect(refreshModels(modelsRefreshTarget(record), 'TEST', { bypassBackoff: true, includeDiscovered: false })) - .resolves.toEqual({ kind: 'generation-mismatch' }); + await expect(refreshModelsExplicit(modelsRefreshTarget(record), 'TEST', false)) + .resolves.toEqual({ kind: 'superseded', mode: 'explicit' }); }); expect(fetch).not.toHaveBeenCalled(); }); +test('explicit refresh follows a newer cache epoch under the same config', async () => { + const { repo, record } = await setupCustom(); + const staleTarget = modelsRefreshTarget(record); + await seedModelsCache(repo.upstreams, record.id, modelsRefreshIdentity(record), { + revision: MODEL_CATALOG_REVISION, + fetchedAt: 1_000, + models: [], + }); + const fetch = vi.fn(() => jsonResponse({ object: 'list', data: [{ id: 'current' }] })); + + await withMockedFetch(fetch, async () => { + await expect(refreshModelsExplicit(staleTarget, 'TEST', true)).resolves.toMatchObject({ + kind: 'refreshed', + mode: 'explicit', + discovered: [{ upstreamModelId: 'current' }], + }); + }); + expect(fetch).toHaveBeenCalledTimes(1); +}); + test('custom explicit refresh returns discovered dashboard models from the same fetch', async () => { const { record } = await setupCustom(); await withMockedFetch( () => jsonResponse({ object: 'list', data: [{ id: 'discovered', display_name: 'Discovered' }] }), async () => { - const result = await refreshModels(modelsRefreshTarget(record), 'TEST', { bypassBackoff: true, includeDiscovered: true }); + const result = await refreshModelsExplicit(modelsRefreshTarget(record), 'TEST', true); expect(result).toMatchObject({ kind: 'refreshed', discovered: [{ upstreamModelId: 'discovered', publicModelId: 'discovered', display_name: 'Discovered' }], @@ -131,3 +275,42 @@ test('custom explicit refresh returns discovered dashboard models from the same }, ); }); + +test('explicit discovery retries after an automatic refresh skips disabled fetching', async () => { + const { repo, record } = await setupCustom(); + await repo.upstreams.save({ + ...record, + config: { ...record.config as Record, modelsFetch: { enabled: false } }, + }); + const disabled = await repo.upstreams.getById(record.id); + if (disabled === null) throw new Error('disabled custom upstream missing'); + + await withMockedFetch( + () => jsonResponse({ object: 'list', data: [{ id: 'discovered' }] }), + async () => { + const result = await refreshModelsExplicit(modelsRefreshTarget(disabled), 'TEST', true); + expect(result).toMatchObject({ kind: 'refreshed', discovered: [{ upstreamModelId: 'discovered' }] }); + }, + ); +}); + +test('successful refresh advances the cache epoch monotonically', async () => { + const { repo, record } = await setupCustom(); + await seedModelsCache(repo.upstreams, record.id, modelsRefreshIdentity(record), { + revision: MODEL_CATALOG_REVISION, + fetchedAt: 1_000, + models: [], + }); + const cached = await repo.upstreams.getById(record.id); + if (cached === null) throw new Error('cached custom upstream missing'); + const now = vi.spyOn(Date, 'now').mockReturnValue(1_000); + try { + await withMockedFetch( + () => jsonResponse({ object: 'list', data: [{ id: 'fresh' }] }), + async () => await refreshModels(modelsRefreshTarget(cached), 'TEST'), + ); + } finally { + now.mockRestore(); + } + expect((await repo.upstreams.getById(record.id))?.modelsCache?.fetchedAt).toBe(1_001); +}); diff --git a/packages/gateway/__tests__/data-plane/providers/registry_test.ts b/packages/gateway/__tests__/data-plane/providers/registry_test.ts index cc86ac589..9de75a40d 100644 --- a/packages/gateway/__tests__/data-plane/providers/registry_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/registry_test.ts @@ -2,7 +2,7 @@ import { test } from 'vitest'; import { MODEL_CATALOG_REVISION } from '../../../src/data-plane/providers/models-cache.ts'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; -import { seedModelsCache, storedModelsCacheGeneration } from '../../repo/models-cache-fixture.ts'; +import { seedModelsCache, storedModelsRefreshIdentity } from '../../repo/models-cache-fixture.ts'; import { buildCopilotUpstreamRecord, buildCustomUpstreamRecord, setupAppTest } from '../../test-utils/app.ts'; import { assertEquals, stubProviderModel } from '@floway-dev/test-utils'; @@ -96,7 +96,7 @@ test('listModelProviders carries each row cached catalog onto its instance', asy const cachedRecord = buildCustomUpstreamRecord({ id: 'up_cached', name: 'Cached', sortOrder: 10 }); await repo.upstreams.save(cachedRecord); await repo.upstreams.save(buildCustomUpstreamRecord({ id: 'up_cold', name: 'Cold', sortOrder: 20 })); - await seedModelsCache(repo.upstreams, 'up_cached', await storedModelsCacheGeneration(repo.upstreams, 'up_cached'), { + await seedModelsCache(repo.upstreams, 'up_cached', await storedModelsRefreshIdentity(repo.upstreams, 'up_cached'), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], diff --git a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts index 286f36be8..11e7859c2 100644 --- a/packages/gateway/__tests__/data-plane/providers/resolution_test.ts +++ b/packages/gateway/__tests__/data-plane/providers/resolution_test.ts @@ -2,7 +2,7 @@ import { describe, expect, test, vi } from 'vitest'; import { listModelProviders } from '../../../src/data-plane/providers/registry.ts'; import { enumerateModelCandidates, enumerateRealModelCandidates } from '../../../src/data-plane/providers/resolution.ts'; -import { modelsRefreshTarget, refreshModels } from '../../../src/execution/models-refresh.ts'; +import { createModelsRefreshScheduler, modelsRefreshTarget, refreshModelsExplicit } from '../../../src/execution/models-refresh.ts'; import { buildCustomUpstreamRecord, copilotModels, setupAppTest, warmModelsForTest } from '../../test-utils/app.ts'; import { directFetcher, type InternalModel, type ProviderModel } from '@floway-dev/provider'; import { assertEquals, jsonResponse, withMockedFetch as withMockedFetchRaw } from '@floway-dev/test-utils'; @@ -23,6 +23,7 @@ const realProviderModels = (model: InternalModel | undefined): Record): void => { promise.catch(err => console.error('[background]', err)); }; +const scheduleRefresh = createModelsRefreshScheduler('TEST', testScheduler); test('enumerateModelCandidates blocks a cold catalog fetch after client disconnect', async () => { const { repo } = await setupAppTest(); @@ -269,10 +270,10 @@ test('enumerateRealModelCandidates only loads the selected providers\' catalogs' async () => { const first = await repo.upstreams.getById(providers[0].upstreamId); if (first === null) throw new Error('first upstream missing'); - await refreshModels(modelsRefreshTarget(first), 'TEST', { bypassBackoff: true, includeDiscovered: false }); + await refreshModelsExplicit(modelsRefreshTarget(first), 'TEST', false); const warmed = (await listModelProviders(null)).find(provider => provider.upstreamId === 'up_first'); if (!warmed) throw new Error('warmed provider missing'); - const { candidates } = await enumerateRealModelCandidates('target-model', 'chat', [warmed], { fetcherForUpstream: () => directFetcher, scheduler: testScheduler, runtimeLocation: 'TEST' }); + const { candidates } = await enumerateRealModelCandidates('target-model', 'chat', [warmed], { fetcherForUpstream: () => directFetcher, scheduleRefresh }); assertEquals(candidates[0]?.model.id, 'target-model'); assertEquals(candidates[0]?.provider.upstreamId, 'up_first'); @@ -316,8 +317,8 @@ test('enumerateRealModelCandidates rejects a model id disabled on that upstream await warmModelsForTest(); const providers = await listModelProviders(null); - const enabled = await enumerateRealModelCandidates('enabled-model', 'chat', providers, { fetcherForUpstream: () => directFetcher, scheduler: testScheduler, runtimeLocation: 'TEST' }); - const disabled = await enumerateRealModelCandidates('disabled-model', 'chat', providers, { fetcherForUpstream: () => directFetcher, scheduler: testScheduler, runtimeLocation: 'TEST' }); + const enabled = await enumerateRealModelCandidates('enabled-model', 'chat', providers, { fetcherForUpstream: () => directFetcher, scheduleRefresh }); + const disabled = await enumerateRealModelCandidates('disabled-model', 'chat', providers, { fetcherForUpstream: () => directFetcher, scheduleRefresh }); assertEquals(enabled.candidates[0]?.model.id, 'enabled-model'); assertEquals(disabled.candidates.length, 0); }); diff --git a/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts b/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts index 115f2b502..a01ff3351 100644 --- a/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts +++ b/packages/gateway/__tests__/data-plane/shared/listing/addressable_test.ts @@ -1,13 +1,14 @@ import { describe, expect, test } from 'vitest'; import { enumerateAddressableModelIds } from '../../../../src/data-plane/shared/listing/addressable.ts'; +import { createModelsRefreshScheduler } from '../../../../src/execution/models-refresh.ts'; import { buildCustomUpstreamRecord, setupAppTest, warmModelsForTest } from '../../../test-utils/app.ts'; -import { directFetcher } from '@floway-dev/provider'; import { jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; const noBackground = (promise: Promise): void => { promise.catch(err => console.error('[background]', err)); }; +const scheduleRefresh = createModelsRefreshScheduler('TEST', noBackground); describe('enumerateAddressableModelIds', () => { test('returns the listed catalog as listed entries when no provider contributes addressable-only forms', async () => { @@ -25,7 +26,7 @@ describe('enumerateAddressableModelIds', () => { }, async () => { await warmModelsForTest(); - const surface = await enumerateAddressableModelIds(null, () => directFetcher, noBackground, 'TEST'); + const surface = await enumerateAddressableModelIds(null, scheduleRefresh); expect(surface.map(e => ({ id: e.id, unlisted: e.unlisted }))).toEqual([ { id: 'shared-model', unlisted: undefined }, ]); @@ -54,7 +55,7 @@ describe('enumerateAddressableModelIds', () => { }, async () => { await warmModelsForTest(); - const surface = await enumerateAddressableModelIds(null, () => directFetcher, noBackground, 'TEST'); + const surface = await enumerateAddressableModelIds(null, scheduleRefresh); const byId = new Map(surface.map(e => [e.id, e])); expect(byId.get('cust/gpt-5.4')?.unlisted).toBeUndefined(); expect(byId.get('gpt-5.4')?.unlisted).toBe(true); @@ -69,7 +70,7 @@ describe('enumerateAddressableModelIds', () => { const { repo } = await setupAppTest(); await repo.upstreams.deleteAll(); - await expect(enumerateAddressableModelIds(null, () => directFetcher, noBackground, 'TEST')) + await expect(enumerateAddressableModelIds(null, scheduleRefresh)) .rejects.toThrow('No upstream provider configured'); }); }); diff --git a/packages/gateway/__tests__/repo/memory.ts b/packages/gateway/__tests__/repo/memory.ts index 4b266dc8c..a451e0b4e 100644 --- a/packages/gateway/__tests__/repo/memory.ts +++ b/packages/gateway/__tests__/repo/memory.ts @@ -748,7 +748,7 @@ class MemoryUpstreamRepo implements UpstreamRepo { return Promise.resolve(found ? cloneUpstreamRecord(found) : null); } - // Mirrors the SQL upsert: config changes advance the generation and clear + // Mirrors the SQL upsert: config changes advance the version and clear // the snapshot; other writes preserve it. New rows always start uncached. save(upstream: UpstreamRecord): Promise { const existing = this.store.get(upstream.id); @@ -833,22 +833,22 @@ class MemoryUpstreamRepo implements UpstreamRepo { } publishModelsRefresh(input: ModelsRefreshSuccessInput): Promise { - const { id, generation, cache } = input; + const { id, configVersion, cacheEpoch, cache } = input; const existing = this.store.get(id); - if (!existing || existing.configVersion !== generation.configVersion) return Promise.resolve(false); + if (!existing || existing.configVersion !== configVersion || (existing.modelsCache?.fetchedAt ?? 0) !== cacheEpoch) return Promise.resolve(false); existing.modelsCache = { revision: cache.revision, fetchedAt: cache.fetchedAt, models: [...cache.models], lastError: null }; this.modelsRefreshes.delete(id); return Promise.resolve(true); } recordModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise { - const { id, generation, error, previousFailureCount, failedAt } = input; + const { id, configVersion, cacheEpoch, error, previousFailureCount, failedAt } = input; const failureCount = previousFailureCount + 1; const retryAt = modelsRefreshRetryAt(failedAt, previousFailureCount); const refresh = this.modelsRefreshes.get(id); if ((refresh?.failureCount ?? 0) !== previousFailureCount) return Promise.resolve(false); const existing = this.store.get(id); - if (!existing || existing.configVersion !== generation.configVersion) return Promise.resolve(false); + if (!existing || existing.configVersion !== configVersion || (existing.modelsCache?.fetchedAt ?? 0) !== cacheEpoch) return Promise.resolve(false); if (existing.modelsCache) existing.modelsCache.lastError = error; else existing.modelsCache = { revision: MODEL_CATALOG_REVISION, fetchedAt: 0, models: [], lastError: error }; this.modelsRefreshes.set(id, { failureCount, retryAt }); @@ -856,9 +856,9 @@ class MemoryUpstreamRepo implements UpstreamRepo { } beginModelsRefresh(input: ModelsRefreshBeginInput): Promise { - const { id, generation, now, bypassBackoff } = input; + const { id, configVersion, cacheEpoch, now, bypassBackoff } = input; const stored = this.store.get(id); - if (!stored || stored.configVersion !== generation.configVersion) return Promise.resolve({ kind: 'generation-mismatch' }); + if (!stored || stored.configVersion !== configVersion || (stored.modelsCache?.fetchedAt ?? 0) !== cacheEpoch) return Promise.resolve({ kind: 'superseded' }); const existing = this.modelsRefreshes.get(id); if (!bypassBackoff && existing !== undefined && existing.retryAt > now) return Promise.resolve({ kind: 'backoff' }); return Promise.resolve({ kind: 'ready', failureCount: existing?.failureCount ?? 0 }); diff --git a/packages/gateway/__tests__/repo/models-cache-fixture.ts b/packages/gateway/__tests__/repo/models-cache-fixture.ts index c765b0550..04d787b8e 100644 --- a/packages/gateway/__tests__/repo/models-cache-fixture.ts +++ b/packages/gateway/__tests__/repo/models-cache-fixture.ts @@ -1,30 +1,36 @@ -import { modelsCacheGeneration } from '../../src/repo/models-cache-contract.ts'; -import type { ModelsCacheGeneration, UpstreamRepo } from '../../src/repo/types.ts'; +import type { ModelsRefreshIdentity, UpstreamRepo } from '../../src/repo/types.ts'; import type { UpstreamModelsCache } from '@floway-dev/provider'; -export const storedModelsCacheGeneration = async ( +type ModelsRefreshRowIdentity = Omit; + +export const modelsRefreshIdentity = (record: { configVersion: number; modelsCache: UpstreamModelsCache | null }): ModelsRefreshRowIdentity => ({ + configVersion: record.configVersion, + cacheEpoch: record.modelsCache?.fetchedAt ?? 0, +}); + +export const storedModelsRefreshIdentity = async ( repo: UpstreamRepo, id: string, -): Promise => { +): Promise => { const record = await repo.getById(id); if (record === null) throw new Error(`Upstream ${id} not found`); - return modelsCacheGeneration(record); + return modelsRefreshIdentity(record); }; export const seedModelsCache = async ( repo: UpstreamRepo, id: string, - generation: ModelsCacheGeneration, + identity: ModelsRefreshRowIdentity, cache: Omit, ): Promise => { - return await repo.publishModelsRefresh({ id, generation, cache }); + return await repo.publishModelsRefresh({ id, ...identity, cache }); }; export const seedModelsCacheError = async ( repo: UpstreamRepo, id: string, - generation: ModelsCacheGeneration, + identity: ModelsRefreshRowIdentity, error: NonNullable, ): Promise => { - return await repo.recordModelsRefreshFailure({ id, generation, error, previousFailureCount: 0, failedAt: -60_000 }); + return await repo.recordModelsRefreshFailure({ id, ...identity, error, previousFailureCount: 0, failedAt: -60_000 }); }; diff --git a/packages/gateway/__tests__/repo/models-refresh_test.ts b/packages/gateway/__tests__/repo/models-refresh_test.ts index 52a034868..dbf46389b 100644 --- a/packages/gateway/__tests__/repo/models-refresh_test.ts +++ b/packages/gateway/__tests__/repo/models-refresh_test.ts @@ -1,8 +1,9 @@ import { describe, expect, test } from 'vitest'; import { InMemoryRepo } from './memory.ts'; +import { modelsRefreshIdentity } from './models-cache-fixture.ts'; import { createSqliteTestDb } from './test-sqlite.ts'; -import { MODEL_CATALOG_REVISION, modelsCacheGeneration } from '../../src/repo/models-cache-contract.ts'; +import { MODEL_CATALOG_REVISION } from '../../src/repo/models-cache-contract.ts'; import { modelsRefreshRetryAt } from '../../src/repo/models-refresh-backoff.ts'; import { SqlRepo } from '../../src/repo/sql.ts'; import type { Repo, StoredUpstreamRecord } from '../../src/repo/types.ts'; @@ -35,24 +36,24 @@ describe.each(factories)('%s models refresh persistence', (_name, createRepo) => test('applies retry backoff and lets an explicit refresh bypass it', async () => { const repo = (await createRepo()).upstreams; await repo.save(record); - const generation = modelsCacheGeneration(record); + const identity = modelsRefreshIdentity(record); let now = 1_800_000_000_000; for (const [failureCount, minutes] of [1, 5, 30, 120, 120].entries()) { - await expect(repo.beginModelsRefresh({ id: record.id, generation, now, bypassBackoff: false })) + await expect(repo.beginModelsRefresh({ id: record.id, ...identity, now, bypassBackoff: false })) .resolves.toEqual({ kind: 'ready', failureCount }); await expect(repo.recordModelsRefreshFailure({ id: record.id, - generation, + ...identity, error: { message: 'failure', at: now }, previousFailureCount: failureCount, failedAt: now, })).resolves.toBe(true); const retryAt = modelsRefreshRetryAt(now, failureCount); expect(retryAt - now).toBe(minutes * 60_000); - await expect(repo.beginModelsRefresh({ id: record.id, generation, now: retryAt - 1, bypassBackoff: false })) + await expect(repo.beginModelsRefresh({ id: record.id, ...identity, now: retryAt - 1, bypassBackoff: false })) .resolves.toEqual({ kind: 'backoff' }); - await expect(repo.beginModelsRefresh({ id: record.id, generation, now: retryAt - 1, bypassBackoff: true })) + await expect(repo.beginModelsRefresh({ id: record.id, ...identity, now: retryAt - 1, bypassBackoff: true })) .resolves.toEqual({ kind: 'ready', failureCount: failureCount + 1 }); now = retryAt; } @@ -61,16 +62,18 @@ describe.each(factories)('%s models refresh persistence', (_name, createRepo) => test('success publishes the catalog and clears failure backoff', async () => { const repo = (await createRepo()).upstreams; await repo.save(record); - const generation = modelsCacheGeneration(record); + const identity = modelsRefreshIdentity(record); const now = 1_800_000_000_000; - await repo.recordModelsRefreshFailure({ id: record.id, generation, error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); + await repo.recordModelsRefreshFailure({ id: record.id, ...identity, error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); await expect(repo.publishModelsRefresh({ id: record.id, - generation, + ...identity, cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: now + 1, models: [] }, })).resolves.toBe(true); - await expect(repo.beginModelsRefresh({ id: record.id, generation, now: now + 2, bypassBackoff: false })) + const refreshed = await repo.getById(record.id); + if (refreshed === null) throw new Error('refreshed upstream missing'); + await expect(repo.beginModelsRefresh({ id: record.id, ...modelsRefreshIdentity(refreshed), now: now + 2, bypassBackoff: false })) .resolves.toEqual({ kind: 'ready', failureCount: 0 }); expect((await repo.getById(record.id))?.modelsCache).toMatchObject({ fetchedAt: now + 1, lastError: null }); }); @@ -78,32 +81,67 @@ describe.each(factories)('%s models refresh persistence', (_name, createRepo) => test('config changes fence stale success and failure publication', async () => { const repo = (await createRepo()).upstreams; await repo.save(record); - const generation = modelsCacheGeneration(record); + const identity = modelsRefreshIdentity(record); const current = await repo.getById(record.id); if (current === null) throw new Error('upstream row missing'); await repo.replaceForModels({ previous: current, upstream: { ...current, config: { tenant: 'next' } } }); - await expect(repo.beginModelsRefresh({ id: record.id, generation, now: 1, bypassBackoff: true })) - .resolves.toEqual({ kind: 'generation-mismatch' }); - await expect(repo.publishModelsRefresh({ id: record.id, generation, cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: 1, models: [] } })) + await expect(repo.beginModelsRefresh({ id: record.id, ...identity, now: 1, bypassBackoff: true })) + .resolves.toEqual({ kind: 'superseded' }); + await expect(repo.publishModelsRefresh({ id: record.id, ...identity, cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: 1, models: [] } })) .resolves.toBe(false); - await expect(repo.recordModelsRefreshFailure({ id: record.id, generation, error: { message: 'old', at: 1 }, previousFailureCount: 0, failedAt: 1 })) + await expect(repo.recordModelsRefreshFailure({ id: record.id, ...identity, error: { message: 'old', at: 1 }, previousFailureCount: 0, failedAt: 1 })) .resolves.toBe(false); }); - test('state and metadata changes preserve the generation and backoff', async () => { + test('cache publication fences stale completions from the same config', async () => { const repo = (await createRepo()).upstreams; await repo.save(record); - const generation = modelsCacheGeneration(record); + const cold = modelsRefreshIdentity(record); + await expect(repo.publishModelsRefresh({ + id: record.id, + ...cold, + cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: 10, models: [] }, + })).resolves.toBe(true); + await expect(repo.recordModelsRefreshFailure({ + id: record.id, + ...cold, + error: { message: 'stale failure', at: 11 }, + previousFailureCount: 0, + failedAt: 11, + })).resolves.toBe(false); + + const fresh = await repo.getById(record.id); + if (fresh === null) throw new Error('fresh upstream missing'); + expect(fresh.modelsCache?.lastError).toBeNull(); + await repo.recordModelsRefreshFailure({ + id: record.id, + ...modelsRefreshIdentity(fresh), + error: { message: 'current failure', at: 12 }, + previousFailureCount: 0, + failedAt: 12, + }); + await expect(repo.publishModelsRefresh({ + id: record.id, + ...cold, + cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: 13, models: [] }, + })).resolves.toBe(false); + expect((await repo.getById(record.id))?.modelsCache).toMatchObject({ fetchedAt: 10, lastError: { message: 'current failure' } }); + }); + + test('state and metadata changes preserve the config version and backoff', async () => { + const repo = (await createRepo()).upstreams; + await repo.save(record); + const identity = modelsRefreshIdentity(record); const now = 1_800_000_000_000; - await repo.recordModelsRefreshFailure({ id: record.id, generation, error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); + await repo.recordModelsRefreshFailure({ id: record.id, ...identity, error: { message: 'failure', at: now }, previousFailureCount: 0, failedAt: now }); await repo.saveState(record.id, () => ({ credential: 'rotated' })); const current = await repo.getById(record.id); if (current === null) throw new Error('upstream row missing'); await repo.replaceForModels({ previous: current, upstream: { ...current, name: 'Renamed' } }); expect((await repo.getById(record.id))?.configVersion).toBe(1); - await expect(repo.beginModelsRefresh({ id: record.id, generation, now: now + 1, bypassBackoff: false })) + await expect(repo.beginModelsRefresh({ id: record.id, ...modelsRefreshIdentity(current), now: now + 1, bypassBackoff: false })) .resolves.toEqual({ kind: 'backoff' }); }); diff --git a/packages/gateway/__tests__/repo/sql_test.ts b/packages/gateway/__tests__/repo/sql_test.ts index 46bc13cad..e5724699c 100644 --- a/packages/gateway/__tests__/repo/sql_test.ts +++ b/packages/gateway/__tests__/repo/sql_test.ts @@ -1,9 +1,8 @@ import { test } from 'vitest'; -import { seedModelsCache, seedModelsCacheError } from './models-cache-fixture.ts'; +import { modelsRefreshIdentity, seedModelsCache, seedModelsCacheError, storedModelsRefreshIdentity } from './models-cache-fixture.ts'; import { createSqliteTestDb } from './test-sqlite.ts'; import { MODEL_CATALOG_REVISION } from '../../src/data-plane/providers/models-cache.ts'; -import { modelsCacheGeneration } from '../../src/repo/models-cache-contract.ts'; import { SqlRepo, UPSTREAM_STATE_WRITE_ATTEMPTS } from '../../src/repo/sql.ts'; import type { StoredUpstreamRecord } from '../../src/repo/types.ts'; import type { SqlDatabase, SqlPreparedStatement } from '@floway-dev/platform'; @@ -30,7 +29,7 @@ const baseRecord = (overrides: Partial = {}): StoredUpstre hue: 210, ...overrides, }); -const generationFor = modelsCacheGeneration; +const identityFor = modelsRefreshIdentity; const ownValue = (value: unknown, key: string): unknown => { if (value === null || typeof value !== 'object' || !Object.hasOwn(value, key)) { @@ -54,7 +53,7 @@ test('SQL upstream repo preserves nested own __proto__ fields in opaque config a test('SQL upstream repo round-trips the cached catalog and its revision', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', identityFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], @@ -67,6 +66,25 @@ test('SQL upstream repo round-trips the cached catalog and its revision', async assertEquals(cached?.lastError, null); }); +test('SQL refresh replaces a catalog from an old schema revision', async () => { + const db = await createSqliteTestDb(); + const repo = new SqlRepo(db).upstreams; + await repo.save(baseRecord()); + await db.prepare('UPDATE upstreams SET models_cache_json = ? WHERE id = ?').bind(JSON.stringify({ + revision: MODEL_CATALOG_REVISION - 1, + fetchedAt: 1_700_000_000_000, + models: [], + lastError: null, + }), 'up_test').run(); + + assertEquals((await repo.getById('up_test'))?.modelsCache, null); + assertEquals(await seedModelsCache(repo, 'up_test', identityFor(baseRecord()), { + revision: MODEL_CATALOG_REVISION, + fetchedAt: 1_700_000_000_001, + models: [stubProviderModel({ id: 'current-model' })], + }), true); +}); + test('SQL upstream repo rejects shape-invalid JSON in a cached catalog with row context', async () => { const db = await createSqliteTestDb(); const repo = new SqlRepo(db).upstreams; @@ -90,7 +108,7 @@ test('SQL upstream repo rejects shape-invalid JSON in a cached catalog with row test('SQL upstream repo preserves opaque provider data while restoring only the model enabledFlags Set', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', identityFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ @@ -122,18 +140,18 @@ test('SQL upstream repo hydrates deeply nested opaque provider data without recu test('SQL upstream repo annotates a cached catalog and successful publication clears the error', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', identityFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], }); - await seedModelsCacheError(repo, 'up_test', generationFor(baseRecord()), { message: 'boom', at: 1_700_000_500_000 }); + await seedModelsCacheError(repo, 'up_test', await storedModelsRefreshIdentity(repo, 'up_test'), { message: 'boom', at: 1_700_000_500_000 }); const annotated = (await repo.getById('up_test'))?.modelsCache; assertEquals(annotated?.lastError, { message: 'boom', at: 1_700_000_500_000 }); assertEquals(annotated?.models.map(model => model.id), ['cached-model']); - await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', await storedModelsRefreshIdentity(repo, 'up_test'), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_001_000_000, models: [stubProviderModel({ id: 'refreshed-model' })], @@ -145,7 +163,7 @@ test('SQL upstream repo persists an immediately-stale empty catalog on first fai const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await seedModelsCacheError(repo, 'up_test', generationFor(baseRecord()), { message: 'boom', at: 1_700_000_500_000 }); + await seedModelsCacheError(repo, 'up_test', identityFor(baseRecord()), { message: 'boom', at: 1_700_000_500_000 }); assertEquals((await repo.getById('up_test'))?.modelsCache, { revision: MODEL_CATALOG_REVISION, @@ -158,7 +176,7 @@ test('SQL upstream repo persists an immediately-stale empty catalog on first fai test('SQL upstream repo catalog-aware replacement can clear the cached catalog atomically', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', identityFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], @@ -174,18 +192,18 @@ test('SQL upstream repo catalog-aware replacement can clear the cached catalog a assertEquals(stored?.name, 'New identity'); assertEquals(stored?.modelsCache, null); - const staleCatalogSaved = await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { + const staleCatalogSaved = await seedModelsCache(repo, 'up_test', identityFor(baseRecord()), { revision: 7, fetchedAt: 1_700_001_000_000, models: [stubProviderModel({ id: 'stale-model' })], }); - const staleErrorSaved = await seedModelsCacheError(repo, 'up_test', generationFor(baseRecord()), { message: 'stale error', at: 1_700_001_000_000 }); + const staleErrorSaved = await seedModelsCacheError(repo, 'up_test', identityFor(baseRecord()), { message: 'stale error', at: 1_700_001_000_000 }); assertEquals(staleCatalogSaved, false); assertEquals(staleErrorSaved, false); assertEquals((await repo.getById('up_test'))?.modelsCache, null); }); -test('SQL model-cache generation accepts semantically equal noncanonical config JSON', async () => { +test('SQL model-cache identity accepts semantically equal noncanonical config JSON', async () => { const db = await createSqliteTestDb(); const repo = new SqlRepo(db).upstreams; const record = baseRecord(); @@ -195,7 +213,7 @@ test('SQL model-cache generation accepts semantically equal noncanonical config const parsed = await repo.getById(record.id); if (!parsed) throw new Error('upstream row missing'); - const saved = await seedModelsCache(repo, record.id, generationFor(parsed), { + const saved = await seedModelsCache(repo, record.id, identityFor(parsed), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_001_000_000, models: [stubProviderModel({ id: 'cached-model' })], @@ -208,7 +226,7 @@ test('SQL model-cache generation accepts semantically equal noncanonical config test('SQL upstream repo save leaves an existing cached catalog alone', async () => { const repo = new SqlRepo(await createSqliteTestDb()).upstreams; await repo.save(baseRecord()); - await seedModelsCache(repo, 'up_test', generationFor(baseRecord()), { + await seedModelsCache(repo, 'up_test', identityFor(baseRecord()), { revision: MODEL_CATALOG_REVISION, fetchedAt: 1_700_000_000_000, models: [stubProviderModel({ id: 'cached-model' })], diff --git a/packages/gateway/__tests__/test-utils/app.ts b/packages/gateway/__tests__/test-utils/app.ts index 476299d40..f827668e3 100644 --- a/packages/gateway/__tests__/test-utils/app.ts +++ b/packages/gateway/__tests__/test-utils/app.ts @@ -314,7 +314,7 @@ export const requestAppWithWarmModels = async (path: string, init: RequestInit): export const warmModelsForTest = async (): Promise => { const upstreams = await getRepo().upstreams.list(); await Promise.allSettled(upstreams.map(async upstream => - await refreshModels(modelsRefreshTarget(upstream), 'TEST', { bypassBackoff: false, includeDiscovered: false }))); + await refreshModels(modelsRefreshTarget(upstream), 'TEST'))); }; export function parseSSEText(text: string): Array<{ event: string; data: string }> { diff --git a/packages/gateway/migrations/0078_upstream_models_refresh.sql b/packages/gateway/migrations/0078_upstream_models_refresh.sql index b69a4b17e..44a485187 100644 --- a/packages/gateway/migrations/0078_upstream_models_refresh.sql +++ b/packages/gateway/migrations/0078_upstream_models_refresh.sql @@ -1,25 +1,11 @@ --- Refresh ownership and the catalog it protects share one row so claims and --- catalog publication can be fenced by a single atomic update. +-- Execution cells coordinate in-flight work; D1 retains retry backoff beside +-- the catalog so completion can be fenced by one upstream-row update. ALTER TABLE upstreams ADD COLUMN models_refresh_json TEXT NULL CHECK ( models_refresh_json IS NULL OR coalesce(( json_valid(models_refresh_json) = 1 - AND json_type(models_refresh_json, '$.failCount') IN ('integer', 'real') - AND json_extract(models_refresh_json, '$.failCount') >= 0 - AND json_extract(models_refresh_json, '$.failCount') = CAST(json_extract(models_refresh_json, '$.failCount') AS INTEGER) - AND json_type(models_refresh_json, '$.retryAt') IN ('integer', 'real') + AND json_type(models_refresh_json, '$.failureCount') = 'integer' + AND json_extract(models_refresh_json, '$.failureCount') >= 0 + AND json_type(models_refresh_json, '$.retryAt') = 'integer' AND json_extract(models_refresh_json, '$.retryAt') >= 0 - AND json_extract(models_refresh_json, '$.retryAt') = CAST(json_extract(models_refresh_json, '$.retryAt') AS INTEGER) - AND ( - ( - json_type(models_refresh_json, '$.claimToken') = 'null' - AND json_type(models_refresh_json, '$.claimedAt') = 'null' - ) OR ( - json_type(models_refresh_json, '$.claimToken') = 'text' - AND length(json_extract(models_refresh_json, '$.claimToken')) > 0 - AND json_type(models_refresh_json, '$.claimedAt') IN ('integer', 'real') - AND json_extract(models_refresh_json, '$.claimedAt') >= 0 - AND json_extract(models_refresh_json, '$.claimedAt') = CAST(json_extract(models_refresh_json, '$.claimedAt') AS INTEGER) - ) - ) ), 0) = 1 ); diff --git a/packages/gateway/migrations/0080_simplify_models_refresh.sql b/packages/gateway/migrations/0080_simplify_models_refresh.sql deleted file mode 100644 index ff374ad4c..000000000 --- a/packages/gateway/migrations/0080_simplify_models_refresh.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Execution cells own in-flight coordination. D1 retains only retry backoff; --- dropping the old column also removes its claim/lease shape constraint. -ALTER TABLE upstreams DROP COLUMN models_refresh_json; -ALTER TABLE upstreams ADD COLUMN models_refresh_json TEXT NULL CHECK ( - models_refresh_json IS NULL OR coalesce(( - json_valid(models_refresh_json) = 1 - AND json_type(models_refresh_json, '$.failureCount') = 'integer' - AND json_extract(models_refresh_json, '$.failureCount') >= 0 - AND json_type(models_refresh_json, '$.retryAt') = 'integer' - AND json_extract(models_refresh_json, '$.retryAt') >= 0 - ), 0) = 1 -); diff --git a/packages/gateway/src/control-plane/models/routes.ts b/packages/gateway/src/control-plane/models/routes.ts index 568585970..b07c7ff97 100644 --- a/packages/gateway/src/control-plane/models/routes.ts +++ b/packages/gateway/src/control-plane/models/routes.ts @@ -1,7 +1,7 @@ import { toPublicModel } from '../../data-plane/models/load.ts'; import { type AddressableIdEntry, enumerateAddressableModelIds, listedRealModels } from '../../data-plane/shared/listing/addressable.ts'; import { mergeAliasesIntoModels } from '../../data-plane/shared/listing/alias.ts'; -import { createPerRequestFetcher } from '../../dial/per-request.ts'; +import { createModelsRefreshScheduler } from '../../execution/models-refresh.ts'; import { effectiveUpstreamIdsFromContext, userFromContext } from '../../middleware/auth.ts'; import type { CtxWithQuery } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; @@ -82,23 +82,21 @@ export const controlPlaneModels = async (c: CtxWithQuery) => // data-plane access to. const isAdmin = userFromContext(c).isAdmin; const upstreamScope = isAdmin ? null : effectiveUpstreamIdsFromContext(c); - // Fetch the upstream list once at the request boundary and thread it - // into every downstream consumer (`createPerRequestFetcher`, - // `enumerateAddressableModelIds`, the hue-join map) so this request - // pays a single `upstreams.list()` round-trip. + // Fetch the upstream list once at the request boundary and thread it into + // catalog enumeration and the hue join. const upstreamRows = await getRepo().upstreams.list(); const runtimeLocation = getRuntimeLocation(c.req.raw); - const fetcherForUpstream = await createPerRequestFetcher(runtimeLocation, upstreamRows); + const scheduleRefresh = createModelsRefreshScheduler(runtimeLocation, backgroundSchedulerFromContext(c)); // Two addressable surfaces: caller-scoped (drives visibility + // `aliasedFrom.targets` narrowing for non-admin) and gateway-wide // (drives the alias's metadata + endpoints + pricing — every caller // sees the same numbers for the same alias). For admin the two are // the same, so skip the second fetch. const [callerAddressable, gatewayAddressable, aliases] = await Promise.all([ - enumerateAddressableModelIds(upstreamScope, fetcherForUpstream, backgroundSchedulerFromContext(c), runtimeLocation, upstreamRows), + enumerateAddressableModelIds(upstreamScope, scheduleRefresh, upstreamRows), isAdmin ? Promise.resolve(null) - : enumerateAddressableModelIds(null, fetcherForUpstream, backgroundSchedulerFromContext(c), runtimeLocation, upstreamRows), + : enumerateAddressableModelIds(null, scheduleRefresh, upstreamRows), includeAliases ? getRepo().modelAliases.list() : Promise.resolve([]), ]); const hueByUpstream = new Map(upstreamRows.map(row => [row.id, row.hue])); diff --git a/packages/gateway/src/control-plane/schemas.ts b/packages/gateway/src/control-plane/schemas.ts index d36e6a3a6..a6ea37445 100644 --- a/packages/gateway/src/control-plane/schemas.ts +++ b/packages/gateway/src/control-plane/schemas.ts @@ -12,7 +12,7 @@ // Deep upstream-config validation (e.g. Azure URL hostname rules, custom // pathOverrides and modelsFetch.endpoint URL parsing, per-model endpoint path // checks) stays with provider validators and handlers, which own the canonical -// error messages. Repository model-aware writes own catalog generations and +// error messages. Repository model-aware writes own catalog versions and // invalidation. The schemas here describe the shape the dashboard sends. import { z } from 'zod'; diff --git a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts index 231ba6edc..6b9042756 100644 --- a/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts +++ b/packages/gateway/src/control-plane/shared/save-upstream-for-models.ts @@ -47,7 +47,7 @@ export const saveUpstreamsAndWarmChangedModels = async ( const runtimeLocation = getRuntimeLocation(c.req.raw); const warmedEntries = await Promise.all(recordsToWarm.map(async record => { try { - await refreshModels(modelsRefreshTarget(record), runtimeLocation, { bypassBackoff: false, includeDiscovered: false }); + await refreshModels(modelsRefreshTarget(record), runtimeLocation); } catch (error) { logInfo('warm_models_cache_failed', { upstream_id: record.id, error: errorMessage(error) }); } diff --git a/packages/gateway/src/control-plane/upstreams/models.ts b/packages/gateway/src/control-plane/upstreams/models.ts index 9d1c5c625..dc94864ce 100644 --- a/packages/gateway/src/control-plane/upstreams/models.ts +++ b/packages/gateway/src/control-plane/upstreams/models.ts @@ -3,7 +3,7 @@ import { resolveControlPlaneFetcher } from './proxy-resolution.ts'; import { isValidProviderKind, upstreamErrorMessage as errorMessage } from './shared.ts'; import { MODEL_LISTING_FAILURE_CODE, MODEL_LISTING_FAILURE_MESSAGE } from '../../data-plane/models/shared.ts'; import { createPreviewProvider } from '../../data-plane/providers/registry.ts'; -import { modelsRefreshTarget, refreshModels } from '../../execution/models-refresh.ts'; +import { isModelsRefreshConfigurationError, modelsRefreshTarget, refreshModelsExplicit } from '../../execution/models-refresh.ts'; import type { AuthedContext } from '../../middleware/auth.ts'; import type { CtxWithJson } from '../../middleware/zod-validator.ts'; import { getRepo } from '../../repo/index.ts'; @@ -105,22 +105,21 @@ export const fetchSavedModels = async (c: AuthedContext<'/:id/list-models'>) => const id = c.req.param('id'); const record = await getRepo().upstreams.getById(id); if (record === null) return c.json({ error: 'Upstream not found' }, 404); + const runtimeLocation = getRuntimeLocation(c.req.raw); try { - const result = await refreshModels(modelsRefreshTarget(record), getRuntimeLocation(c.req.raw), { - bypassBackoff: true, - includeDiscovered: record.kind === 'custom', - }); + const result = await refreshModelsExplicit(modelsRefreshTarget(record), runtimeLocation, record.kind === 'custom'); if (result.kind !== 'refreshed') throw new Error(`Upstream ${id} changed during models refresh`); const refreshed = await getRepo().upstreams.getById(id); if (refreshed === null) throw new Error(`Upstream ${id} disappeared after models refresh`); - const data = result.discovered ?? refreshed.modelsCache?.models.map(reshapeModelForDashboard); + const data = record.kind === 'custom' ? result.discovered : refreshed.modelsCache?.models.map(reshapeModelForDashboard); if (data === undefined) throw new Error(`Upstream ${id} models refresh did not publish a catalog`); return c.json({ data, modelsCache: modelsCacheStatus(refreshed) }); } catch (e) { if (e instanceof ProviderModelsUnavailableError) { return c.json({ error: { message: MODEL_LISTING_FAILURE_MESSAGE, type: 'api_error', code: MODEL_LISTING_FAILURE_CODE } }, 502); } + if (isModelsRefreshConfigurationError(e)) return c.json({ error: errorMessage(e) }, 400); if (malformedConfigResponse(e)) return c.json({ error: errorMessage(e) }, 400); throw e; } diff --git a/packages/gateway/src/data-plane/codex/models.ts b/packages/gateway/src/data-plane/codex/models.ts index 715428116..c1e993e27 100644 --- a/packages/gateway/src/data-plane/codex/models.ts +++ b/packages/gateway/src/data-plane/codex/models.ts @@ -16,9 +16,8 @@ import { resolveCodexCatalog, type CatalogModel, type CodexCatalog, type CodexCatalogCapabilities } from './catalog.ts'; import { synthesizeCatalogEntry } from './synthesize.ts'; +import type { ModelsRefreshScheduler } from '../../execution/models-refresh.ts'; import { enumerateAddressableModelIds, type AddressableIdEntry } from '../shared/listing/addressable.ts'; -import type { BackgroundScheduler } from '@floway-dev/platform'; -import type { Fetcher } from '@floway-dev/provider'; // Pure transformation: client catalog + addressable entries → // codex-shaped catalog (drops unlisted alternates and non-chat kinds). @@ -62,13 +61,11 @@ export const assembleCodexCatalog = ( export const loadCodexCatalog = async ( userAgent: string | undefined, upstreamIds: readonly string[] | null, - fetcherForUpstream: (upstreamId: string) => Fetcher, - scheduler: BackgroundScheduler, - runtimeLocation: string, + scheduleRefresh: ModelsRefreshScheduler, ): Promise => { const [resolution, addressable] = await Promise.all([ resolveCodexCatalog(userAgent), - enumerateAddressableModelIds(upstreamIds, fetcherForUpstream, scheduler, runtimeLocation), + enumerateAddressableModelIds(upstreamIds, scheduleRefresh), ]); return assembleCodexCatalog(resolution.catalog, addressable, resolution.capabilities); }; diff --git a/packages/gateway/src/data-plane/models/gemini.ts b/packages/gateway/src/data-plane/models/gemini.ts index d189ad83d..190bf810b 100644 --- a/packages/gateway/src/data-plane/models/gemini.ts +++ b/packages/gateway/src/data-plane/models/gemini.ts @@ -1,6 +1,6 @@ import type { Context } from 'hono'; -import { createPerRequestFetcher } from '../../dial/per-request.ts'; +import { createModelsRefreshScheduler, type ModelsRefreshScheduler } from '../../execution/models-refresh.ts'; import { effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts'; import { getRepo } from '../../repo/index.ts'; import type { ModelAliasesRepo } from '../../repo/types.ts'; @@ -9,9 +9,8 @@ import { getRuntimeLocation } from '../../runtime/runtime-info.ts'; import { geminiStatusForHttpStatus } from '../chat/gemini/errors.ts'; import { enumerateAddressableModelIds, listedRealModels } from '../shared/listing/addressable.ts'; import { mergeAliasesIntoModels } from '../shared/listing/alias.ts'; -import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ModelPricing } from '@floway-dev/protocols/common'; -import type { InternalModel, Fetcher } from '@floway-dev/provider'; +import type { InternalModel } from '@floway-dev/provider'; type GeminiGenerationMethod = 'generateContent' | 'streamGenerateContent' | 'countTokens'; @@ -64,16 +63,14 @@ const geminiModelLoadError = (error: unknown): Response => // step with /v1/models and the dashboard's /api/models. const loadGeminiModels = async ( upstreamFilter: readonly string[] | null, - fetcherForUpstream: (upstreamId: string) => Fetcher, - scheduler: BackgroundScheduler, - runtimeLocation: string, + scheduleRefresh: ModelsRefreshScheduler, aliasRepo: ModelAliasesRepo, ): Promise => { const [callerAddressable, gatewayAddressable, aliases] = await Promise.all([ - enumerateAddressableModelIds(upstreamFilter, fetcherForUpstream, scheduler, runtimeLocation), + enumerateAddressableModelIds(upstreamFilter, scheduleRefresh), upstreamFilter === null ? Promise.resolve(null) - : enumerateAddressableModelIds(null, fetcherForUpstream, scheduler, runtimeLocation), + : enumerateAddressableModelIds(null, scheduleRefresh), aliasRepo.list(), ]); const gatewayAddressableModelIds = gatewayAddressable ?? callerAddressable; @@ -93,8 +90,8 @@ const loadGeminiModels = async ( export const serveGeminiModels = async (c: Context): Promise => { try { - const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw)); - return Response.json({ models: await loadGeminiModels(effectiveUpstreamIdsFromContext(c), fetcherForUpstream, backgroundSchedulerFromContext(c), getRuntimeLocation(c.req.raw), getRepo().modelAliases) }); + const scheduleRefresh = createModelsRefreshScheduler(getRuntimeLocation(c.req.raw), backgroundSchedulerFromContext(c)); + return Response.json({ models: await loadGeminiModels(effectiveUpstreamIdsFromContext(c), scheduleRefresh, getRepo().modelAliases) }); } catch (error) { return geminiModelLoadError(error); } @@ -106,8 +103,8 @@ export const serveGeminiModelInfo = async (c: Context): Promise => { const modelId = rawModelId.replace(/^models\//, ''); try { - const fetcherForUpstream = await createPerRequestFetcher(getRuntimeLocation(c.req.raw)); - const model = (await loadGeminiModels(effectiveUpstreamIdsFromContext(c), fetcherForUpstream, backgroundSchedulerFromContext(c), getRuntimeLocation(c.req.raw), getRepo().modelAliases)).find(candidate => candidate.baseModelId === modelId || candidate.name === `models/${modelId}`); + const scheduleRefresh = createModelsRefreshScheduler(getRuntimeLocation(c.req.raw), backgroundSchedulerFromContext(c)); + const model = (await loadGeminiModels(effectiveUpstreamIdsFromContext(c), scheduleRefresh, getRepo().modelAliases)).find(candidate => candidate.baseModelId === modelId || candidate.name === `models/${modelId}`); if (!model) return geminiError(404, `Model not found: ${modelId}`); return Response.json(model); } catch (error) { diff --git a/packages/gateway/src/data-plane/models/http.ts b/packages/gateway/src/data-plane/models/http.ts index f7f71cdcb..ec88b0b4c 100644 --- a/packages/gateway/src/data-plane/models/http.ts +++ b/packages/gateway/src/data-plane/models/http.ts @@ -6,7 +6,7 @@ import type { Context } from 'hono'; import { loadModels } from './load.ts'; -import { createPerRequestFetcher } from '../../dial/per-request.ts'; +import { createModelsRefreshScheduler } from '../../execution/models-refresh.ts'; import { effectiveUpstreamIdsFromContext } from '../../middleware/auth.ts'; import { getRepo } from '../../repo/index.ts'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; @@ -74,15 +74,14 @@ export const serveModels = async (c: Context): Promise => { try { const userAgent = c.req.header('user-agent'); const runtimeLocation = getRuntimeLocation(c.req.raw); - const fetcherForUpstream = await createPerRequestFetcher(runtimeLocation); const upstreamIds = effectiveUpstreamIdsFromContext(c); - const scheduler = backgroundSchedulerFromContext(c); + const scheduleRefresh = createModelsRefreshScheduler(runtimeLocation, backgroundSchedulerFromContext(c)); if (isCodexUserAgent(userAgent)) { - return Response.json(await loadCodexCatalog(userAgent, upstreamIds, fetcherForUpstream, scheduler, runtimeLocation)); + return Response.json(await loadCodexCatalog(userAgent, upstreamIds, scheduleRefresh)); } - const publicCatalog = await loadModels(upstreamIds, fetcherForUpstream, scheduler, runtimeLocation, getRepo().modelAliases); + const publicCatalog = await loadModels(upstreamIds, scheduleRefresh, getRepo().modelAliases); // The Claude Code CLI's model discovery request identifies itself with // a `claude-code/` User-Agent (built from the CLI's `n_()` // helper — verified in the v2.1.206 binary). The CLI's other request diff --git a/packages/gateway/src/data-plane/models/load.ts b/packages/gateway/src/data-plane/models/load.ts index 607801f44..af2f4a8db 100644 --- a/packages/gateway/src/data-plane/models/load.ts +++ b/packages/gateway/src/data-plane/models/load.ts @@ -1,9 +1,9 @@ +import type { ModelsRefreshScheduler } from '../../execution/models-refresh.ts'; import type { ModelAliasesRepo } from '../../repo/types.ts'; import { enumerateAddressableModelIds, listedRealModels } from '../shared/listing/addressable.ts'; import { mergeAliasesIntoModels } from '../shared/listing/alias.ts'; -import type { BackgroundScheduler } from '@floway-dev/platform'; import type { PublicModel, PublicModelsResponse } from '@floway-dev/protocols/common'; -import type { Fetcher, InternalModel } from '@floway-dev/provider'; +import type { InternalModel } from '@floway-dev/provider'; // Project an `InternalModel` onto the public-facing `/v1/models` wire DTO. // `endpoints` rides through as the merged upstream wire surface — the @@ -42,9 +42,7 @@ export const toPublicModel = (model: InternalModel): PublicModel => { export const loadModels = async ( upstreamFilter: readonly string[] | null, - fetcherForUpstream: (upstreamId: string) => Fetcher, - scheduler: BackgroundScheduler, - runtimeLocation: string, + scheduleRefresh: ModelsRefreshScheduler, aliasRepo: ModelAliasesRepo, ): Promise => { // Data-plane responses always narrow `aliasedFrom.targets` to the @@ -52,10 +50,10 @@ export const loadModels = async ( // ids), but the alias's metadata is still computed gateway-wide so // every caller sees the same numbers. const [callerAddressable, gatewayAddressable, aliases] = await Promise.all([ - enumerateAddressableModelIds(upstreamFilter, fetcherForUpstream, scheduler, runtimeLocation), + enumerateAddressableModelIds(upstreamFilter, scheduleRefresh), upstreamFilter === null ? Promise.resolve(null) - : enumerateAddressableModelIds(null, fetcherForUpstream, scheduler, runtimeLocation), + : enumerateAddressableModelIds(null, scheduleRefresh), aliasRepo.list(), ]); const gatewayAddressableModelIds = gatewayAddressable ?? callerAddressable; diff --git a/packages/gateway/src/data-plane/providers/catalog.ts b/packages/gateway/src/data-plane/providers/catalog.ts index fcad607a0..461ac095e 100644 --- a/packages/gateway/src/data-plane/providers/catalog.ts +++ b/packages/gateway/src/data-plane/providers/catalog.ts @@ -1,9 +1,9 @@ import { unionEndpoints } from './endpoint-union.ts'; import { readUpstreamModelsSnapshotAndScheduleRefresh, MODEL_CATALOG_REVISION } from './models-cache.ts'; import type { GatewayProvider } from './registry.ts'; -import type { BackgroundScheduler } from '@floway-dev/platform'; +import type { ModelsRefreshScheduler } from '../../execution/models-refresh.ts'; import { kindForEndpoints } from '@floway-dev/protocols/common'; -import { isAbortError, type Fetcher, type InternalModel, type Provider, type ProviderModel, type UpstreamRecord } from '@floway-dev/provider'; +import type { InternalModel, Provider, ProviderModel, UpstreamRecord } from '@floway-dev/provider'; interface ProviderModelsResult { models: InternalModel[]; @@ -12,10 +12,7 @@ interface ProviderModelsResult { // endpoint reads this to render `upstreams: [{kind, id, name}]` per row; // the alias listing reads it to project per-target upstream chips. upstreamsByPublicId: Map; - sawSuccess: boolean; - lastError: unknown; - // Upstreams carrying a persisted catalog-refresh error, plus any provider - // whose snapshot access failed synchronously, in provider order. + // Upstreams carrying a persisted catalog-refresh error, in provider order. failedUpstreams: string[]; } @@ -81,45 +78,21 @@ const mergeIntoCatalog = ( instances.push(instance); }; -const collectProviderModels = async ( +const collectProviderModels = ( providers: readonly GatewayProvider[], - fetcherForUpstream: (upstreamId: string) => Fetcher, - scheduler: BackgroundScheduler, - runtimeLocation: string, -): Promise => { + scheduleRefresh: ModelsRefreshScheduler, +): ProviderModelsResult => { const byId = new Map(); const upstreamsByPublicId = new Map(); - let sawSuccess = false; - let lastError: unknown = null; const failedUpstreams: string[] = []; // Catalog reads never await upstream I/O. Each result is the persisted // snapshot carried by the provider; a cold or stale snapshot separately // triggers background refresh through the supplied scheduler. - const fetchOne = (instance: GatewayProvider) => { - const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(instance, { - scheduler, - runtimeLocation, - }); - return { instance, models: snapshot.models, lastError: snapshot.lastError }; - }; - - const settled = await Promise.allSettled(providers.map(async provider => fetchOne(provider))); - - for (const [index, result] of settled.entries()) { - if (result.status === 'rejected') { - // Snapshot setup failures stay isolated per provider. Cancellation is - // the exception because the caller has withdrawn the whole operation. - const error = result.reason; - if (isAbortError(error)) throw error; - lastError = error; - failedUpstreams.push(providers[index].name); - continue; - } - sawSuccess = true; - const { instance, models: providedModels, lastError: cachedError } = result.value; + for (const instance of providers) { + const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(instance, scheduleRefresh); + const { models: providedModels, lastError: cachedError } = snapshot; if (cachedError) { - lastError = new Error(cachedError.message); failedUpstreams.push(instance.name); } // Operator-disabled public model ids vanish entirely for this upstream: @@ -157,7 +130,7 @@ const collectProviderModels = async ( } } - return { models: [...byId.values()], upstreamsByPublicId, sawSuccess, lastError, failedUpstreams }; + return { models: [...byId.values()], upstreamsByPublicId, failedUpstreams }; }; // How many catalog entries this upstream's stored catalog would surface, under @@ -219,21 +192,17 @@ export const compareModelIds = (a: string, b: string): number => { // shares its provider list across the alias resolver and the candidate // walk — pass providers through to avoid the duplicate upstreams.list() // DB query. -export const getModelsFromProviders = async ( +export const getModelsFromProviders = ( providers: readonly GatewayProvider[], - fetcherForUpstream: (upstreamId: string) => Fetcher, - scheduler: BackgroundScheduler, - runtimeLocation: string, -): Promise<{ models: InternalModel[]; upstreamsByPublicId: Map; failedUpstreams: readonly string[] }> => { + scheduleRefresh: ModelsRefreshScheduler, +): { models: InternalModel[]; upstreamsByPublicId: Map; failedUpstreams: readonly string[] } => { if (providers.length === 0) { throw new Error('No upstream provider configured — connect GitHub Copilot or add a Custom/Azure upstream in the dashboard'); } - const { models, upstreamsByPublicId, sawSuccess, lastError, failedUpstreams } = await collectProviderModels(providers, fetcherForUpstream, scheduler, runtimeLocation); + const { models, upstreamsByPublicId, failedUpstreams } = collectProviderModels(providers, scheduleRefresh); // TODO: surface `failedUpstreams` on each listing endpoint's wire response // so partial-listing failures reach clients. - if (sawSuccess) return { models: models.sort((a, b) => compareModelIds(a.id, b.id)), upstreamsByPublicId, failedUpstreams }; - if (lastError) throw lastError; - return { models: [], upstreamsByPublicId, failedUpstreams }; + return { models: models.sort((a, b) => compareModelIds(a.id, b.id)), upstreamsByPublicId, failedUpstreams }; }; diff --git a/packages/gateway/src/data-plane/providers/models-cache.ts b/packages/gateway/src/data-plane/providers/models-cache.ts index 7ced8d707..81d79190e 100644 --- a/packages/gateway/src/data-plane/providers/models-cache.ts +++ b/packages/gateway/src/data-plane/providers/models-cache.ts @@ -1,7 +1,6 @@ import type { GatewayProvider } from './registry.ts'; -import { scheduleModelsRefresh } from '../../execution/models-refresh.ts'; +import type { ModelsRefreshScheduler } from '../../execution/models-refresh.ts'; import { MODEL_CATALOG_REVISION } from '../../repo/models-cache-contract.ts'; -import type { BackgroundScheduler } from '@floway-dev/platform'; import type { ProviderModel, UpstreamModelsCache } from '@floway-dev/provider'; const SOFT_MS = 10 * 60 * 1000; @@ -13,30 +12,23 @@ export interface ModelsSnapshot { readonly lastError: UpstreamModelsCache['lastError']; } -interface ModelsSnapshotReadOptions { - scheduler: BackgroundScheduler; - runtimeLocation: string; -} - // Capture one immutable snapshot before scheduling any refresh work so its -// models and error metadata always describe the same durable generation. +// models and error metadata always describe the same durable snapshot. export const readUpstreamModelsSnapshotAndScheduleRefresh = ( instance: GatewayProvider, - options: ModelsSnapshotReadOptions, + scheduleRefresh: ModelsRefreshScheduler, ): ModelsSnapshot => { - const { scheduler, runtimeLocation } = options; const cached = instance.modelsCache?.revision === MODEL_CATALOG_REVISION ? instance.modelsCache : null; const snapshot = { models: cached?.models ?? [], lastError: cached?.lastError ?? null, }; if (!cached || Date.now() - cached.fetchedAt >= SOFT_MS) { - const fetchedAt = instance.modelsCache?.fetchedAt; - scheduleModelsRefresh({ + scheduleRefresh({ upstreamId: instance.upstreamId, - configVersion: instance.modelsCacheGeneration.configVersion, - cacheEpoch: fetchedAt === undefined || fetchedAt === 0 ? null : fetchedAt, - }, runtimeLocation, scheduler); + configVersion: instance.configVersion, + cacheEpoch: instance.modelsCache?.fetchedAt ?? 0, + }); } return snapshot; }; diff --git a/packages/gateway/src/data-plane/providers/registry.ts b/packages/gateway/src/data-plane/providers/registry.ts index f626281d0..1209b95b1 100644 --- a/packages/gateway/src/data-plane/providers/registry.ts +++ b/packages/gateway/src/data-plane/providers/registry.ts @@ -1,6 +1,5 @@ import { getRepo } from '../../repo/index.ts'; -import { modelsCacheGeneration } from '../../repo/models-cache-contract.ts'; -import type { ModelsCacheGeneration, StoredUpstreamRecord } from '../../repo/types.ts'; +import type { StoredUpstreamRecord } from '../../repo/types.ts'; import type { FlagDefaults, Provider, ProviderModule, UpstreamProviderKind, UpstreamRecord } from '@floway-dev/provider'; import { azureProviderModule } from '@floway-dev/provider-azure'; import { claudeCodeProviderModule } from '@floway-dev/provider-claude-code'; @@ -19,7 +18,7 @@ const providersByKind: Record = { }; export type GatewayProvider = Provider & { - readonly modelsCacheGeneration: ModelsCacheGeneration; + readonly configVersion: number; }; export const createProvider = ( @@ -28,7 +27,7 @@ export const createProvider = ( const provider = providersByKind[record.kind].create(record); return { ...provider, - modelsCacheGeneration: modelsCacheGeneration(record), + configVersion: record.configVersion, }; }; diff --git a/packages/gateway/src/data-plane/providers/resolution.ts b/packages/gateway/src/data-plane/providers/resolution.ts index f256b19a1..0b4d1c966 100644 --- a/packages/gateway/src/data-plane/providers/resolution.ts +++ b/packages/gateway/src/data-plane/providers/resolution.ts @@ -4,6 +4,7 @@ import { internalModelFromProviderModel } from './catalog.ts'; import { readUpstreamModelsSnapshotAndScheduleRefresh } from './models-cache.ts'; import { listModelProviders, type GatewayProvider } from './registry.ts'; import { createPerRequestFetcher } from '../../dial/per-request.ts'; +import { createModelsRefreshScheduler, type ModelsRefreshScheduler } from '../../execution/models-refresh.ts'; import { getRepo } from '../../repo/index.ts'; import type { ModelAliasRecord } from '../../repo/types.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; @@ -30,11 +31,10 @@ const enumerateOneUpstreamCandidates = async ( kind: ModelKind, context: { fetcher: Fetcher; - scheduler: BackgroundScheduler; - runtimeLocation: string; + scheduleRefresh: ModelsRefreshScheduler; }, ): Promise<{ candidates: ModelCandidate[]; sawAnyId: boolean; modelsError: boolean }> => { - const { fetcher, scheduler, runtimeLocation } = context; + const { fetcher, scheduleRefresh } = context; const cfg = provider.modelPrefix; const lookupIds: string[] = []; if (cfg === null) { @@ -47,7 +47,7 @@ const enumerateOneUpstreamCandidates = async ( } if (lookupIds.length === 0) return { candidates: [], sawAnyId: false, modelsError: false }; - const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(provider, { scheduler, runtimeLocation }); + const snapshot = readUpstreamModelsSnapshotAndScheduleRefresh(provider, scheduleRefresh); const disabled = new Set(provider.disabledPublicModelIds); const candidates: ModelCandidate[] = []; let sawAnyId = false; @@ -65,7 +65,7 @@ const enumerateOneUpstreamCandidates = async ( // Walk every visible upstream in configured order. Snapshot reads never wait // for upstream model-list I/O; cold and stale rows submit background refresh. // Client disconnect prevents snapshot work that has not dispatched. Once a -// refresh is scheduled, the scheduler owns its lifetime. Inference lifecycle +// refresh reaches its execution cell, it is detached from the request. Inference lifecycle // policy is applied later, where a selected candidate is actually dispatched. // // `sawAnyId` aggregates the per-upstream signal: true when at least one @@ -79,8 +79,7 @@ export const enumerateRealModelCandidates = async ( providers: readonly GatewayProvider[], context: { fetcherForUpstream: (upstreamId: string) => Fetcher; - scheduler: BackgroundScheduler; - runtimeLocation: string; + scheduleRefresh: ModelsRefreshScheduler; clientDisconnectSignal?: AbortSignal; }, ): Promise<{ @@ -88,7 +87,7 @@ export const enumerateRealModelCandidates = async ( readonly sawAnyId: boolean; readonly failedUpstreams: readonly string[]; }> => { - const { fetcherForUpstream, scheduler, runtimeLocation, clientDisconnectSignal } = context; + const { fetcherForUpstream, scheduleRefresh, clientDisconnectSignal } = context; const settled = await Promise.allSettled(providers.map(provider => { clientDisconnectSignal?.throwIfAborted(); return enumerateOneUpstreamCandidates( @@ -97,8 +96,7 @@ export const enumerateRealModelCandidates = async ( kind, { fetcher: fetcherForUpstream(provider.upstreamId), - scheduler, - runtimeLocation, + scheduleRefresh, }, ); })); @@ -227,8 +225,7 @@ export const enumerateModelCandidates = async ({ const providers = await listModelProviders(upstreamIds); const resolutionContext = { fetcherForUpstream: createFetcherForUpstream, - scheduler, - runtimeLocation, + scheduleRefresh: createModelsRefreshScheduler(runtimeLocation, scheduler), clientDisconnectSignal, }; diff --git a/packages/gateway/src/data-plane/shared/listing/addressable.ts b/packages/gateway/src/data-plane/shared/listing/addressable.ts index 5eebd482c..d60f731cd 100644 --- a/packages/gateway/src/data-plane/shared/listing/addressable.ts +++ b/packages/gateway/src/data-plane/shared/listing/addressable.ts @@ -11,12 +11,12 @@ // DTO) read `limits` / `chat` / `endpoints` directly off the entry without // a second registry round trip. +import type { ModelsRefreshScheduler } from '../../../execution/models-refresh.ts'; import type { StoredUpstreamRecord } from '../../../repo/types.ts'; import { compareModelIds, getModelsFromProviders } from '../../providers/catalog.ts'; -import { readUpstreamModelsSnapshotAndScheduleRefresh } from '../../providers/models-cache.ts'; +import { MODEL_CATALOG_REVISION } from '../../providers/models-cache.ts'; import { listModelProviders } from '../../providers/registry.ts'; -import type { BackgroundScheduler } from '@floway-dev/platform'; -import { isAbortError, type Fetcher, type InternalModel, type Provider } from '@floway-dev/provider'; +import type { InternalModel, Provider } from '@floway-dev/provider'; export interface AddressableIdEntry { // The inbound model id the data plane will accept verbatim. @@ -53,9 +53,7 @@ export const listedRealModels = (entries: readonly AddressableIdEntry[]): readon // same separate background refresh trigger. export const enumerateAddressableModelIds = async ( upstreamFilter: readonly string[] | null, - fetcherForUpstream: (upstreamId: string) => Fetcher, - scheduler: BackgroundScheduler, - runtimeLocation: string, + scheduleRefresh: ModelsRefreshScheduler, preFetchedUpstreams?: readonly StoredUpstreamRecord[], ): Promise => { // Resolve providers once and thread them into the catalog assembly so @@ -66,7 +64,7 @@ export const enumerateAddressableModelIds = async ( // hint behavior on a brand-new gateway. `preFetchedUpstreams` avoids // an additional round-trip when the caller has the list already. const providers = await listModelProviders(upstreamFilter, preFetchedUpstreams); - const { models: realModels, upstreamsByPublicId } = await getModelsFromProviders(providers, fetcherForUpstream, scheduler, runtimeLocation); + const { models: realModels, upstreamsByPublicId } = getModelsFromProviders(providers, scheduleRefresh); const byId = new Map(realModels.map(model => [model.id, model] as const)); const entries: AddressableIdEntry[] = []; @@ -78,20 +76,19 @@ export const enumerateAddressableModelIds = async ( }; for (const model of realModels) { - push({ id: model.id, unlisted: undefined, model, upstreams: upstreamsByPublicId.get(model.id) ?? [] }); + const upstreams = upstreamsByPublicId.get(model.id); + if (upstreams === undefined) throw new Error(`Listed model ${model.id} has no upstream index`); + push({ id: model.id, unlisted: undefined, model, upstreams }); } - // Prefix alternates reuse the same persisted provider snapshots as the - // listed surface. Repeated access may join the same L1 refresh trigger, but - // never performs upstream model-list I/O in this request. - const perUpstream = await Promise.allSettled(providers.map(async provider => { + // Prefix alternates reuse the provider snapshots read by the listed surface. + for (const provider of providers) { const cfg = provider.modelPrefix; const addressableOnly = cfg !== null ? cfg.addressable.filter(form => !cfg.listed.includes(form)) : []; - if (cfg === null || addressableOnly.length === 0) return [] as AddressableIdEntry[]; + if (cfg === null || addressableOnly.length === 0) continue; - const upstreamModels = readUpstreamModelsSnapshotAndScheduleRefresh(provider, { scheduler, runtimeLocation }).models; + const upstreamModels = provider.modelsCache?.revision === MODEL_CATALOG_REVISION ? provider.modelsCache.models : []; const disabled = new Set(provider.disabledPublicModelIds); - const out: AddressableIdEntry[] = []; // The canonical listed form for this upstream — the row the listing // surface emitted, and the row an addressable-only prefix alternate @@ -104,25 +101,14 @@ export const enumerateAddressableModelIds = async ( ? `${cfg.prefix}${upstreamModel.id}` : upstreamModel.id; const canonical = byId.get(canonicalPublicId); - if (canonical === undefined) continue; - const canonicalUpstreams = upstreamsByPublicId.get(canonicalPublicId) ?? []; + if (canonical === undefined) throw new Error(`Addressable model ${canonicalPublicId} is missing from the listed catalog`); + const canonicalUpstreams = upstreamsByPublicId.get(canonicalPublicId); + if (canonicalUpstreams === undefined) throw new Error(`Addressable model ${canonicalPublicId} has no upstream index`); for (const form of addressableOnly) { const id = form === 'prefixed' ? `${cfg.prefix}${upstreamModel.id}` : upstreamModel.id; - out.push({ id, unlisted: true, model: canonical, upstreams: canonicalUpstreams }); + push({ id, unlisted: true, model: canonical, upstreams: canonicalUpstreams }); } } - - return out; - })); - - for (const result of perUpstream) { - if (result.status === 'rejected') { - // Snapshot setup failures omit only that provider; cancellation still - // withdraws the whole caller operation. - if (isAbortError(result.reason)) throw result.reason; - continue; - } - for (const entry of result.value) push(entry); } // Stable id ordering matches the listed surface so consumers can rely on diff --git a/packages/gateway/src/dial/per-request.ts b/packages/gateway/src/dial/per-request.ts index bbb6ae58d..914bf979c 100644 --- a/packages/gateway/src/dial/per-request.ts +++ b/packages/gateway/src/dial/per-request.ts @@ -6,27 +6,35 @@ import { getSocketDial } from '@floway-dev/platform'; import { directFetcher, type Fetcher, type UpstreamRecord } from '@floway-dev/provider'; import { runDirectConnectRequest, runProxiedRequest } from '@floway-dev/proxy'; +export class InvalidProxyConfigurationError extends Error { + constructor(message: string, cause?: unknown) { + super(message, cause === undefined ? undefined : { cause }); + this.name = 'InvalidProxyConfigurationError'; + } +} + // Parse failures on individual proxy rows are isolated to the upstreams that -// actually reference them: a single malformed URL must not take down every -// other upstream in the same request. Per-upstream fetchers built against a -// bad row throw at call time rather than at build time, mirroring how the -// dial layer surfaces other dial-time failures. +// actually request their fetcher: a single malformed URL does not take down +// every other upstream in the same request. // // `preFetchedUpstreams` lets a caller reuse a list it already loaded on // this request instead of paying a second `upstreams.list()` round-trip. -export const createPerRequestFetcher = async ( +const createFetcherResolver = async ( runtimeLocation: string | null, - preFetchedUpstreams?: readonly UpstreamRecord[], + preFetchedUpstreams: readonly UpstreamRecord[] | undefined, + validation: 'lazy' | 'eager', ): Promise<(upstreamId: string) => Fetcher> => { const repo = getRepo(); const upstreams = preFetchedUpstreams ?? await repo.upstreams.list(); + const configuredById = new Map(upstreams.map(u => [u.id, u.proxyFallbackList] as const)); const fallbackById = new Map(upstreams.map(u => [ u.id, u.proxyFallbackList.filter(entry => entryMatchesColo(entry, runtimeLocation)), ] as const)); const referencedProxyIds = new Set(); - for (const list of fallbackById.values()) { + const catalogLists = validation === 'eager' ? configuredById.values() : fallbackById.values(); + for (const list of catalogLists) { for (const entry of list) { if (!isDirectFallbackId(entry.id)) referencedProxyIds.add(entry.id); } @@ -43,14 +51,16 @@ export const createPerRequestFetcher = async ( if (list === undefined) { throw new Error(`unknown upstream id requested from per-request fetcher: ${upstreamId}`); } - const badRefs = list.filter(entry => proxyParseErrors.has(entry.id)); - if (badRefs.length > 0) { - const first = badRefs[0]!.id; + const validationList = validation === 'eager' ? configuredById.get(upstreamId)! : list; + const bad = validationList.find(entry => proxyParseErrors.has(entry.id)); + if (bad !== undefined) { + const first = bad.id; const err = proxyParseErrors.get(first)!; - return async () => { - throw new Error(`upstream ${upstreamId} references malformed proxy ${first}: ${err.message}`); - }; + if (validation === 'eager') throw new InvalidProxyConfigurationError(`upstream ${upstreamId} references malformed proxy ${first}: ${err.message}`, err); + return async () => { throw new Error(`upstream ${upstreamId} references malformed proxy ${first}: ${err.message}`); }; } + const unknown = validationList.find(entry => !isDirectFallbackId(entry.id) && !proxyById.has(entry.id)); + if (validation === 'eager' && unknown !== undefined) throw new InvalidProxyConfigurationError(`unknown proxy id in fallback list: ${unknown.id}`); return createFetcher({ repo, upstreamId, @@ -64,3 +74,20 @@ export const createPerRequestFetcher = async ( }); }; }; + +export const createPerRequestFetcher = ( + runtimeLocation: string | null, + preFetchedUpstreams?: readonly UpstreamRecord[], +): Promise<(upstreamId: string) => Fetcher> => createFetcherResolver(runtimeLocation, preFetchedUpstreams, 'lazy'); + +export const createValidatedPerRequestFetcher = ( + runtimeLocation: string | null, + preFetchedUpstreams?: readonly UpstreamRecord[], +): Promise<(upstreamId: string) => Fetcher> => createFetcherResolver(runtimeLocation, preFetchedUpstreams, 'eager'); + +export const validateUpstreamProxyConfiguration = async ( + runtimeLocation: string | null, + upstream: UpstreamRecord, +): Promise => { + (await createValidatedPerRequestFetcher(runtimeLocation, [upstream]))(upstream.id); +}; diff --git a/packages/gateway/src/execution/handler.ts b/packages/gateway/src/execution/handler.ts index 216214716..72311ea44 100644 --- a/packages/gateway/src/execution/handler.ts +++ b/packages/gateway/src/execution/handler.ts @@ -1,4 +1,5 @@ -import { executeModelsRefresh, modelsRefreshExecutionError, type ModelsRefreshExecutionInput } from './models-refresh.ts'; +import { executeModelsRefresh, isModelsRefreshConfigurationError, type ModelsRefreshExecutionInput } from './models-refresh.ts'; +import { ProviderModelsUnavailableError } from '@floway-dev/provider'; export const handleExecutionRequest = async (request: Request): Promise => { const url = new URL(request.url); @@ -9,7 +10,11 @@ export const handleExecutionRequest = async (request: Request): Promise const input = value as Record; if (typeof input.upstreamId !== 'string' || input.upstreamId === '') throw new TypeError('Models refresh upstreamId must be a non-empty string'); if (!Number.isSafeInteger(input.configVersion) || (input.configVersion as number) < 1) throw new TypeError('Models refresh configVersion must be a positive integer'); - if (input.cacheEpoch !== null && (!Number.isSafeInteger(input.cacheEpoch) || (input.cacheEpoch as number) < 0)) throw new TypeError('Models refresh cacheEpoch must be a non-negative integer or null'); + if (!Number.isSafeInteger(input.cacheEpoch) || (input.cacheEpoch as number) < 0) throw new TypeError('Models refresh cacheEpoch must be a non-negative integer'); if (input.runtimeLocation !== null && typeof input.runtimeLocation !== 'string') throw new TypeError('Models refresh runtimeLocation must be a string or null'); - if (typeof input.bypassBackoff !== 'boolean') throw new TypeError('Models refresh bypassBackoff must be a boolean'); - if (typeof input.includeDiscovered !== 'boolean') throw new TypeError('Models refresh includeDiscovered must be a boolean'); + if (input.mode !== 'automatic' && input.mode !== 'explicit') throw new TypeError('Models refresh mode must be automatic or explicit'); return { upstreamId: input.upstreamId, configVersion: input.configVersion as number, - cacheEpoch: input.cacheEpoch as number | null, + cacheEpoch: input.cacheEpoch as number, runtimeLocation: input.runtimeLocation as string | null, - bypassBackoff: input.bypassBackoff, - includeDiscovered: input.includeDiscovered, + mode: input.mode, }; }; diff --git a/packages/gateway/src/execution/models-refresh.ts b/packages/gateway/src/execution/models-refresh.ts index f7a74913d..53b958ff1 100644 --- a/packages/gateway/src/execution/models-refresh.ts +++ b/packages/gateway/src/execution/models-refresh.ts @@ -1,37 +1,38 @@ import { createProvider } from '../data-plane/providers/registry.ts'; -import { createPerRequestFetcher } from '../dial/per-request.ts'; +import { createPerRequestFetcher, createValidatedPerRequestFetcher, InvalidProxyConfigurationError, validateUpstreamProxyConfiguration } from '../dial/per-request.ts'; import { getRepo } from '../repo/index.ts'; -import { MODEL_CATALOG_REVISION, modelsCacheGeneration } from '../repo/models-cache-contract.ts'; +import { MODEL_CATALOG_REVISION } from '../repo/models-cache-contract.ts'; import type { StoredUpstreamRecord } from '../repo/types.ts'; import { getExecutionCellNamespace } from '../runtime/execution.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; -import { ProviderModelsUnavailableError, type ProviderModel, type UpstreamModelConfig } from '@floway-dev/provider'; +import { ProviderModelsUnavailableError, type Fetcher, type ProviderModel, type UpstreamModelConfig } from '@floway-dev/provider'; import { assertCustomUpstreamRecord, fetchCustomModels, projectCustomDiscoveredModels, projectCustomModels } from '@floway-dev/provider-custom'; export interface ModelsRefreshExecutionInput { upstreamId: string; configVersion: number; - cacheEpoch: number | null; + cacheEpoch: number; runtimeLocation: string | null; - bypassBackoff: boolean; - includeDiscovered: boolean; + mode: 'automatic' | 'explicit'; } -export type ModelsRefreshExecutionResult = +export type ModelsRefreshExecutionResult = ({ + mode: ModelsRefreshExecutionInput['mode']; +} & ( | { kind: 'refreshed'; discovered?: UpstreamModelConfig[] } - | { kind: 'backoff' | 'generation-mismatch' }; - -interface ModelsRefreshExecutionError { - kind: 'provider-unavailable' | 'error'; - message: string; -} + | { kind: 'backoff' | 'superseded' } +)); export type ModelsRefreshTarget = Pick; +export type ModelsRefreshScheduler = (target: ModelsRefreshTarget) => void; -const cacheEpoch = (record: Pick): number | null => { - const fetchedAt = record.modelsCache?.fetchedAt; - return fetchedAt === undefined || fetchedAt === 0 ? null : fetchedAt; -}; +class ModelsRefreshUnavailableError extends ProviderModelsUnavailableError { + constructor(readonly mode: ModelsRefreshExecutionInput['mode']) { + super(null); + } +} + +const cacheEpoch = (record: Pick): number => record.modelsCache?.fetchedAt ?? 0; export const modelsRefreshTarget = (record: StoredUpstreamRecord): ModelsRefreshTarget => ({ upstreamId: record.id, @@ -40,95 +41,125 @@ export const modelsRefreshTarget = (record: StoredUpstreamRecord): ModelsRefresh }); const errorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error); +export const isModelsRefreshConfigurationError = (error: unknown): error is InvalidProxyConfigurationError => + error instanceof InvalidProxyConfigurationError; export const executeModelsRefresh = async (input: ModelsRefreshExecutionInput): Promise => { const repo = getRepo().upstreams; const record = await repo.getById(input.upstreamId); if (record === null || record.configVersion !== input.configVersion - || cacheEpoch(record) !== input.cacheEpoch) return { kind: 'generation-mismatch' }; + || cacheEpoch(record) !== input.cacheEpoch) return { kind: 'superseded', mode: input.mode }; - const generation = modelsCacheGeneration(record); const beginning = await repo.beginModelsRefresh({ id: record.id, - generation, + configVersion: input.configVersion, + cacheEpoch: input.cacheEpoch, now: Date.now(), - bypassBackoff: input.bypassBackoff, + bypassBackoff: input.mode === 'explicit', }); - if (beginning.kind !== 'ready') return beginning; + if (beginning.kind !== 'ready') return { ...beginning, mode: input.mode }; try { - const fetcher = (await createPerRequestFetcher(input.runtimeLocation, [record]))(record.id); + const createFetcher = input.mode === 'explicit' ? createValidatedPerRequestFetcher : createPerRequestFetcher; + const fetcher: Fetcher = (await createFetcher(input.runtimeLocation, [record]))(record.id); let models: ProviderModel[]; let discovered: UpstreamModelConfig[] | undefined; - if (record.kind === 'custom' && input.includeDiscovered) { - const response = await fetchCustomModels(assertCustomUpstreamRecord(record).config, fetcher); - models = projectCustomModels(record, response); - discovered = projectCustomDiscoveredModels(record, response); + if (record.kind === 'custom') { + const custom = assertCustomUpstreamRecord(record); + if (input.mode === 'explicit' || custom.config.modelsFetch.enabled) { + const response = await fetchCustomModels(custom.config, fetcher); + models = projectCustomModels(record, response); + discovered = projectCustomDiscoveredModels(record, response); + } else { + models = projectCustomModels(record); + } } else { models = [...await createProvider(record).instance.getProvidedModels(fetcher)]; } const published = await repo.publishModelsRefresh({ id: record.id, - generation, - cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: Date.now(), models }, + configVersion: input.configVersion, + cacheEpoch: input.cacheEpoch, + cache: { revision: MODEL_CATALOG_REVISION, fetchedAt: Math.max(Date.now(), input.cacheEpoch + 1), models }, }); - return published ? { kind: 'refreshed', ...(discovered ? { discovered } : {}) } : { kind: 'generation-mismatch' }; + return published + ? { kind: 'refreshed', mode: input.mode, ...(discovered ? { discovered } : {}) } + : { kind: 'superseded', mode: input.mode }; } catch (error) { + if (input.mode === 'explicit' && isModelsRefreshConfigurationError(error)) throw error; const failedAt = Date.now(); - await repo.recordModelsRefreshFailure({ - id: record.id, - generation, - error: { message: errorMessage(error), at: failedAt }, - previousFailureCount: beginning.failureCount, - failedAt, - }); + try { + await repo.recordModelsRefreshFailure({ + id: record.id, + configVersion: input.configVersion, + cacheEpoch: input.cacheEpoch, + error: { message: errorMessage(error), at: failedAt }, + previousFailureCount: beginning.failureCount, + failedAt, + }); + } catch (recordError) { + throw new AggregateError([error, recordError], errorMessage(error)); + } throw error; } }; -const executionInput = ( - target: ModelsRefreshTarget, - runtimeLocation: string | null, - options: Pick, -): ModelsRefreshExecutionInput => ({ - ...target, - runtimeLocation, - ...options, -}); - const executeThroughCell = async (input: ModelsRefreshExecutionInput): Promise => { - const cellId = `models:${input.upstreamId}:${input.configVersion}:${input.cacheEpoch ?? 'cold'}`; + const cellId = JSON.stringify(['models', input.upstreamId, input.configVersion, input.cacheEpoch]); const response = await getExecutionCellNamespace().fetch(cellId, new Request('https://execution.floway/models/refresh', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(input), })); if (response.ok) return await response.json() as ModelsRefreshExecutionResult; - const error = await response.json() as ModelsRefreshExecutionError; - if (error.kind === 'provider-unavailable') throw new ProviderModelsUnavailableError(null); - throw new Error(error.message); + const error = await response.json() as { kind?: unknown; message?: unknown; mode?: unknown }; + if (response.status === 502 && error.kind === 'provider-unavailable' && (error.mode === 'automatic' || error.mode === 'explicit')) { + throw new ModelsRefreshUnavailableError(error.mode); + } + if (response.status === 400 && error.kind === 'invalid-configuration' && typeof error.message === 'string') { + throw new InvalidProxyConfigurationError(error.message); + } + throw new Error(`Unexpected models refresh execution response: HTTP ${response.status}`); }; -export const refreshModels = async ( +export const refreshModels = ( + target: ModelsRefreshTarget, + runtimeLocation: string | null, +): Promise => executeThroughCell({ ...target, runtimeLocation, mode: 'automatic' }); + +export const refreshModelsExplicit = async ( target: ModelsRefreshTarget, runtimeLocation: string | null, - options: Pick, + requiresDiscovery: boolean, ): Promise => { - const input = executionInput(target, runtimeLocation, options); - const result = await executeThroughCell(input); - return options.bypassBackoff && result.kind === 'backoff' ? await executeThroughCell(input) : result; + let current = target; + while (true) { + let superseded = false; + try { + const result = await executeThroughCell({ ...current, runtimeLocation, mode: 'explicit' }); + if (result.kind !== 'superseded' && result.mode !== 'automatic') return result; + if (result.kind === 'refreshed' && (!requiresDiscovery || result.discovered !== undefined)) { + const record = await getRepo().upstreams.getById(target.upstreamId); + if (record === null || record.configVersion !== target.configVersion) return { kind: 'superseded', mode: 'explicit' }; + await validateUpstreamProxyConfiguration(runtimeLocation, record); + return result; + } + superseded = result.kind === 'superseded'; + } catch (error) { + if (!(error instanceof ModelsRefreshUnavailableError) || error.mode !== 'automatic') throw error; + } + const record = await getRepo().upstreams.getById(target.upstreamId); + if (record === null || record.configVersion !== target.configVersion) return { kind: 'superseded', mode: 'explicit' }; + const next = modelsRefreshTarget(record); + if (superseded && next.cacheEpoch === current.cacheEpoch) return { kind: 'superseded', mode: 'explicit' }; + current = next; + } }; -export const scheduleModelsRefresh = ( - target: ModelsRefreshTarget, +export const createModelsRefreshScheduler = ( runtimeLocation: string | null, scheduler: BackgroundScheduler, -): void => { - scheduler(refreshModels(target, runtimeLocation, { bypassBackoff: false, includeDiscovered: false }).then(() => {})); +): ModelsRefreshScheduler => target => { + scheduler(refreshModels(target, runtimeLocation)); }; - -export const modelsRefreshExecutionError = (error: unknown): ModelsRefreshExecutionError => ({ - kind: error instanceof ProviderModelsUnavailableError ? 'provider-unavailable' : 'error', - message: errorMessage(error), -}); diff --git a/packages/gateway/src/index.ts b/packages/gateway/src/index.ts index ab4b1bb70..8b62a684a 100644 --- a/packages/gateway/src/index.ts +++ b/packages/gateway/src/index.ts @@ -2,7 +2,7 @@ export { app } from './app.ts'; export { initRepo } from './repo/index.ts'; export { FileDumpStore } from './repo/dump-store.ts'; export { SqlRepo } from './repo/sql.ts'; -export { MODEL_CATALOG_REVISION, modelsCacheGeneration } from './repo/models-cache-contract.ts'; +export { MODEL_CATALOG_REVISION } from './repo/models-cache-contract.ts'; export { initBackgroundSchedulerResolver } from './runtime/background.ts'; export { initExecutionCellNamespace } from './runtime/execution.ts'; export { initDumpBroker, initDumpStore } from './dump/registry.ts'; diff --git a/packages/gateway/src/repo/models-cache-contract.ts b/packages/gateway/src/repo/models-cache-contract.ts index 7ed72f7b4..5d3e19906 100644 --- a/packages/gateway/src/repo/models-cache-contract.ts +++ b/packages/gateway/src/repo/models-cache-contract.ts @@ -1,14 +1,4 @@ -import type { ModelsCacheGeneration, StoredUpstreamRecord } from './types.ts'; - // Persisted ProviderModel rows contain code-derived metadata as well as the // upstream response. Increment this whenever that derived catalog contract or // its serialization changes so older rows become cold across deployments. export const MODEL_CATALOG_REVISION = 5; - -// Refresh publication survives provider-managed state writes such as token -// rotation, but is fenced whenever static request inputs or egress policy change. -export const modelsCacheGeneration = ( - record: Pick, -): ModelsCacheGeneration => ({ - configVersion: record.configVersion, -}); diff --git a/packages/gateway/src/repo/sql.ts b/packages/gateway/src/repo/sql.ts index d8be48213..4e5cabefe 100644 --- a/packages/gateway/src/repo/sql.ts +++ b/packages/gateway/src/repo/sql.ts @@ -876,6 +876,12 @@ class SqlWebSearchConfigRepo implements WebSearchConfigRepo { // declaring the row unwritable, not a derived figure. export const UPSTREAM_STATE_WRITE_ATTEMPTS = 4; +const MODELS_CACHE_EPOCH_SQL = `CASE + WHEN json_extract(models_cache_json, '$.revision') = ${MODEL_CATALOG_REVISION} + THEN coalesce(json_extract(models_cache_json, '$.fetchedAt'), 0) + ELSE 0 +END`; + class SqlUpstreamRepo implements UpstreamRepo { constructor(private db: SqlDatabase) {} @@ -1058,16 +1064,16 @@ class SqlUpstreamRepo implements UpstreamRepo { } async publishModelsRefresh(input: ModelsRefreshSuccessInput): Promise { - const { id, generation, cache } = input; + const { id, configVersion, cacheEpoch, cache } = input; const result = await this.db - .prepare('UPDATE upstreams SET models_cache_json = ?, models_refresh_json = NULL WHERE id = ? AND config_version = ?') - .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, generation.configVersion) + .prepare(`UPDATE upstreams SET models_cache_json = ?, models_refresh_json = NULL WHERE id = ? AND config_version = ? AND ${MODELS_CACHE_EPOCH_SQL} = ?`) + .bind(encodeUpstreamModelsCache({ ...cache, lastError: null }), id, configVersion, cacheEpoch) .run(); return (result.meta.changes ?? 0) > 0; } async recordModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise { - const { id, generation, error, previousFailureCount, failedAt } = input; + const { id, configVersion, cacheEpoch, error, previousFailureCount, failedAt } = input; const failureCount = previousFailureCount + 1; const retryAt = modelsRefreshRetryAt(failedAt, previousFailureCount); // A cold failure remains immediately stale while preserving the error for @@ -1079,22 +1085,25 @@ class SqlUpstreamRepo implements UpstreamRepo { models_cache_json = CASE WHEN models_cache_json IS NULL THEN ? ELSE json_set(models_cache_json, '$.lastError', json(?)) END, models_refresh_json = json_object('failureCount', CAST(? AS INTEGER), 'retryAt', CAST(? AS INTEGER)) WHERE id = ? AND config_version = ? + AND ${MODELS_CACHE_EPOCH_SQL} = ? AND coalesce(json_extract(models_refresh_json, '$.failureCount'), 0) = ?`, ) - .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, generation.configVersion, previousFailureCount) + .bind(coldFailure, JSON.stringify(error), failureCount, retryAt, id, configVersion, cacheEpoch, previousFailureCount) .run(); return (result.meta.changes ?? 0) > 0; } async beginModelsRefresh(input: ModelsRefreshBeginInput): Promise { - const { id, generation, now, bypassBackoff } = input; + const { id, configVersion, cacheEpoch, now, bypassBackoff } = input; const row = await this.db.prepare( `SELECT coalesce(json_extract(models_refresh_json, '$.failureCount'), 0) AS failure_count, coalesce(json_extract(models_refresh_json, '$.retryAt'), 0) AS retry_at - FROM upstreams WHERE id = ? AND config_version = ?`, - ).bind(id, generation.configVersion).first<{ failure_count: number; retry_at: number }>(); - if (row === null) return { kind: 'generation-mismatch' }; + FROM upstreams + WHERE id = ? AND config_version = ? + AND ${MODELS_CACHE_EPOCH_SQL} = ?`, + ).bind(id, configVersion, cacheEpoch).first<{ failure_count: number; retry_at: number }>(); + if (row === null) return { kind: 'superseded' }; if (!bypassBackoff && row.retry_at > now) return { kind: 'backoff' }; return { kind: 'ready', failureCount: row.failure_count }; } diff --git a/packages/gateway/src/repo/types.ts b/packages/gateway/src/repo/types.ts index 1dfe86785..7c9096ac2 100644 --- a/packages/gateway/src/repo/types.ts +++ b/packages/gateway/src/repo/types.ts @@ -3,8 +3,8 @@ import type { AgentSetupRepository } from '@floway-dev/agent-setup'; import type { AliasSelection, AliasTarget, AnnouncedMetadata, BillingMetric, DecimalString, ModelKind, PricingSelector } from '@floway-dev/protocols/common'; import type { PerformanceTelemetryContext, UpstreamModelsCache, UpstreamRecord } from '@floway-dev/provider'; -// Persistence-owned catalog generation. Provider config, flag overrides, and -// catalog transport advance it; runtime state and non-model metadata do not. +// Provider config, flag overrides, and catalog transport advance this version; +// runtime state and non-model metadata do not. export type StoredUpstreamRecord = UpstreamRecord & { configVersion: number }; export interface ApiKey { @@ -367,39 +367,30 @@ export interface UpstreamRepo { recordModelsRefreshFailure(input: ModelsRefreshFailureInput): Promise; } -export interface ModelsRefreshBeginInput { +export interface ModelsRefreshIdentity { id: string; - generation: ModelsCacheGeneration; + configVersion: number; + cacheEpoch: number; +} + +export interface ModelsRefreshBeginInput extends ModelsRefreshIdentity { now: number; bypassBackoff: boolean; } -export interface ModelsRefreshSuccessInput { - id: string; - generation: ModelsCacheGeneration; +export interface ModelsRefreshSuccessInput extends ModelsRefreshIdentity { cache: Omit; } -export interface ModelsRefreshFailureInput { - id: string; - generation: ModelsCacheGeneration; +export interface ModelsRefreshFailureInput extends ModelsRefreshIdentity { error: NonNullable; previousFailureCount: number; failedAt: number; } -export interface ModelsRefreshReady { - kind: 'ready'; - failureCount: number; -} - -export type ModelsRefreshBeginResult = ModelsRefreshReady +export type ModelsRefreshBeginResult = { kind: 'ready'; failureCount: number } | { kind: 'backoff' } - | { kind: 'generation-mismatch' }; - -export interface ModelsCacheGeneration { - configVersion: number; -} + | { kind: 'superseded' }; export interface ProxyRecord { id: string; diff --git a/packages/gateway/src/scheduled/models-refresh.ts b/packages/gateway/src/scheduled/models-refresh.ts index e3620c3d5..b2f24c0bd 100644 --- a/packages/gateway/src/scheduled/models-refresh.ts +++ b/packages/gateway/src/scheduled/models-refresh.ts @@ -1,14 +1,15 @@ -import { modelsRefreshTarget, scheduleModelsRefresh } from '../execution/models-refresh.ts'; +import { createModelsRefreshScheduler, modelsRefreshTarget } from '../execution/models-refresh.ts'; import { getRepo } from '../repo/index.ts'; import { hasLocationIndependentEgress } from '../repo/proxy-fallback-list.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; export const scheduleModelsCacheRefreshes = async (runtimeLocation: string | null, scheduler: BackgroundScheduler): Promise => { + const scheduleRefresh = createModelsRefreshScheduler(runtimeLocation, scheduler); const upstreams = (await getRepo().upstreams.list()).filter(upstream => upstream.enabled && (runtimeLocation !== null || hasLocationIndependentEgress(upstream.proxyFallbackList))); for (const upstream of upstreams) { try { - scheduleModelsRefresh(modelsRefreshTarget(upstream), runtimeLocation, scheduler); + scheduleRefresh(modelsRefreshTarget(upstream)); } catch (error) { console.error(`[scheduled] models.refresh failed for ${upstream.id}`, error); }